From 8af33c80c9a4a783f026cb983e2f97f5c2f9bfa4 Mon Sep 17 00:00:00 2001 From: Mick van Dijke <12992260+mickvandijke@users.noreply.github.com> Date: Mon, 3 Aug 2026 17:42:03 +0200 Subject: [PATCH 01/31] feat: add direct browser public file client --- .gitignore | 2 + README.md | 26 +- .../ADR-0003-direct-browser-read-client.md | 115 +++ web/README.md | 94 ++ web/index.html | 113 +++ web/package-lock.json | 815 ++++++++++++++++++ web/package.json | 19 + web/src/file.js | 174 ++++ web/src/file.test.js | 106 +++ web/src/main.js | 247 ++++++ web/src/manifest.js | 117 +++ web/src/manifest.test.js | 65 ++ web/src/protocol.js | 426 +++++++++ web/src/protocol.test.js | 223 +++++ web/src/style.css | 219 +++++ 15 files changed, 2760 insertions(+), 1 deletion(-) create mode 100644 docs/adr/ADR-0003-direct-browser-read-client.md create mode 100644 web/README.md create mode 100644 web/index.html create mode 100644 web/package-lock.json create mode 100644 web/package.json create mode 100644 web/src/file.js create mode 100644 web/src/file.test.js create mode 100644 web/src/main.js create mode 100644 web/src/manifest.js create mode 100644 web/src/manifest.test.js create mode 100644 web/src/protocol.js create mode 100644 web/src/protocol.test.js create mode 100644 web/src/style.css diff --git a/.gitignore b/.gitignore index fc94dea5..b71f648b 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,6 @@ /target +/web/node_modules/ +/web/dist/ .cargo/config.toml .claude/plans/ .claude/scheduled_tasks.lock diff --git a/README.md b/README.md index 0f02eeb9..4725a7fe 100644 --- a/README.md +++ b/README.md @@ -4,13 +4,37 @@ A unified CLI and Rust library for storing data on the Autonomi decentralized ne ## Overview -This project provides two crates: +This project provides two Rust crates and a browser client: - **ant-core** — A headless Rust library containing all business logic: data storage/retrieval with self-encryption and EVM payments, node lifecycle management, and local devnet tooling. Designed to be consumed by any frontend (CLI, GUI, AI agents, REST clients). - **ant-cli** — A thin CLI binary (`ant`) built on `ant-core`. +- **web** — A direct WebTransport client and test site. It performs browser-side + closest-node lookup, reconstructs complete public self-encrypted files, and + saves them without a data gateway. Data on Autonomi is **content-addressed**. Files are split into encrypted chunks (via [self-encryption](https://en.wikipedia.org/wiki/Convergent_encryption)), each stored at an XOR address derived from its content. A `DataMap` tracks which chunks belong to a file. Payments for storage are made on an EVM-compatible blockchain (Arbitrum). +### Direct browser testnet client + +The browser client works with an ADR-0009-enabled `ant-node` checkout. Start +the browser-enabled node devnet, then run the site: + +```bash +# In ant-node-web-support +cargo run --features webtransport-poc --bin ant-devnet -- \ + --preset minimal --base-port 23000 \ + --webtransport --webtransport-base-port 24000 \ + --serve-port 25000 --enable-logging + +# In ant-client-web-support/web +npm ci +npm run dev +``` + +Open `http://127.0.0.1:5173`; the default public test file and direct node +endpoints are loaded from the local browser manifest. See [web/README.md](web/README.md) +for the protocol flow and LAN configuration. + ## Installation ### Linux / macOS diff --git a/docs/adr/ADR-0003-direct-browser-read-client.md b/docs/adr/ADR-0003-direct-browser-read-client.md new file mode 100644 index 00000000..b44c7a4d --- /dev/null +++ b/docs/adr/ADR-0003-direct-browser-read-client.md @@ -0,0 +1,115 @@ +# ADR-0003: Direct browser read client over WebTransport + +- **Status:** Proposed +- **Date:** 2026-08-03 +- **Decision owners:** +- **Reviewers:** +- **Supersedes:** none +- **Superseded by:** none +- **Related:** ant-node ADR-0009; W3C WebTransport + +## Context + +The Autonomi web client must perform closest-node lookup and immutable-data +download itself. Sending those operations through an HTTP application gateway +would make the gateway an availability, privacy, and bandwidth chokepoint. +Browsers cannot use the native Saorsa QUIC protocol, but they can establish +WebTransport sessions with browser-compatible node listeners. + +This repository owns the client and UI side of that split. Nodes own transport +termination, local DHT answers, storage reads, endpoint records, and testnet +bootstrap-manifest production under ant-node ADR-0009. + +## Decision Drivers + +- File bytes must travel directly from a storage node to the browser. +- The browser must own iterative XOR lookup rather than ask a gateway to do it. +- Self-signed node certificates must be authenticated through explicit hashes. +- Downloaded immutable content must be verified before it is exposed to users. +- Local testnets need a reproducible bootstrap and default-file workflow. + +## Considered Options + +1. **Use the daemon REST API as a data gateway.** Rejected for lookup and file + bytes because it would not exercise a full browser client. +2. **Compile the complete native Rust client to WebAssembly.** Deferred because + the native transport, EVM, and filesystem dependency graph is not currently + browser-compatible. +3. **Implement a narrow JavaScript WebTransport read client (chosen).** It maps + directly to the versioned browser node protocol and keeps the application + boundary small enough to audit. + +## Decision + +The `web/` package will implement the direct browser read client: + +- load a versioned browser bootstrap manifest containing node URLs, + certificate hashes, peer IDs, and published immutable-file metadata; +- authenticate every seed and discovered endpoint by certificate hash and + verify its `HELLO` peer ID; +- perform iterative `FIND_NODE` queries using 256-bit XOR ordering, `K = 20`, + and `ALPHA = 3`; +- query closest direct endpoints with `GET_CHUNK`, retrying `not_found` and + unavailable nodes without routing bytes through the manifest service; +- fetch the public MessagePack DataMap and every resolved encrypted data chunk; +- reconstruct the file with the native `self_encryption 0.36` BLAKE3 KDF, + ChaCha20-Poly1305 authentication, and Brotli decompression; +- verify encrypted-record addresses, per-chunk plaintext hashes and sizes, and + the final whole-file BLAKE3 hash before allowing a save; +- expose a small test site that loads the local testnet manifest, displays the + startup-published file, and downloads it through the browser save flow. + +The local browser manifest is bootstrap metadata, not a gateway. Production +clients will replace its unsigned endpoint list with the ML-DSA-signed records +defined by ant-node ADR-0009. + +For the local vertical slice, the bootstrap manifest carries a resolved JSON +view of the public root DataMap alongside its ordinary on-network DataMap +address. The browser still fetches and verifies that public DataMap record and +all file bytes directly from nodes. Production discovery must replace this +unsigned resolved view with parsing and validation of the signed/on-network +metadata chain. + +## Consequences + +### Positive + +- Lookup and data transfer remain decentralized at the application layer. +- A local five-node testnet can validate multiple direct node connections, + certificate pins, lookup convergence, fallback, and content verification. +- The browser protocol is independent of native Rust serialization details. + +### Negative / Trade-offs + +- The current client reconstructs files in memory and the local launcher caps + public files at 64 MiB; streaming-to-disk is not yet implemented. +- JavaScript lookup behavior must remain aligned with native Kademlia rules. +- Certificate and endpoint verification adds bootstrap-record lifecycle work. + +### Neutral / Operational + +- The manifest HTTP service carries only small bootstrap metadata. +- WebTransport still requires a secure browser context; localhost qualifies + for development. +- Node and client repositories must run compatible browser protocol versions. + +## Validation + +- Unit tests cover fixed-width identifiers, XOR ordering, response framing, + manifest validation, and BLAKE3 mismatch rejection. +- The browser production bundle builds without Node-specific runtime APIs. +- A fixed vector generated by native `self_encryption 0.36` verifies browser + KDF, authenticated decryption, Brotli reconstruction, and tamper rejection. +- A live node integration test starts five WebTransport-enabled nodes, + publishes a public DataMap and encrypted chunks, connects with the advertised + certificate hash, retrieves every record, and reconstructs the exact file. +- Before acceptance, run interactive tests on current Chrome, Firefox, and + Safari and add shared lookup convergence vectors with the native client. +- Revisit this decision when streaming file reconstruction or production signed + endpoint discovery is implemented. + +## Notes for AI-assisted work + +AI tools may help draft this ADR, but **must not mark it Accepted without human +review**. Accepted ADRs are immutable: create a new superseding ADR rather than +editing an Accepted ADR. diff --git a/web/README.md b/web/README.md new file mode 100644 index 00000000..dbbdaf20 --- /dev/null +++ b/web/README.md @@ -0,0 +1,94 @@ +# Autonomi direct browser client + +This web application is the browser-facing client for ADR-0009. It loads a +local testnet bootstrap manifest, connects directly to storage nodes over +WebTransport, performs the XOR closest-node lookup in JavaScript, retrieves a +public DataMap and every encrypted file chunk, reconstructs the complete file, +and verifies its whole-file BLAKE3 hash before saving it. + +The node-side WebTransport listener and testnet manifest API live in the +`ant-node-web-support` sibling repository. No HTTP gateway performs lookup or +proxies file bytes. + +## Requirements + +- Rust 1.88 or newer for the node's optional `wtransport` dependency. +- Node.js 20.19+ or 22.12+. +- A current browser implementing WebTransport certificate hashes. + +## Run the browser-enabled testnet + +From `ant-node-web-support`: + +```bash +cargo run --features webtransport-poc --bin ant-devnet -- \ + --preset minimal \ + --base-port 23000 \ + --webtransport \ + --webtransport-base-port 24000 \ + --serve-port 25000 \ + --enable-logging +``` + +This starts five native nodes on UDP 23000-23004 and five direct browser +listeners on UDP 24000-24004. It also: + +- self-encrypts the built-in `autonomi-browser-testnet.txt` and publishes its + encrypted chunks and public DataMap through the ordinary node PUT handler + using devnet-prepaid cache entries; +- exposes all direct node URLs and certificate pins at + `http://127.0.0.1:25000/api/browser-manifest.json`; +- includes the public DataMap address, plaintext BLAKE3 hash, resolved chunk + metadata, filename, size, and replica count in that manifest. + +Pass `--public-file /path/to/file` to publish another file instead. The built-in +file is generated as 5 MiB so the demo exercises whole-file reconstruction. A +custom file may be up to 64 MiB in this local launcher. + +## Run the site + +From this directory: + +```bash +npm ci +npm run dev +``` + +Open `http://127.0.0.1:5173`. The page automatically loads the testnet +manifest from port 25000 and fills in the default file: + +1. **Load testnet** refreshes the manifest and direct endpoint catalog. +2. **Connect** performs a pinned WebTransport `HELLO` with the first node. +3. **Find closest** runs the iterative lookup in the browser. +4. **Download and save file** opens the browser save flow, fetches the public + DataMap and encrypted chunks from direct closest storage nodes, reconstructs + the whole file, verifies BLAKE3, and retains a **Save again** link. + +The browser receives the DataMap and all encrypted file bytes from UDP +24000-24004 over HTTP/3, not from the manifest server on TCP 25000. The +manifest server is bootstrap metadata only. + +For a LAN test, start the node devnet with `--host `, serve Vite with +`npm run dev -- --host 0.0.0.0`, and add the exact site Origin with +`--webtransport-origin http://:5173`. Change the manifest URL in the +page to `http://:25000/api/browser-manifest.json`. + +## Verify the client + +```bash +npm test +npm run build +``` + +The tests cover fixed-width IDs, XOR ordering, response framing, browser +manifest validation, and a native `self_encryption 0.36` compatibility vector. +Cross-repository live verification additionally starts the node testnet, +downloads all public-file records through WebTransport, and reconstructs the +original bytes. + +## Current boundary + +The local testnet manifest is intentionally unsigned bootstrap material. A +production deployment still needs ML-DSA-signed endpoint records, certificate +overlap/rotation, network dissemination, relayed WebTransport, and production +traffic quotas as specified by ADR-0009. diff --git a/web/index.html b/web/index.html new file mode 100644 index 00000000..9578a3c2 --- /dev/null +++ b/web/index.html @@ -0,0 +1,113 @@ + + + + + + + Autonomi WebTransport PoC + + + +
+
+

ADR-0009 · interoperability proof

+

Direct browser client

+

+ Connect directly to an ant-node, query its local closest-node view, + reconstruct a complete public file, and save it from the browser. +

+
+ +
+

Local testnet

+ +
+ + Not loaded +
+ +
+ +
+

Direct node endpoint

+

+ Populated from the testnet manifest. It can also be entered manually. +

+ + +
+ + Disconnected +
+
+ +
+

Browser-side closest lookup

+ +
+ + +
+
+ +
+

Direct public file download

+

+ Fetches the public DataMap and every encrypted file chunk directly, + reconstructs and verifies the complete file, then opens the browser save flow. +

+ +
+ + + +
+
+ +
+

Protocol log

+

+      
+
+ + diff --git a/web/package-lock.json b/web/package-lock.json new file mode 100644 index 00000000..8f5fd2f9 --- /dev/null +++ b/web/package-lock.json @@ -0,0 +1,815 @@ +{ + "name": "ant-client-web", + "version": "0.1.0", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "ant-client-web", + "version": "0.1.0", + "dependencies": { + "@noble/ciphers": "2.2.0", + "@noble/hashes": "2.2.0", + "brotli-dec-wasm": "2.3.2" + }, + "devDependencies": { + "vite": "8.2.0" + } + }, + "node_modules/@noble/ciphers": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/@noble/ciphers/-/ciphers-2.2.0.tgz", + "integrity": "sha512-Z6pjIZ/8IJcCGzb2S/0Px5J81yij85xASuk1teLNeg75bfT07MV3a/O2Mtn1I2se43k3lkVEcFaR10N4cgQcZA==", + "license": "MIT", + "engines": { + "node": ">= 20.19.0" + }, + "funding": { + "url": "https://paulmillr.com/funding/" + } + }, + "node_modules/@noble/hashes": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/@noble/hashes/-/hashes-2.2.0.tgz", + "integrity": "sha512-IYqDGiTXab6FniAgnSdZwgWbomxpy9FtYvLKs7wCUs2a8RkITG+DFGO1DM9cr+E3/RgADRpFjrKVaJ1z6sjtEg==", + "license": "MIT", + "engines": { + "node": ">= 20.19.0" + }, + "funding": { + "url": "https://paulmillr.com/funding/" + } + }, + "node_modules/@oxc-project/types": { + "version": "0.142.0", + "resolved": "https://registry.npmjs.org/@oxc-project/types/-/types-0.142.0.tgz", + "integrity": "sha512-7W+2q5AKQVU36fkaryontrHn3YDt1RyUYXatw9i5H8ocYe2sPKSFB6eS8WNPeRKiN1qAWWZUPm7gwFzJGrccqQ==", + "dev": true, + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/Boshen" + } + }, + "node_modules/@rolldown/binding-android-arm64": { + "version": "1.2.2", + "resolved": "https://registry.npmjs.org/@rolldown/binding-android-arm64/-/binding-android-arm64-1.2.2.tgz", + "integrity": "sha512-l7x215OGvo1s52JWmR8U/DAVzEDWBCIbTm28aeJV/WDTSHgcKXaZTuBT0hJMs5NggilfJTW3clZVvd24yfKJxA==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-darwin-arm64": { + "version": "1.2.2", + "resolved": "https://registry.npmjs.org/@rolldown/binding-darwin-arm64/-/binding-darwin-arm64-1.2.2.tgz", + "integrity": "sha512-9u9Xv6c1AJZT0FfwH5vrMG5Jjcwhc1MlyrPu0XfTqkzsmqfks2M6W/o5XwAJgVVN/jHpqqngC1WevHKKTIUtIA==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-darwin-x64": { + "version": "1.2.2", + "resolved": "https://registry.npmjs.org/@rolldown/binding-darwin-x64/-/binding-darwin-x64-1.2.2.tgz", + "integrity": "sha512-9W1mbGZAfW3oqd85bhBkmpyHCCzL1TeG/zFFP3vg7b0rlly8cxOcre5nXwz+LHazCwac2MNWgPdPCHndABjpWQ==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-freebsd-x64": { + "version": "1.2.2", + "resolved": "https://registry.npmjs.org/@rolldown/binding-freebsd-x64/-/binding-freebsd-x64-1.2.2.tgz", + "integrity": "sha512-0p1lhiCSCyaerFwtrdZQUx7NqGk6LQnaRKWX7tFQqwQgvX0rjM15cIkm3pax1UpEakK14C4mOxx/jSqCBdBRqQ==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-linux-arm-gnueabihf": { + "version": "1.2.2", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-arm-gnueabihf/-/binding-linux-arm-gnueabihf-1.2.2.tgz", + "integrity": "sha512-e+cOJXrJ2L3zx6YzqPg+f6Wbk3V1cKB8bOhbaYdVYN3DdquzNdRAmrbETz1qnt5yp/c7JNlNjmITiA2cVneQ7w==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-linux-arm64-gnu": { + "version": "1.2.2", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-arm64-gnu/-/binding-linux-arm64-gnu-1.2.2.tgz", + "integrity": "sha512-JsSMsj6sNat/MuhG5fnBD7QgbtpHKVe30x5/bAVirDHdhoQRXJkF6xc0Jqk8O4fiCUQAzMOoH9wZi3m60c8wtg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-linux-arm64-musl": { + "version": "1.2.2", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-arm64-musl/-/binding-linux-arm64-musl-1.2.2.tgz", + "integrity": "sha512-B5G/zJdHaoJn9vD50eGHWkiWfmq8Uhi3IiLPJTzmZTrAalk1bztUikSXo0qga18ibE0IXboyeMUnhPjhAJ45wQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-linux-ppc64-gnu": { + "version": "1.2.2", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-ppc64-gnu/-/binding-linux-ppc64-gnu-1.2.2.tgz", + "integrity": "sha512-6mC/awzKka8W6EoekjegpfGkjz8jXWDX63pqu/HYVpyKtZfu65Jsh4QAH3Kej3CAv/c1oGX7psTmFEbr0mDxLA==", + "cpu": [ + "ppc64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-linux-s390x-gnu": { + "version": "1.2.2", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-s390x-gnu/-/binding-linux-s390x-gnu-1.2.2.tgz", + "integrity": "sha512-412MX9fJLdA1IK28EZnc8jYv2HRTleOZgfLQumJ5zy7OeJLZlg/CETwFaXjNmGVxG51cFHpKLqb5LKvBC+HsHA==", + "cpu": [ + "s390x" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-linux-x64-gnu": { + "version": "1.2.2", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-x64-gnu/-/binding-linux-x64-gnu-1.2.2.tgz", + "integrity": "sha512-Q/+HI/ToJafZ1iCqGgVQXUEkIjufHCTF0gBQ2a5o3cg7GJ2h0qyq3nvvSmU+bGda2/7ygXpTY4TM6gO9OhQ0ZA==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-linux-x64-musl": { + "version": "1.2.2", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-x64-musl/-/binding-linux-x64-musl-1.2.2.tgz", + "integrity": "sha512-ZKp/w41n6wCvxzxQHtQSbuphfX3Y4cCvbjkKHusrLx4lh+JWLTU7StSltO/DKARISzbj368d+qUaCjI8K2wzXw==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-openharmony-arm64": { + "version": "1.2.2", + "resolved": "https://registry.npmjs.org/@rolldown/binding-openharmony-arm64/-/binding-openharmony-arm64-1.2.2.tgz", + "integrity": "sha512-pxE6xD4KS3eAROkKK5yrhB9/3+vhlhVGMvlQLbdpzrBGDbKrnzx3RLwPaHvasLo6jgaiBLn7e04Df9C4tYhjmA==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openharmony" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-win32-arm64-msvc": { + "version": "1.2.2", + "resolved": "https://registry.npmjs.org/@rolldown/binding-win32-arm64-msvc/-/binding-win32-arm64-msvc-1.2.2.tgz", + "integrity": "sha512-4MqEue5re+xIZzAWsB8sj0P1kqZySWqIuN4t6QaIO/YA6SFwySOLruvWQFzfmqk8LBK2P30KCSJwf6mJCZZ5/A==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-win32-x64-msvc": { + "version": "1.2.2", + "resolved": "https://registry.npmjs.org/@rolldown/binding-win32-x64-msvc/-/binding-win32-x64-msvc-1.2.2.tgz", + "integrity": "sha512-NweNxxD0Nf9t8v7kodun45Ijp3EIwYY+uydPP6qBEYvfBqhIjN6dZMzlQja3tqX/aLs3F3Uz+AxDpKgRhpOZQg==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/pluginutils": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/@rolldown/pluginutils/-/pluginutils-1.0.1.tgz", + "integrity": "sha512-2j9bGt5Jh8hj+vPtgzPtl72j0yRxHAyumoo6TNfAjsLB04UtpSvPbPcDcBMxz7n+9CYB0c1GxQFxYRg2jimqGw==", + "dev": true, + "license": "MIT" + }, + "node_modules/brotli-dec-wasm": { + "version": "2.3.2", + "resolved": "https://registry.npmjs.org/brotli-dec-wasm/-/brotli-dec-wasm-2.3.2.tgz", + "integrity": "sha512-5H+k8eVLIJY6B4olN2HP9QzJAxcplf0jV7mWnkpxvOSeUE9Npg3dQ2pgLn30a9MUFHNro1iSmtdu6VNdlb+TIw==", + "license": "MIT OR Apache-2.0" + }, + "node_modules/detect-libc": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/detect-libc/-/detect-libc-2.1.2.tgz", + "integrity": "sha512-Btj2BOOO83o3WyH59e8MgXsxEQVcarkUOpEYrubB0urwnN10yQ364rsiByU11nZlqWYZm05i/of7io4mzihBtQ==", + "dev": true, + "license": "Apache-2.0", + "engines": { + "node": ">=8" + } + }, + "node_modules/fdir": { + "version": "6.5.0", + "resolved": "https://registry.npmjs.org/fdir/-/fdir-6.5.0.tgz", + "integrity": "sha512-tIbYtZbucOs0BRGqPJkshJUYdL+SDH7dVM8gjy+ERp3WAUjLEFJE+02kanyHtwjWOnwrKYBiwAmM0p4kLJAnXg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=12.0.0" + }, + "peerDependencies": { + "picomatch": "^3 || ^4" + }, + "peerDependenciesMeta": { + "picomatch": { + "optional": true + } + } + }, + "node_modules/fsevents": { + "version": "2.3.3", + "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.3.tgz", + "integrity": "sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw==", + "dev": true, + "hasInstallScript": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^8.16.0 || ^10.6.0 || >=11.0.0" + } + }, + "node_modules/lightningcss": { + "version": "1.33.0", + "resolved": "https://registry.npmjs.org/lightningcss/-/lightningcss-1.33.0.tgz", + "integrity": "sha512-WkUDrojuJs0xkgGf2udWxa3yGBRxPtxUkB79i6aCZLRgc7PM8fZe9TosfPDcvEpQZbuFASnHYmRLBLUbmLOIIA==", + "dev": true, + "license": "MPL-2.0", + "dependencies": { + "detect-libc": "^2.0.3" + }, + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + }, + "optionalDependencies": { + "lightningcss-android-arm64": "1.33.0", + "lightningcss-darwin-arm64": "1.33.0", + "lightningcss-darwin-x64": "1.33.0", + "lightningcss-freebsd-x64": "1.33.0", + "lightningcss-linux-arm-gnueabihf": "1.33.0", + "lightningcss-linux-arm64-gnu": "1.33.0", + "lightningcss-linux-arm64-musl": "1.33.0", + "lightningcss-linux-x64-gnu": "1.33.0", + "lightningcss-linux-x64-musl": "1.33.0", + "lightningcss-win32-arm64-msvc": "1.33.0", + "lightningcss-win32-x64-msvc": "1.33.0" + } + }, + "node_modules/lightningcss-android-arm64": { + "version": "1.33.0", + "resolved": "https://registry.npmjs.org/lightningcss-android-arm64/-/lightningcss-android-arm64-1.33.0.tgz", + "integrity": "sha512-gEpRTalKdosp4Bb8qWtc2iOgE5SeIHlpS1up9bFq2wAyYhl1UdTObYiHe98zEM9SQvSoqQZ1IQD0JNpg3Ml5pg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MPL-2.0", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/lightningcss-darwin-arm64": { + "version": "1.33.0", + "resolved": "https://registry.npmjs.org/lightningcss-darwin-arm64/-/lightningcss-darwin-arm64-1.33.0.tgz", + "integrity": "sha512-Sciaz8eenNTKn9b3t7+xr0ipTp9YxKQY4npwQ3mrRuL0BAVHBLyZxofhaKBAVtzmtRZ/zTyo0/to4B1uWG/Djg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MPL-2.0", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/lightningcss-darwin-x64": { + "version": "1.33.0", + "resolved": "https://registry.npmjs.org/lightningcss-darwin-x64/-/lightningcss-darwin-x64-1.33.0.tgz", + "integrity": "sha512-Z5UPAxzrjlWNNyGy6i65cJzzvgJ5D3T6wMvs+gWpY9d7qRhANrxqAp6LhxIgZhWEw18RfJTGcRxjuLIBr+m8XQ==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MPL-2.0", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/lightningcss-freebsd-x64": { + "version": "1.33.0", + "resolved": "https://registry.npmjs.org/lightningcss-freebsd-x64/-/lightningcss-freebsd-x64-1.33.0.tgz", + "integrity": "sha512-QQM/Ti/hQajJwCY+RiWuCZ9sdtI/XQk7nDK5vC8kkdwixezOlDgvDx7+RT+QjK6FcFT4MpsuoBnHIo/O3StRRg==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MPL-2.0", + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/lightningcss-linux-arm-gnueabihf": { + "version": "1.33.0", + "resolved": "https://registry.npmjs.org/lightningcss-linux-arm-gnueabihf/-/lightningcss-linux-arm-gnueabihf-1.33.0.tgz", + "integrity": "sha512-N7FVBe6iS24MlM6R/4RBTxGhQheZGs7tiQ9U32UtF75NzP5Q7xWPRqLBCKxlRQRk3rY1jCIPLzx7WzOhuUIRLQ==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MPL-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/lightningcss-linux-arm64-gnu": { + "version": "1.33.0", + "resolved": "https://registry.npmjs.org/lightningcss-linux-arm64-gnu/-/lightningcss-linux-arm64-gnu-1.33.0.tgz", + "integrity": "sha512-j2v/itmy4HlNxlc6voKXYgBqNi0Ng2LShg4z7GufpEgs05P+2suBVyi9I6YHq5uoVFx9ETin3eCEhLVyXGQnKg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MPL-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/lightningcss-linux-arm64-musl": { + "version": "1.33.0", + "resolved": "https://registry.npmjs.org/lightningcss-linux-arm64-musl/-/lightningcss-linux-arm64-musl-1.33.0.tgz", + "integrity": "sha512-yiO5ROMuYQgXbC60yjZU5CYSFZGKXL0HFATXt9mHJn1+zW55oCtMI9NfcVhYLMFDL7gV7oBPon/EmMMGg2OvtQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MPL-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/lightningcss-linux-x64-gnu": { + "version": "1.33.0", + "resolved": "https://registry.npmjs.org/lightningcss-linux-x64-gnu/-/lightningcss-linux-x64-gnu-1.33.0.tgz", + "integrity": "sha512-ar+Ju7LmcN0Jo4FpL4hpFybwNG9/3A/Br5KW2n2jyODg3MEZXaDYADdemoNS+BDNfMgKvylJLj4S5tyRActuAg==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MPL-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/lightningcss-linux-x64-musl": { + "version": "1.33.0", + "resolved": "https://registry.npmjs.org/lightningcss-linux-x64-musl/-/lightningcss-linux-x64-musl-1.33.0.tgz", + "integrity": "sha512-RYiYbkokw0trfKqqzfF55lginwEPrD3OJDfTuJzFs1MK6iFnDenaz1fqLLtX4ITG3OktJQXOeTaw1awrBAlZPw==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MPL-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/lightningcss-win32-arm64-msvc": { + "version": "1.33.0", + "resolved": "https://registry.npmjs.org/lightningcss-win32-arm64-msvc/-/lightningcss-win32-arm64-msvc-1.33.0.tgz", + "integrity": "sha512-1K+MPfLSFVpphzpdbfkhlWk6wBrTObBzS2T6db10PNOZgR9GoVsAWzwNyuhUYYbTp23j+4RrncfujZ4uAzXvwA==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MPL-2.0", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/lightningcss-win32-x64-msvc": { + "version": "1.33.0", + "resolved": "https://registry.npmjs.org/lightningcss-win32-x64-msvc/-/lightningcss-win32-x64-msvc-1.33.0.tgz", + "integrity": "sha512-OlEICDx/Xl0FqSp4bry8zFnCvGpig3Gl4gCquvYwHuqJKEC1+n9NgDniFvqHGmMv1ZkqDJrDqKKSykTDX+ehuA==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MPL-2.0", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/nanoid": { + "version": "3.3.17", + "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.17.tgz", + "integrity": "sha512-xQLf0A3HOMlgHq0n247/LRuAOYmB7dXJ/DvAxGvsSBij45XtBSmQycu+F8ODbHwns/XyFZagyL1+J0Offw1E0g==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "license": "MIT", + "bin": { + "nanoid": "bin/nanoid.cjs" + }, + "engines": { + "node": "^10 || ^12 || ^13.7 || ^14 || >=15.0.1" + } + }, + "node_modules/picocolors": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/picocolors/-/picocolors-1.1.1.tgz", + "integrity": "sha512-xceH2snhtb5M9liqDsmEw56le376mTZkEX/jEb/RxNFyegNul7eNslCXP9FDj/Lcu0X8KEyMceP2ntpaHrDEVA==", + "dev": true, + "license": "ISC" + }, + "node_modules/picomatch": { + "version": "4.0.5", + "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.5.tgz", + "integrity": "sha512-RvwwcruNjI1ncT5xRakeyS9Lf8lcItv34KD+aif+VH9kduAyfYBipGh12274xtenIPZ119/R9BdTBa8gAwSh0A==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/jonschlinkert" + } + }, + "node_modules/postcss": { + "version": "8.5.25", + "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.5.25.tgz", + "integrity": "sha512-DTPx3RWSSnWyzLxQnlH0rJP+EW5ekl16ZU4/psbIhA0e53kJfdgaN5vKM+xP7yJtXVu+nfdVFmlgFDEKAe4Pyw==", + "dev": true, + "funding": [ + { + "type": "opencollective", + "url": "https://opencollective.com/postcss/" + }, + { + "type": "tidelift", + "url": "https://tidelift.com/funding/github/npm/postcss" + }, + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "license": "MIT", + "dependencies": { + "nanoid": "^3.3.16", + "picocolors": "^1.1.1", + "source-map-js": "^1.2.1" + }, + "engines": { + "node": "^10 || ^12 || >=14" + } + }, + "node_modules/rolldown": { + "version": "1.2.2", + "resolved": "https://registry.npmjs.org/rolldown/-/rolldown-1.2.2.tgz", + "integrity": "sha512-opwpo1tQBAcpSUJDt94B7hhLNGOKjCdE//XXjeLrnx9b83bjnw45tXdg1b09yEw/VLFBJGZpwRULMmOZo7ol+A==", + "dev": true, + "license": "MIT", + "dependencies": { + "@oxc-project/types": "=0.142.0", + "@rolldown/pluginutils": "^1.0.0" + }, + "bin": { + "rolldown": "bin/cli.mjs" + }, + "engines": { + "node": "^20.19.0 || >=22.12.0" + }, + "optionalDependencies": { + "@rolldown/binding-android-arm64": "1.2.2", + "@rolldown/binding-darwin-arm64": "1.2.2", + "@rolldown/binding-darwin-x64": "1.2.2", + "@rolldown/binding-freebsd-x64": "1.2.2", + "@rolldown/binding-linux-arm-gnueabihf": "1.2.2", + "@rolldown/binding-linux-arm64-gnu": "1.2.2", + "@rolldown/binding-linux-arm64-musl": "1.2.2", + "@rolldown/binding-linux-ppc64-gnu": "1.2.2", + "@rolldown/binding-linux-s390x-gnu": "1.2.2", + "@rolldown/binding-linux-x64-gnu": "1.2.2", + "@rolldown/binding-linux-x64-musl": "1.2.2", + "@rolldown/binding-openharmony-arm64": "1.2.2", + "@rolldown/binding-win32-arm64-msvc": "1.2.2", + "@rolldown/binding-win32-x64-msvc": "1.2.2" + } + }, + "node_modules/source-map-js": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/source-map-js/-/source-map-js-1.2.1.tgz", + "integrity": "sha512-UXWMKhLOwVKb728IUtQPXxfYU+usdybtUrK/8uGE8CQMvrhOpwvzDBwj0QhSL7MQc7vIsISBG8VQ8+IDQxpfQA==", + "dev": true, + "license": "BSD-3-Clause", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/tinyglobby": { + "version": "0.2.17", + "resolved": "https://registry.npmjs.org/tinyglobby/-/tinyglobby-0.2.17.tgz", + "integrity": "sha512-wXR/dYpcqKmfWpEdZjiKJOwCNFndD0DMnrW/cYjVGttEkBfVgcLFHoNrlj47mjOVic9yyNu65alsgF4NQyTa2g==", + "dev": true, + "license": "MIT", + "dependencies": { + "fdir": "^6.5.0", + "picomatch": "^4.0.4" + }, + "engines": { + "node": ">=12.0.0" + }, + "funding": { + "url": "https://github.com/sponsors/SuperchupuDev" + } + }, + "node_modules/vite": { + "version": "8.2.0", + "resolved": "https://registry.npmjs.org/vite/-/vite-8.2.0.tgz", + "integrity": "sha512-pn+CFpM0lwDeKwmOq1ZaBK/9sjorZcgqxki6MbY/jPEVd9vichIlmlD4HmQ5wdP5EgqQCFRaACBxMC7uEGc6lQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "lightningcss": "^1.33.0", + "picomatch": "^4.0.5", + "postcss": "^8.5.23", + "rolldown": "~1.2.0", + "tinyglobby": "^0.2.17" + }, + "bin": { + "vite": "bin/vite.js" + }, + "engines": { + "node": "^20.19.0 || >=22.12.0" + }, + "funding": { + "url": "https://github.com/vitejs/vite?sponsor=1" + }, + "optionalDependencies": { + "fsevents": "~2.3.3" + }, + "peerDependencies": { + "@types/node": "^20.19.0 || >=22.12.0", + "@vitejs/devtools": "^0.4.0", + "esbuild": "^0.27.0 || ^0.28.0", + "jiti": ">=1.21.0", + "less": "^4.0.0", + "sass": "^1.70.0", + "sass-embedded": "^1.70.0", + "stylus": ">=0.54.8", + "sugarss": "^5.0.0", + "terser": "^5.16.0", + "tsx": "^4.8.1", + "yaml": "^2.4.2" + }, + "peerDependenciesMeta": { + "@types/node": { + "optional": true + }, + "@vitejs/devtools": { + "optional": true + }, + "esbuild": { + "optional": true + }, + "jiti": { + "optional": true + }, + "less": { + "optional": true + }, + "sass": { + "optional": true + }, + "sass-embedded": { + "optional": true + }, + "stylus": { + "optional": true + }, + "sugarss": { + "optional": true + }, + "terser": { + "optional": true + }, + "tsx": { + "optional": true + }, + "yaml": { + "optional": true + } + } + } + } +} diff --git a/web/package.json b/web/package.json new file mode 100644 index 00000000..19cad25c --- /dev/null +++ b/web/package.json @@ -0,0 +1,19 @@ +{ + "name": "ant-client-web", + "version": "0.1.0", + "private": true, + "type": "module", + "scripts": { + "dev": "vite --host 127.0.0.1 --port 5173", + "build": "vite build", + "test": "node --test" + }, + "dependencies": { + "@noble/ciphers": "2.2.0", + "@noble/hashes": "2.2.0", + "brotli-dec-wasm": "2.3.2" + }, + "devDependencies": { + "vite": "8.2.0" + } +} diff --git a/web/src/file.js b/web/src/file.js new file mode 100644 index 00000000..a857f567 --- /dev/null +++ b/web/src/file.js @@ -0,0 +1,174 @@ +import { chacha20poly1305 } from "@noble/ciphers/chacha.js"; +import { blake3 } from "@noble/hashes/blake3.js"; +import { bytesToHex, getChunkFromClosest, hexToBytes } from "./protocol.js"; + +const KDF_CONTEXT = new TextEncoder().encode("self_encryption/chunk/v2"); +const PAD_SIZE = 52; +const KEY_SIZE = 32; +const NONCE_SIZE = 12; +const DERIVED_SIZE = PAD_SIZE + KEY_SIZE + NONCE_SIZE; +const MAX_DOWNLOAD_CONCURRENCY = 6; + +function writeU64LittleEndian(target, offset, value) { + const numeric = BigInt(value); + new DataView(target.buffer, target.byteOffset, target.byteLength).setBigUint64( + offset, + numeric, + true, + ); +} + +function predecessorIndices(index, count) { + if (!Number.isSafeInteger(index) || index < 0 || index >= count || count < 3) { + throw new Error(`Invalid self-encryption chunk index ${index}/${count}`); + } + if (index === 0) return [count - 1, count - 2]; + if (index === 1) return [0, count - 1]; + return [index - 1, index - 2]; +} + +export function deriveChunkMaterial(chunk, sourceHashes, childLevel = 0) { + const [previous, previousPrevious] = predecessorIndices(chunk.index, sourceHashes.length); + const context = new Uint8Array(32 * 3 + 8 * 2); + context.set(sourceHashes[chunk.index], 0); + context.set(sourceHashes[previous], 32); + context.set(sourceHashes[previousPrevious], 64); + writeU64LittleEndian(context, 96, chunk.index); + writeU64LittleEndian(context, 104, childLevel); + + const derived = blake3(context, { context: KDF_CONTEXT, dkLen: DERIVED_SIZE }); + return { + pad: derived.slice(0, PAD_SIZE), + key: derived.slice(PAD_SIZE, PAD_SIZE + KEY_SIZE), + nonce: derived.slice(PAD_SIZE + KEY_SIZE), + }; +} + +export async function decryptSelfEncryptedChunk( + chunk, + encryptedContent, + sourceHashes, + childLevel = 0, + decompress = decompressBrotli, +) { + const { pad, key, nonce } = deriveChunkMaterial(chunk, sourceHashes, childLevel); + const ciphertext = new Uint8Array(encryptedContent.length); + for (let index = 0; index < encryptedContent.length; index += 1) { + ciphertext[index] = encryptedContent[index] ^ pad[index % pad.length]; + } + + let compressed; + try { + compressed = chacha20poly1305(key, nonce).decrypt(ciphertext); + } catch (error) { + throw new Error(`Chunk ${chunk.index} authentication failed`, { cause: error }); + } + + let plaintext; + try { + plaintext = await decompress(compressed); + } catch (error) { + throw new Error(`Chunk ${chunk.index} Brotli decompression failed`, { cause: error }); + } + if (plaintext.length !== chunk.src_size) { + throw new Error( + `Chunk ${chunk.index} reconstructed ${plaintext.length} bytes, expected ${chunk.src_size}`, + ); + } + const sourceHash = bytesToHex(blake3(plaintext)); + if (sourceHash !== chunk.src_hash) { + throw new Error( + `Chunk ${chunk.index} plaintext hash mismatch: expected ${chunk.src_hash}, received ${sourceHash}`, + ); + } + return plaintext; +} + +async function decompressBrotli(compressed) { + const { default: brotliPromise } = await import("brotli-dec-wasm"); + const brotli = await brotliPromise; + return brotli.decompress(compressed); +} + +async function mapWithConcurrency(items, concurrency, operation) { + const results = new Array(items.length); + let next = 0; + const workerCount = Math.min(items.length, concurrency); + await Promise.all( + Array.from({ length: workerCount }, async () => { + while (next < items.length) { + const index = next; + next += 1; + results[index] = await operation(items[index], index); + } + }), + ); + return results; +} + +export async function downloadPublicFile( + seedEndpoints, + file, + { + concurrency = 3, + onProgress = () => {}, + downloadChunk = getChunkFromClosest, + decompress = decompressBrotli, + } = {}, +) { + hexToBytes(file.address, 32); + hexToBytes(file.blake3, 32); + if (!Number.isSafeInteger(concurrency) || concurrency < 1) { + throw new Error("Download concurrency must be a positive integer"); + } + const boundedConcurrency = Math.min(concurrency, MAX_DOWNLOAD_CONCURRENCY); + + onProgress(`Fetching public DataMap ${file.address}`); + const dataMap = await downloadChunk(seedEndpoints, file.address, { onProgress }); + if (dataMap.content.length !== file.data_map_size) { + throw new Error( + `Public DataMap has ${dataMap.content.length} bytes, expected ${file.data_map_size}`, + ); + } + onProgress(`Verified public DataMap (${dataMap.content.length} bytes)`); + + const sourceHashes = file.chunks.map((chunk) => hexToBytes(chunk.src_hash, 32)); + const plaintextChunks = await mapWithConcurrency( + file.chunks, + boundedConcurrency, + async (chunk) => { + onProgress( + `Fetching encrypted file chunk ${chunk.index + 1}/${file.chunks.length} (${chunk.dst_hash})`, + ); + const downloaded = await downloadChunk(seedEndpoints, chunk.dst_hash, { + onProgress, + }); + const plaintext = await decryptSelfEncryptedChunk( + chunk, + downloaded.content, + sourceHashes, + 0, + decompress, + ); + onProgress(`Reconstructed file chunk ${chunk.index + 1}/${file.chunks.length}`); + return plaintext; + }, + ); + + const totalSize = plaintextChunks.reduce((total, chunk) => total + chunk.length, 0); + if (totalSize !== file.size) { + throw new Error(`Reconstructed file has ${totalSize} bytes, expected ${file.size}`); + } + const content = new Uint8Array(totalSize); + let offset = 0; + for (const chunk of plaintextChunks) { + content.set(chunk, offset); + offset += chunk.length; + } + const hash = bytesToHex(blake3(content)); + if (hash !== file.blake3) { + throw new Error(`Whole-file BLAKE3 mismatch: expected ${file.blake3}, received ${hash}`); + } + onProgress(`Verified complete ${file.name} as ${hash}`); + return { content, hash, dataMapNode: dataMap.node }; +} diff --git a/web/src/file.test.js b/web/src/file.test.js new file mode 100644 index 00000000..fbe46329 --- /dev/null +++ b/web/src/file.test.js @@ -0,0 +1,106 @@ +import assert from "node:assert/strict"; +import test from "node:test"; +import { brotliDecompressSync } from "node:zlib"; +import { blake3 } from "@noble/hashes/blake3.js"; +import { decryptSelfEncryptedChunk, downloadPublicFile } from "./file.js"; +import { bytesToHex, hexToBytes } from "./protocol.js"; + +// Generated by self_encryption 0.36 from +// "browser whole-file fixture\n" repeated 160 times. This is a compatibility +// vector, not a JavaScript-generated round trip. +const chunks = [ + { + index: 0, + dst_hash: "c024c6884a2f39be7ba07c3d9636efedeb94df7397fcd38bac5ae904643c5cc9", + src_hash: "a049deadbb9eb8ef8102a99698ec8484ba4ba9f1d82089af28b31d70783aae8b", + src_size: 1440, + encrypted: + "7508d7886030b903dce0178555d3df9115240fb3efe6029c4b57c05f84d82b85aedb898c09345c65a55dc32ff125e6af96ceab", + }, + { + index: 1, + dst_hash: "350a88e6eb0b2a3e774107a212a272b4191af69ca4366a4b91f5a1e5872c459a", + src_hash: "a15219e6c26233304a4e733946b97282c2d7ac270e6bd31b7ebdd22ee428cbab", + src_size: 1440, + encrypted: + "5244d1869387ef32f8bec83ff7227fac29d3963a198f271f4ec74b84c1f364bdf2f7964850ecb51c36e30deef5e96c440782e17401", + }, + { + index: 2, + dst_hash: "d73db5a8b0be3b571b40d2b80ff490fe45e135f1992c5863ecb78e25d00ceddb", + src_hash: "ab7ff601a29bd95610dea57dad100e54d53a837db06604afa3c2eaf482d2a82e", + src_size: 1440, + encrypted: + "601efca604bda1cdcde9e79722e5e71d9fcd5764c0fd52a9d0dbf6aeaec53fc462cbc01a6a372a2247ab7a847538ed8e623609", + }, +]; + +const decompress = (input) => new Uint8Array(brotliDecompressSync(input)); + +test("decrypts a native self_encryption 0.36 whole-file vector", async () => { + const sourceHashes = chunks.map((chunk) => hexToBytes(chunk.src_hash, 32)); + const plaintextChunks = await Promise.all( + chunks.map((chunk) => + decryptSelfEncryptedChunk(chunk, hexToBytes(chunk.encrypted), sourceHashes, 0, decompress), + ), + ); + const plaintext = new Uint8Array( + plaintextChunks.reduce((total, chunk) => total + chunk.length, 0), + ); + let offset = 0; + for (const chunk of plaintextChunks) { + plaintext.set(chunk, offset); + offset += chunk.length; + } + + assert.equal(new TextDecoder().decode(plaintext), "browser whole-file fixture\n".repeat(160)); + assert.equal( + bytesToHex(blake3(plaintext)), + "e0e422267ac59c56bf032d6d830035d343369d20147dd5f6b63351a29b015f22", + ); +}); + +test("rejects an authenticated self-encryption chunk after tampering", async () => { + const sourceHashes = chunks.map((chunk) => hexToBytes(chunk.src_hash, 32)); + const tampered = hexToBytes(chunks[0].encrypted); + tampered[0] ^= 1; + await assert.rejects( + decryptSelfEncryptedChunk(chunks[0], tampered, sourceHashes, 0, decompress), + /authentication failed/, + ); +}); + +test("downloads a public DataMap and reconstructs a complete saveable file", async () => { + const content = new TextEncoder().encode("browser whole-file fixture\n".repeat(160)); + const dataMapContent = new TextEncoder().encode("native public DataMap fixture"); + const dataMapAddress = bytesToHex(blake3(dataMapContent)); + const encryptedByAddress = new Map( + chunks.map((chunk) => [chunk.dst_hash, hexToBytes(chunk.encrypted)]), + ); + encryptedByAddress.set(dataMapAddress, dataMapContent); + const requested = []; + const downloadChunk = async (_seeds, address) => { + requested.push(address); + const bytes = encryptedByAddress.get(address); + if (!bytes) throw new Error(`No fixture record ${address}`); + return { content: bytes, node: { peer_id: "11".repeat(32) } }; + }; + + const result = await downloadPublicFile( + [], + { + name: "fixture.txt", + address: dataMapAddress, + size: content.length, + content_type: "text/plain", + blake3: bytesToHex(blake3(content)), + data_map_size: dataMapContent.length, + chunks, + }, + { downloadChunk, decompress }, + ); + + assert.deepEqual(result.content, content); + assert.equal(result.hash, bytesToHex(blake3(content))); + assert.deepEqual(new Set(requested), new Set([dataMapAddress, ...chunks.map((c) => c.dst_hash)])); +}); diff --git a/web/src/main.js b/web/src/main.js new file mode 100644 index 00000000..70536d5a --- /dev/null +++ b/web/src/main.js @@ -0,0 +1,247 @@ +import "./style.css"; +import { + BrowserNodeClient, + bytesToHex, + hexToBytes, + iterativeFindClosest, +} from "./protocol.js"; +import { downloadPublicFile } from "./file.js"; +import { fetchBrowserManifest } from "./manifest.js"; + +const elements = { + manifestUrl: document.querySelector("#manifest-url"), + loadManifest: document.querySelector("#load-manifest"), + manifestState: document.querySelector("#manifest-state"), + publicFile: document.querySelector("#public-file"), + publicFileName: document.querySelector("#public-file-name"), + publicFileAddress: document.querySelector("#public-file-address"), + publicFileChunks: document.querySelector("#public-file-chunks"), + publicFileReplicas: document.querySelector("#public-file-replicas"), + endpointUrl: document.querySelector("#endpoint-url"), + certificateHash: document.querySelector("#certificate-hash"), + connect: document.querySelector("#connect"), + connectionState: document.querySelector("#connection-state"), + lookupTarget: document.querySelector("#lookup-target"), + randomTarget: document.querySelector("#random-target"), + findClosest: document.querySelector("#find-closest"), + fileAddress: document.querySelector("#file-address"), + downloadFile: document.querySelector("#download-file"), + downloadState: document.querySelector("#download-state"), + downloadLink: document.querySelector("#download-link"), + log: document.querySelector("#log"), +}; + +let client; +let browserManifest; +let downloadObjectUrl; + +function timestamp() { + return new Date().toLocaleTimeString(); +} + +function log(message, value) { + const suffix = value === undefined ? "" : `\n${JSON.stringify(value, null, 2)}`; + elements.log.textContent += `[${timestamp()}] ${message}${suffix}\n`; + elements.log.scrollTop = elements.log.scrollHeight; +} + +function endpointFromForm() { + return { + url: elements.endpointUrl.value.trim(), + certificateSha256: elements.certificateHash.value.trim(), + }; +} + +function seedEndpoints() { + return browserManifest?.endpoints?.length + ? browserManifest.endpoints + : [endpointFromForm()]; +} + +async function loadManifest() { + elements.manifestState.classList.remove("connected"); + elements.manifestState.textContent = "Loading…"; + const manifest = await fetchBrowserManifest(elements.manifestUrl.value.trim()); + browserManifest = manifest; + + const first = manifest.endpoints[0]; + elements.endpointUrl.value = first.url; + elements.certificateHash.value = first.certificate_sha256; + client?.close(); + client = undefined; + elements.connectionState.classList.remove("connected"); + elements.connectionState.textContent = "Disconnected"; + + const file = manifest.files[0]; + if (file) { + elements.fileAddress.value = file.address; + elements.publicFileName.textContent = `${file.name} · ${file.size.toLocaleString()} bytes`; + elements.publicFileAddress.textContent = file.address; + elements.publicFileChunks.textContent = `${file.chunks.length} encrypted data chunks + public DataMap`; + elements.publicFileReplicas.textContent = `${file.replicas} node${file.replicas === 1 ? "" : "s"}`; + elements.publicFile.hidden = false; + } else { + elements.publicFile.hidden = true; + } + + elements.manifestState.textContent = `${manifest.endpoints.length} direct nodes · ${manifest.files.length} file${manifest.files.length === 1 ? "" : "s"}`; + elements.manifestState.classList.add("connected"); + log(`Loaded browser manifest ${manifest.network_id}`, manifest); + return manifest; +} + +async function connectedClient() { + if (client) return client; + const next = new BrowserNodeClient(endpointFromForm()); + const hello = await next.hello(); + client = next; + elements.connectionState.textContent = `Connected · ${hello.peer_id.slice(0, 16)}…`; + elements.connectionState.classList.add("connected"); + log("HELLO", hello); + return next; +} + +function reportError(context, error) { + elements.connectionState.textContent = `${context} failed`; + log(`${context} failed: ${error.message}`); + console.error(error); +} + +elements.randomTarget.addEventListener("click", () => { + const target = crypto.getRandomValues(new Uint8Array(32)); + elements.lookupTarget.value = bytesToHex(target); +}); + +elements.loadManifest.addEventListener("click", async () => { + try { + await loadManifest(); + } catch (error) { + elements.manifestState.textContent = "Load failed"; + log(`Manifest load failed: ${error.message}`); + console.error(error); + } +}); + +elements.connect.addEventListener("click", async () => { + client?.close(); + client = undefined; + elements.connectionState.classList.remove("connected"); + elements.connectionState.textContent = "Connecting…"; + try { + await connectedClient(); + } catch (error) { + reportError("Connection", error); + } +}); + +elements.findClosest.addEventListener("click", async () => { + try { + const target = elements.lookupTarget.value.trim(); + hexToBytes(target, 32); + log(`Starting iterative lookup for ${target}`); + const result = await iterativeFindClosest(seedEndpoints(), target, { + onProgress: (message) => log(message), + }); + log("Closest nodes", { + nodes: result.nodes, + queried: result.queried, + failures: result.failures.map(({ peerId, error }) => ({ + peerId, + message: error.message, + })), + }); + for (const lookupClient of result.clients.values()) lookupClient.close(); + } catch (error) { + reportError("Lookup", error); + } +}); + +elements.downloadFile.addEventListener("click", async () => { + elements.downloadState.classList.remove("connected"); + elements.downloadState.textContent = "Preparing save…"; + elements.downloadFile.disabled = true; + try { + const address = elements.fileAddress.value.trim().toLowerCase(); + hexToBytes(address, 32); + const published = browserManifest?.files.find((file) => file.address === address); + if (!published) { + throw new Error("That public file address is not described by the loaded testnet manifest"); + } + const saveHandle = await chooseSaveHandle(published.name); + elements.downloadState.textContent = "Downloading…"; + log( + `Downloading complete public file ${published.name} (${published.size.toLocaleString()} bytes)`, + ); + const { content, hash, dataMapNode } = await downloadPublicFile( + seedEndpoints(), + published, + { + onProgress: (message) => log(message), + }, + ); + const savedDirectly = await exposeSavedFile(published, content, saveHandle); + elements.downloadState.textContent = `${ + savedDirectly ? "Saved" : "Browser download started" + } · ${content.length.toLocaleString()} bytes`; + elements.downloadState.classList.add("connected"); + log( + `${ + savedDirectly ? "Saved" : "Started browser save for" + } whole-file BLAKE3-verified ${published.name} from direct nodes as ${hash}`, + { data_map_node: dataMapNode.peer_id, chunks: published.chunks.length }, + ); + } catch (error) { + elements.downloadState.textContent = error.name === "AbortError" ? "Save cancelled" : "Failed"; + log(`File download failed: ${error.message}`); + console.error(error); + } finally { + elements.downloadFile.disabled = false; + } +}); + +async function chooseSaveHandle(name) { + if (typeof window.showSaveFilePicker !== "function") return undefined; + return window.showSaveFilePicker({ suggestedName: name }); +} + +async function exposeSavedFile(file, content, saveHandle) { + if (saveHandle) { + const writable = await saveHandle.createWritable(); + try { + await writable.write(content); + await writable.close(); + } catch (error) { + try { + await writable.abort(error); + } catch (abortError) { + log(`Could not abort failed save: ${abortError.message}`); + } + throw error; + } + } + + if (downloadObjectUrl) URL.revokeObjectURL(downloadObjectUrl); + downloadObjectUrl = URL.createObjectURL( + new Blob([content], { type: file.content_type ?? "application/octet-stream" }), + ); + const anchor = document.createElement("a"); + anchor.href = downloadObjectUrl; + anchor.download = file.name; + anchor.textContent = saveHandle + ? `Save ${file.name} again` + : `Save ${file.name} (${content.length.toLocaleString()} bytes)`; + elements.downloadLink.replaceChildren(anchor); + if (!saveHandle) anchor.click(); + return Boolean(saveHandle); +} + +window.addEventListener("beforeunload", () => { + if (downloadObjectUrl) URL.revokeObjectURL(downloadObjectUrl); +}); + +elements.randomTarget.click(); +log("Ready. Loading the local browser testnet manifest…"); +loadManifest().catch((error) => { + elements.manifestState.textContent = "Not running"; + log(`Local manifest not available yet: ${error.message}`); +}); diff --git a/web/src/manifest.js b/web/src/manifest.js new file mode 100644 index 00000000..9bd8a5d6 --- /dev/null +++ b/web/src/manifest.js @@ -0,0 +1,117 @@ +import { hexToBytes } from "./protocol.js"; + +export const BROWSER_MANIFEST_VERSION = 2; +const MAX_PUBLIC_FILE_BYTES = 64 * 1024 * 1024; +const MAX_DATA_MAP_BYTES = 4 * 1024 * 1024; +const MAX_FILE_CHUNKS = 1024; + +export function parseBrowserManifest(value) { + if (!value || value.version !== BROWSER_MANIFEST_VERSION) { + throw new Error(`Unsupported browser manifest version ${value?.version}`); + } + if (typeof value.network_id !== "string" || value.network_id.length === 0) { + throw new Error("Browser manifest has no network ID"); + } + if (!Array.isArray(value.endpoints) || value.endpoints.length === 0) { + throw new Error("Browser manifest contains no WebTransport endpoints"); + } + const endpoints = value.endpoints.map((endpoint) => { + if (!endpoint || typeof endpoint.url !== "string") { + throw new Error("Browser manifest endpoint has no URL"); + } + if (!endpoint.url.startsWith("https://")) { + throw new Error(`WebTransport endpoint must use HTTPS: ${endpoint.url}`); + } + hexToBytes(endpoint.peer_id ?? "", 32); + hexToBytes(endpoint.certificate_sha256 ?? "", 32); + return { + peer_id: endpoint.peer_id.toLowerCase(), + url: endpoint.url, + certificate_sha256: endpoint.certificate_sha256.toLowerCase(), + }; + }); + + const files = (value.files ?? []).map((file) => { + if (!file || typeof file.name !== "string" || file.name.length === 0) { + throw new Error("Browser manifest file has no name"); + } + hexToBytes(file.address ?? "", 32); + if ( + !Number.isSafeInteger(file.size) || + file.size < 3 || + file.size > MAX_PUBLIC_FILE_BYTES + ) { + throw new Error(`Invalid public file size ${file.size}`); + } + hexToBytes(file.blake3 ?? "", 32); + if ( + !Number.isSafeInteger(file.data_map_size) || + file.data_map_size < 1 || + file.data_map_size > MAX_DATA_MAP_BYTES + ) { + throw new Error(`Invalid DataMap size ${file.data_map_size}`); + } + if ( + !Array.isArray(file.chunks) || + file.chunks.length < 3 || + file.chunks.length > MAX_FILE_CHUNKS + ) { + throw new Error("Public file has an invalid self-encryption chunk list"); + } + const chunks = file.chunks + .map((chunk) => { + if (!Number.isSafeInteger(chunk.index) || chunk.index < 0) { + throw new Error(`Invalid file chunk index ${chunk.index}`); + } + hexToBytes(chunk.dst_hash ?? "", 32); + hexToBytes(chunk.src_hash ?? "", 32); + if (!Number.isSafeInteger(chunk.src_size) || chunk.src_size < 1) { + throw new Error(`Invalid plaintext chunk size ${chunk.src_size}`); + } + return { + index: chunk.index, + dst_hash: chunk.dst_hash.toLowerCase(), + src_hash: chunk.src_hash.toLowerCase(), + src_size: chunk.src_size, + }; + }) + .sort((left, right) => left.index - right.index); + chunks.forEach((chunk, index) => { + if (chunk.index !== index) { + throw new Error("File chunk indices must be contiguous from zero"); + } + }); + const reconstructedSize = chunks.reduce((total, chunk) => total + chunk.src_size, 0); + if (reconstructedSize !== file.size) { + throw new Error( + `File chunk sizes total ${reconstructedSize}, expected ${file.size}`, + ); + } + return { + name: file.name, + address: file.address.toLowerCase(), + size: file.size, + content_type: file.content_type || "application/octet-stream", + blake3: file.blake3.toLowerCase(), + data_map_size: file.data_map_size, + chunks, + replicas: Number.isSafeInteger(file.replicas) ? file.replicas : 0, + }; + }); + + return { + version: value.version, + network_id: value.network_id, + created_at: value.created_at, + endpoints, + files, + }; +} + +export async function fetchBrowserManifest(url) { + const response = await fetch(url, { cache: "no-store" }); + if (!response.ok) { + throw new Error(`Manifest request failed with HTTP ${response.status}`); + } + return parseBrowserManifest(await response.json()); +} diff --git a/web/src/manifest.test.js b/web/src/manifest.test.js new file mode 100644 index 00000000..f9949eb3 --- /dev/null +++ b/web/src/manifest.test.js @@ -0,0 +1,65 @@ +import assert from "node:assert/strict"; +import test from "node:test"; +import { parseBrowserManifest } from "./manifest.js"; + +test("browser manifest validates and normalizes endpoints and files", () => { + const manifest = parseBrowserManifest({ + version: 2, + network_id: "local-test", + created_at: "2026-08-03T00:00:00Z", + endpoints: [ + { + peer_id: "AA".repeat(32), + url: "https://127.0.0.1:22000/autonomi/webtransport/v1", + certificate_sha256: "BB".repeat(32), + }, + ], + files: [ + { + name: "hello.txt", + address: "CC".repeat(32), + size: 12, + content_type: "text/plain", + blake3: "DD".repeat(32), + data_map_size: 128, + chunks: [ + { index: 2, dst_hash: "13".repeat(32), src_hash: "23".repeat(32), src_size: 4 }, + { index: 0, dst_hash: "11".repeat(32), src_hash: "21".repeat(32), src_size: 4 }, + { index: 1, dst_hash: "12".repeat(32), src_hash: "22".repeat(32), src_size: 4 }, + ], + replicas: 5, + }, + ], + }); + + assert.equal(manifest.endpoints[0].peer_id, "aa".repeat(32)); + assert.equal(manifest.files[0].address, "cc".repeat(32)); + assert.equal(manifest.files[0].blake3, "dd".repeat(32)); + assert.deepEqual( + manifest.files[0].chunks.map((chunk) => chunk.index), + [0, 1, 2], + ); + assert.equal(manifest.files[0].replicas, 5); +}); + +test("browser manifest rejects missing endpoints and malformed hashes", () => { + assert.throws( + () => parseBrowserManifest({ version: 2, network_id: "test", endpoints: [] }), + /no WebTransport endpoints/, + ); + assert.throws( + () => + parseBrowserManifest({ + version: 2, + network_id: "test", + endpoints: [ + { + peer_id: "wrong", + url: "https://127.0.0.1:22000/path", + certificate_sha256: "bb".repeat(32), + }, + ], + }), + /hexadecimal|Expected 32 bytes/, + ); +}); diff --git a/web/src/protocol.js b/web/src/protocol.js new file mode 100644 index 00000000..7779fc5e --- /dev/null +++ b/web/src/protocol.js @@ -0,0 +1,426 @@ +import { blake3 } from "@noble/hashes/blake3.js"; +import { bytesToHex as nobleBytesToHex } from "@noble/hashes/utils.js"; + +export const PROTOCOL_VERSION = 1; +export const PROTOCOL_NAME = "autonomi.web.poc.v1"; +export const MAX_CHUNK_SIZE = 4 * 1024 * 1024; +export const MAX_RESPONSE_HEADER_BYTES = 64 * 1024; +const MAX_RESPONSE_BYTES = 4 + MAX_RESPONSE_HEADER_BYTES + MAX_CHUNK_SIZE; +const encoder = new TextEncoder(); +const decoder = new TextDecoder("utf-8", { fatal: true }); + +let nextRequestId = 1; + +export function hexToBytes(value, expectedLength) { + const normalized = value.trim().replace(/^0x/i, "").replaceAll(":", ""); + if (!/^[0-9a-f]*$/i.test(normalized) || normalized.length % 2 !== 0) { + throw new Error("Expected an even-length hexadecimal value"); + } + const bytes = Uint8Array.from( + normalized.match(/.{2}/g)?.map((pair) => Number.parseInt(pair, 16)) ?? [], + ); + if (expectedLength !== undefined && bytes.length !== expectedLength) { + throw new Error(`Expected ${expectedLength} bytes, received ${bytes.length}`); + } + return bytes; +} + +export function bytesToHex(bytes) { + return nobleBytesToHex(bytes); +} + +export function xorDistance(peerId, target) { + const peer = hexToBytes(peerId, 32); + const key = hexToBytes(target, 32); + let distance = 0n; + for (let index = 0; index < peer.length; index += 1) { + distance = (distance << 8n) | BigInt(peer[index] ^ key[index]); + } + return distance; +} + +export function verifyChunk(address, content) { + const expected = address.trim().replace(/^0x/i, "").toLowerCase(); + hexToBytes(expected, 32); + const actual = nobleBytesToHex(blake3(content)); + if (actual !== expected) { + throw new Error(`BLAKE3 mismatch: expected ${expected}, received ${actual}`); + } + return actual; +} + +export function parseResponseFrame(frame) { + if (!(frame instanceof Uint8Array) || frame.length < 4) { + throw new Error("Response ended before its four-byte header length"); + } + const view = new DataView(frame.buffer, frame.byteOffset, frame.byteLength); + const headerLength = view.getUint32(0, false); + if (headerLength === 0 || headerLength > MAX_RESPONSE_HEADER_BYTES) { + throw new Error(`Invalid response header length ${headerLength}`); + } + const contentOffset = 4 + headerLength; + if (contentOffset > frame.length) { + throw new Error("Response ended inside its JSON header"); + } + + let header; + try { + header = JSON.parse(decoder.decode(frame.subarray(4, contentOffset))); + } catch (error) { + throw new Error(`Invalid response JSON: ${error.message}`, { cause: error }); + } + if (header.version !== PROTOCOL_VERSION) { + throw new Error(`Unsupported response version ${header.version}`); + } + if ( + !Number.isSafeInteger(header.content_length) || + header.content_length < 0 || + header.content_length > MAX_CHUNK_SIZE + ) { + throw new Error(`Invalid response content length ${header.content_length}`); + } + if (frame.length !== contentOffset + header.content_length) { + throw new Error( + `Response length mismatch: declared ${header.content_length} content bytes`, + ); + } + return { header, content: frame.slice(contentOffset) }; +} + +async function readAll(readable, limit = MAX_RESPONSE_BYTES) { + const reader = readable.getReader(); + const chunks = []; + let total = 0; + try { + while (true) { + const { value, done } = await reader.read(); + if (done) break; + if (!(value instanceof Uint8Array)) { + throw new Error("WebTransport returned a non-byte stream chunk"); + } + total += value.length; + if (total > limit) { + await reader.cancel("response exceeded client limit"); + throw new Error(`Response exceeded the ${limit}-byte client limit`); + } + chunks.push(value); + } + } finally { + reader.releaseLock(); + } + + const result = new Uint8Array(total); + let offset = 0; + for (const chunk of chunks) { + result.set(chunk, offset); + offset += chunk.length; + } + return result; +} + +function normalizeEndpoint(endpoint) { + if (!endpoint || typeof endpoint.url !== "string") { + throw new Error("Endpoint URL is required"); + } + const certificateSha256 = + endpoint.certificate_sha256 ?? endpoint.certificateSha256; + return { + url: endpoint.url, + peerId: endpoint.peer_id ?? endpoint.peerId, + certificateSha256, + certificateBytes: hexToBytes(certificateSha256 ?? "", 32), + }; +} + +export class BrowserNodeClient { + constructor(endpoint) { + this.endpoint = normalizeEndpoint(endpoint); + this.transport = undefined; + this.peerId = undefined; + } + + async connect() { + if (this.transport) return; + if (typeof WebTransport === "undefined") { + throw new Error("This browser does not expose the WebTransport API"); + } + const transport = new WebTransport(this.endpoint.url, { + serverCertificateHashes: [ + { algorithm: "sha-256", value: this.endpoint.certificateBytes }, + ], + }); + try { + await transport.ready; + } catch (error) { + transport.close(); + throw error; + } + this.transport = transport; + } + + async request(type, fields = {}) { + await this.connect(); + const requestId = nextRequestId; + nextRequestId += 1; + const stream = await this.transport.createBidirectionalStream(); + const writer = stream.writable.getWriter(); + try { + await writer.write( + encoder.encode( + JSON.stringify({ + version: PROTOCOL_VERSION, + request_id: requestId, + type, + ...fields, + }), + ), + ); + await writer.close(); + } finally { + writer.releaseLock(); + } + + const response = parseResponseFrame(await readAll(stream.readable)); + if (response.header.request_id !== requestId) { + throw new Error( + `Response ID ${response.header.request_id} does not match request ${requestId}`, + ); + } + if (response.header.status === "error") { + const error = new Error(response.header.message ?? "Node returned an error"); + error.code = response.header.code; + throw error; + } + return response; + } + + async hello() { + const { header } = await this.request("hello"); + if (header.type !== "hello") throw new Error("Expected a HELLO response"); + if (header.protocol !== PROTOCOL_NAME) { + throw new Error(`Unsupported browser protocol ${header.protocol}`); + } + const advertisedEndpoint = normalizeEndpoint(header.endpoint); + if ( + advertisedEndpoint.url !== this.endpoint.url || + advertisedEndpoint.certificateSha256.toLowerCase() !== + this.endpoint.certificateSha256.toLowerCase() + ) { + throw new Error("Node advertised a different WebTransport endpoint"); + } + hexToBytes(header.peer_id, 32); + if ( + this.endpoint.peerId && + header.peer_id.toLowerCase() !== this.endpoint.peerId.toLowerCase() + ) { + throw new Error( + `Endpoint identity mismatch: expected ${this.endpoint.peerId}, received ${header.peer_id}`, + ); + } + this.peerId = header.peer_id; + return header; + } + + async findNode(target, count = 20) { + hexToBytes(target, 32); + const { header } = await this.request("find_node", { target, count }); + if (header.type !== "nodes") throw new Error("Expected a NODES response"); + if (header.target.toLowerCase() !== target.toLowerCase()) { + throw new Error("Node returned results for a different lookup target"); + } + if (!Array.isArray(header.nodes)) { + throw new Error("Node returned an invalid node list"); + } + for (const node of header.nodes) hexToBytes(node.peer_id, 32); + return header.nodes; + } + + async getChunk(address) { + hexToBytes(address, 32); + const response = await this.request("get_chunk", { address }); + if (response.header.status === "not_found") { + const error = new Error(`Chunk ${address} was not found on this node`); + error.code = "not_found"; + throw error; + } + if (response.header.type !== "chunk") { + throw new Error("Expected a CHUNK response"); + } + if (response.header.address.toLowerCase() !== address.toLowerCase()) { + throw new Error("Node returned a different chunk address"); + } + if (response.header.size !== response.content.length) { + throw new Error("Chunk metadata size does not match its content"); + } + const hash = verifyChunk(address, response.content); + return { content: response.content, hash }; + } + + close() { + this.transport?.close({ closeCode: 0, reason: "client closed" }); + this.transport = undefined; + } +} + +function endpointKey(endpoint) { + const normalized = normalizeEndpoint(endpoint); + return `${normalized.url}|${normalized.certificateSha256}`; +} + +export async function iterativeFindClosest( + seedEndpoints, + target, + { k = 20, alpha = 3, maxIterations = 20, onProgress = () => {} } = {}, +) { + hexToBytes(target, 32); + if (!Array.isArray(seedEndpoints) || seedEndpoints.length === 0) { + throw new Error("At least one seed endpoint is required"); + } + + const clients = new Map(); + const known = new Map(); + const queried = new Set(); + const failures = []; + + const clientFor = (endpoint) => { + const key = endpointKey(endpoint); + let client = clients.get(key); + if (!client) { + client = new BrowserNodeClient(endpoint); + clients.set(key, client); + } + return client; + }; + + await Promise.all( + seedEndpoints.map(async (endpoint) => { + const seedName = endpoint?.peer_id ?? endpoint?.peerId ?? endpoint?.url ?? "seed"; + try { + const client = clientFor(endpoint); + const hello = await client.hello(); + known.set(hello.peer_id, { + peer_id: hello.peer_id, + native_addresses: [], + reliability: 1, + webtransport: hello.endpoint, + }); + onProgress(`Connected seed ${hello.peer_id}`); + } catch (error) { + failures.push({ peerId: seedName, error }); + onProgress(`Seed ${seedName} failed: ${error.message}`); + } + }), + ); + if (known.size === 0) { + for (const client of clients.values()) client.close(); + const detail = failures.map(({ error }) => error.message).join("; "); + throw new Error(`Could not connect to any WebTransport seed: ${detail}`); + } + + for (let iteration = 0; iteration < maxIterations; iteration += 1) { + const ordered = [...known.values()].sort((left, right) => { + const a = xorDistance(left.peer_id, target); + const b = xorDistance(right.peer_id, target); + return a < b ? -1 : a > b ? 1 : 0; + }); + const candidates = ordered + .filter((node) => node.webtransport && !queried.has(node.peer_id)) + .slice(0, alpha); + if (candidates.length === 0) break; + + let discovered = 0; + await Promise.all( + candidates.map(async (candidate) => { + queried.add(candidate.peer_id); + try { + const candidateClient = clientFor({ + ...candidate.webtransport, + peer_id: candidate.peer_id, + }); + if (!candidateClient.peerId) await candidateClient.hello(); + const nodes = await candidateClient.findNode(target, k); + onProgress( + `Iteration ${iteration + 1}: ${candidate.peer_id} returned ${nodes.length} nodes`, + ); + for (const node of nodes) { + const existing = known.get(node.peer_id); + if (!existing) discovered += 1; + known.set(node.peer_id, { + ...existing, + ...node, + webtransport: node.webtransport ?? existing?.webtransport, + }); + } + } catch (error) { + failures.push({ peerId: candidate.peer_id, error }); + onProgress(`Query ${candidate.peer_id} failed: ${error.message}`); + } + }), + ); + + const remainingQueryable = [...known.values()] + .sort((left, right) => { + const a = xorDistance(left.peer_id, target); + const b = xorDistance(right.peer_id, target); + return a < b ? -1 : a > b ? 1 : 0; + }) + .slice(0, k) + .some((node) => node.webtransport && !queried.has(node.peer_id)); + if (discovered === 0 && !remainingQueryable) break; + } + + const nodes = [...known.values()] + .sort((left, right) => { + const a = xorDistance(left.peer_id, target); + const b = xorDistance(right.peer_id, target); + return a < b ? -1 : a > b ? 1 : 0; + }) + .slice(0, k); + + return { nodes, queried: [...queried], failures, clients }; +} + +export async function getChunkFromClosest( + seedEndpoints, + address, + { onProgress = () => {}, ...lookupOptions } = {}, +) { + hexToBytes(address, 32); + const lookup = await iterativeFindClosest(seedEndpoints, address, { + ...lookupOptions, + onProgress, + }); + const attempted = []; + + try { + for (const node of lookup.nodes) { + if (!node.webtransport) continue; + const endpoint = { ...node.webtransport, peer_id: node.peer_id }; + const key = endpointKey(endpoint); + let client = lookup.clients.get(key); + if (!client) { + client = new BrowserNodeClient(endpoint); + lookup.clients.set(key, client); + } + + try { + if (!client.peerId) await client.hello(); + onProgress(`Requesting ${address} from ${node.peer_id}`); + const chunk = await client.getChunk(address); + return { ...chunk, node, lookup }; + } catch (error) { + attempted.push({ peerId: node.peer_id, error }); + onProgress(`Node ${node.peer_id} did not return the file: ${error.message}`); + } + } + } finally { + for (const client of lookup.clients.values()) client.close(); + } + + const detail = attempted + .map(({ peerId, error }) => `${peerId}: ${error.message}`) + .join("; "); + throw new Error( + `No closest WebTransport node returned chunk ${address}${detail ? ` (${detail})` : ""}`, + ); +} diff --git a/web/src/protocol.test.js b/web/src/protocol.test.js new file mode 100644 index 00000000..a8a48bfc --- /dev/null +++ b/web/src/protocol.test.js @@ -0,0 +1,223 @@ +import assert from "node:assert/strict"; +import test from "node:test"; +import { blake3 } from "@noble/hashes/blake3.js"; +import { + bytesToHex, + getChunkFromClosest, + hexToBytes, + parseResponseFrame, + verifyChunk, + xorDistance, +} from "./protocol.js"; + +test("hex conversion enforces fixed widths", () => { + const value = "ab".repeat(32); + assert.equal(bytesToHex(hexToBytes(value, 32)), value); + assert.throws(() => hexToBytes("abcd", 32), /Expected 32 bytes/); + assert.throws(() => hexToBytes("zz", 1), /hexadecimal/); +}); + +test("XOR distance is an unsigned 256-bit ordering value", () => { + assert.equal(xorDistance("00".repeat(32), "00".repeat(32)), 0n); + assert.equal(xorDistance("00".repeat(31) + "01", "00".repeat(32)), 1n); + assert(xorDistance("80" + "00".repeat(31), "00".repeat(32)) > 1n); +}); + +test("response framing preserves a raw binary body", () => { + const header = new TextEncoder().encode( + JSON.stringify({ + version: 1, + request_id: 9, + status: "ok", + content_length: 3, + type: "chunk", + address: "11".repeat(32), + size: 3, + }), + ); + const frame = new Uint8Array(4 + header.length + 3); + new DataView(frame.buffer).setUint32(0, header.length, false); + frame.set(header, 4); + frame.set([1, 2, 3], 4 + header.length); + + const parsed = parseResponseFrame(frame); + assert.equal(parsed.header.request_id, 9); + assert.deepEqual([...parsed.content], [1, 2, 3]); +}); + +test("BLAKE3 verification accepts the canonical empty hash", () => { + const emptyHash = + "af1349b9f5f9a1a6a0404dea36dcc9499bcb25c9adc112b7cc9a93cae41f3262"; + assert.equal(verifyChunk(emptyHash, new Uint8Array()), emptyHash); + assert.throws(() => verifyChunk("00".repeat(32), new Uint8Array()), /BLAKE3 mismatch/); +}); + +test("browser lookup discovers a direct node and downloads a verified chunk", async (t) => { + const content = new TextEncoder().encode("direct browser test file"); + const address = bytesToHex(blake3(content)); + const seedPeer = "ff".repeat(32); + const storagePeer = address; + const seed = { + peer_id: seedPeer, + url: "https://seed.test/autonomi/webtransport/v1", + certificate_sha256: "11".repeat(32), + }; + const storage = { + peer_id: storagePeer, + url: "https://storage.test/autonomi/webtransport/v1", + certificate_sha256: "22".repeat(32), + }; + const calls = []; + const routes = new Map([ + [ + seed.url, + (request) => { + calls.push(["seed", request.type]); + if (request.type === "hello") return helloResponse(request, seed); + if (request.type === "find_node") { + return response(request, { + type: "nodes", + target: request.target, + nodes: [browserNode(storage), browserNode(seed)], + }); + } + throw new Error(`Unexpected seed request ${request.type}`); + }, + ], + [ + storage.url, + (request) => { + calls.push(["storage", request.type]); + if (request.type === "hello") return helloResponse(request, storage); + if (request.type === "find_node") { + return response(request, { + type: "nodes", + target: request.target, + nodes: [browserNode(storage), browserNode(seed)], + }); + } + if (request.type === "get_chunk") { + return response( + request, + { type: "chunk", address: request.address, size: content.length }, + content, + ); + } + throw new Error(`Unexpected storage request ${request.type}`); + }, + ], + ]); + + const previousWebTransport = globalThis.WebTransport; + globalThis.WebTransport = mockWebTransport(routes); + t.after(() => { + globalThis.WebTransport = previousWebTransport; + }); + + const downloaded = await getChunkFromClosest([seed], address); + assert.deepEqual(downloaded.content, content); + assert.equal(downloaded.hash, address); + assert.equal(downloaded.node.peer_id, storagePeer); + assert.deepEqual(calls, [ + ["seed", "hello"], + ["seed", "find_node"], + ["storage", "hello"], + ["storage", "find_node"], + ["storage", "get_chunk"], + ]); +}); + +function browserNode(endpoint) { + return { + peer_id: endpoint.peer_id, + native_addresses: [], + reliability: 1, + webtransport: { + url: endpoint.url, + certificate_sha256: endpoint.certificate_sha256, + }, + }; +} + +function helloResponse(request, endpoint) { + return response(request, { + type: "hello", + protocol: "autonomi.web.poc.v1", + peer_id: endpoint.peer_id, + max_chunk_size: 4 * 1024 * 1024, + endpoint: { + url: endpoint.url, + certificate_sha256: endpoint.certificate_sha256, + }, + capabilities: ["find_node", "get_chunk"], + }); +} + +function response(request, fields, content = new Uint8Array()) { + return { + header: { + version: 1, + request_id: request.request_id, + status: "ok", + content_length: content.length, + ...fields, + }, + content, + }; +} + +function encodeResponse({ header, content }) { + const headerBytes = new TextEncoder().encode(JSON.stringify(header)); + const frame = new Uint8Array(4 + headerBytes.length + content.length); + new DataView(frame.buffer).setUint32(0, headerBytes.length, false); + frame.set(headerBytes, 4); + frame.set(content, 4 + headerBytes.length); + return frame; +} + +function mockWebTransport(routes) { + return class MockWebTransport { + constructor(url, options) { + this.url = url; + this.options = options; + this.ready = routes.has(url) + ? Promise.resolve() + : Promise.reject(new Error(`No mock endpoint for ${url}`)); + } + + async createBidirectionalStream() { + const requestChunks = []; + let responseController; + const readable = new ReadableStream({ + start(controller) { + responseController = controller; + }, + }); + const writable = new WritableStream({ + write(chunk) { + requestChunks.push(chunk); + }, + close: () => { + try { + const length = requestChunks.reduce((total, chunk) => total + chunk.length, 0); + const encoded = new Uint8Array(length); + let offset = 0; + for (const chunk of requestChunks) { + encoded.set(chunk, offset); + offset += chunk.length; + } + const request = JSON.parse(new TextDecoder().decode(encoded)); + const handler = routes.get(this.url); + responseController.enqueue(encodeResponse(handler(request))); + responseController.close(); + } catch (error) { + responseController.error(error); + } + }, + }); + return { readable, writable }; + } + + close() {} + }; +} diff --git a/web/src/style.css b/web/src/style.css new file mode 100644 index 00000000..d65ff0ae --- /dev/null +++ b/web/src/style.css @@ -0,0 +1,219 @@ +:root { + color-scheme: light dark; + font-family: + Inter, ui-sans-serif, system-ui, -apple-system, BlinkMacSystemFont, "Segoe UI", + sans-serif; + color: #17211b; + background: #edf3ee; + font-synthesis: none; +} + +* { + box-sizing: border-box; +} + +body { + margin: 0; + min-width: 320px; +} + +main { + width: min(880px, calc(100% - 2rem)); + margin: 3rem auto; +} + +header { + margin-bottom: 2rem; +} + +h1, +h2, +p { + margin-top: 0; +} + +h1 { + margin-bottom: 0.65rem; + font-size: clamp(2rem, 5vw, 3.6rem); + letter-spacing: -0.04em; +} + +h2 { + font-size: 1rem; + letter-spacing: 0.02em; +} + +.section-note { + color: #68766c; + font-size: 0.88rem; +} + +.eyebrow { + color: #25764a; + font-size: 0.78rem; + font-weight: 750; + letter-spacing: 0.12em; + text-transform: uppercase; +} + +section { + margin: 1rem 0; + padding: 1.25rem; + border: 1px solid #cad7cd; + border-radius: 14px; + background: #fff; + box-shadow: 0 10px 35px rgb(30 62 42 / 6%); +} + +label { + display: grid; + gap: 0.4rem; + margin-top: 0.8rem; + color: #425046; + font-size: 0.86rem; + font-weight: 650; +} + +input { + width: 100%; + padding: 0.72rem 0.8rem; + border: 1px solid #bdc9bf; + border-radius: 8px; + color: #17211b; + background: #f9fbf9; + font: 0.87rem ui-monospace, SFMono-Regular, Menlo, Consolas, monospace; +} + +input:focus { + border-color: #25764a; + outline: 3px solid rgb(37 118 74 / 16%); +} + +.actions { + display: flex; + flex-wrap: wrap; + align-items: center; + gap: 0.7rem; + margin-top: 1rem; +} + +button { + padding: 0.68rem 1rem; + border: 1px solid #195d39; + border-radius: 8px; + color: white; + background: #25764a; + font: inherit; + font-weight: 700; + cursor: pointer; +} + +button:hover { + background: #195d39; +} + +button:disabled { + cursor: wait; + opacity: 0.6; +} + +button.secondary { + color: #195d39; + background: transparent; +} + +output { + color: #7a4233; + font-size: 0.88rem; + font-weight: 650; +} + +output.connected { + color: #25764a; +} + +.file-card { + display: grid; + gap: 0.6rem; + margin: 1rem 0 0; + padding: 0.9rem; + border-radius: 8px; + background: #f2f7f3; +} + +.file-card[hidden] { + display: none; +} + +.file-card div { + display: grid; + grid-template-columns: 8rem minmax(0, 1fr); + gap: 0.75rem; +} + +.file-card dt { + color: #68766c; + font-size: 0.82rem; + font-weight: 700; +} + +.file-card dd { + margin: 0; + overflow-wrap: anywhere; + font: 0.8rem ui-monospace, SFMono-Regular, Menlo, Consolas, monospace; +} + +pre { + min-height: 180px; + max-height: 430px; + margin: 0; + overflow: auto; + white-space: pre-wrap; + word-break: break-word; + color: #d9f3df; + background: #122219; + padding: 1rem; + border-radius: 8px; + font-size: 0.78rem; + line-height: 1.5; +} + +a { + color: #195d39; + font-weight: 700; +} + +@media (prefers-color-scheme: dark) { + :root { + color: #e8eee9; + background: #111713; + } + + section { + border-color: #34453a; + background: #1a241d; + } + + label { + color: #b8c7bc; + } + + input { + border-color: #46574b; + color: #edf5ef; + background: #111713; + } + + .section-note, + .file-card dt { + color: #a6b5aa; + } + + .file-card { + background: #111713; + } + + a { + color: #7bd29e; + } +} From be0e5f591a9a9258c16966f43458ea372d9263af Mon Sep 17 00:00:00 2001 From: Mick van Dijke <12992260+mickvandijke@users.noreply.github.com> Date: Wed, 5 Aug 2026 10:24:51 +0200 Subject: [PATCH 02/31] feat(web): consume self-contained node multiaddresses --- .../ADR-0003-direct-browser-read-client.md | 34 +++- web/README.md | 14 +- web/index.html | 18 +- web/src/main.js | 11 +- web/src/manifest.js | 19 +- web/src/manifest.test.js | 34 ++-- web/src/protocol.js | 188 +++++++++++++++--- web/src/protocol.test.js | 70 +++++-- 8 files changed, 285 insertions(+), 103 deletions(-) diff --git a/docs/adr/ADR-0003-direct-browser-read-client.md b/docs/adr/ADR-0003-direct-browser-read-client.md index b44c7a4d..8e94e1e4 100644 --- a/docs/adr/ADR-0003-direct-browser-read-client.md +++ b/docs/adr/ADR-0003-direct-browser-read-client.md @@ -2,6 +2,7 @@ - **Status:** Proposed - **Date:** 2026-08-03 +- **Last amended:** 2026-08-04 - **Decision owners:** - **Reviewers:** - **Supersedes:** none @@ -24,7 +25,8 @@ bootstrap-manifest production under ant-node ADR-0009. - File bytes must travel directly from a storage node to the browser. - The browser must own iterative XOR lookup rather than ask a gateway to do it. -- Self-signed node certificates must be authenticated through explicit hashes. +- Self-signed node certificates must be authenticated through hashes embedded + in self-contained node multiaddresses, without a separate client argument. - Downloaded immutable content must be verified before it is exposed to users. - Local testnets need a reproducible bootstrap and default-file workflow. @@ -43,10 +45,12 @@ bootstrap-manifest production under ant-node ADR-0009. The `web/` package will implement the direct browser read client: -- load a versioned browser bootstrap manifest containing node URLs, - certificate hashes, peer IDs, and published immutable-file metadata; -- authenticate every seed and discovered endpoint by certificate hash and - verify its `HELLO` peer ID; +- load a versioned browser bootstrap manifest containing WebTransport + multiaddresses and published immutable-file metadata; +- accept a single multiaddress per seed, extract its one or two + `/certhash` SHA-256 multihashes internally, and pass them to WebTransport; +- require `/p2p/` in every address, verify its `HELLO` peer ID, and + reject discovered endpoint/peer mismatches; - perform iterative `FIND_NODE` queries using 256-bit XOR ordering, `K = 20`, and `ALPHA = 3`; - query closest direct endpoints with `GET_CHUNK`, retrying `not_found` and @@ -63,6 +67,20 @@ The local browser manifest is bootstrap metadata, not a gateway. Production clients will replace its unsigned endpoint list with the ML-DSA-signed records defined by ant-node ADR-0009. +The JavaScript API and demo never accept a certificate hash separately from an +endpoint. Their sole dialing input is a canonical address of the form +`/.../quic-v1/webtransport/certhash//p2p/`. Repeated +`/certhash` components permit current/next certificate overlap. Keeping the +transport location, accepted TLS key, and expected peer identity in one value +prevents callers from accidentally combining fields belonging to different +nodes. + +Rust nodes construct and validate this syntax through +`saorsa_transport::TransportAddr::WebTransport` wrapped by +`saorsa_core::MultiAddr`; it is not a browser-specific string type. The +JavaScript parser is the browser implementation of that same canonical wire +format and is covered by matching current/next-pin fixtures. + For the local vertical slice, the bootstrap manifest carries a resolved JSON view of the public root DataMap alongside its ordinary on-network DataMap address. The browser still fetches and verifies that public DataMap record and @@ -76,7 +94,8 @@ metadata chain. - Lookup and data transfer remain decentralized at the application layer. - A local five-node testnet can validate multiple direct node connections, - certificate pins, lookup convergence, fallback, and content verification. + multiaddress-embedded certificate pins, lookup convergence, fallback, and + content verification. - The browser protocol is independent of native Rust serialization details. ### Negative / Trade-offs @@ -102,7 +121,8 @@ metadata chain. KDF, authenticated decryption, Brotli reconstruction, and tamper rejection. - A live node integration test starts five WebTransport-enabled nodes, publishes a public DataMap and encrypted chunks, connects with the advertised - certificate hash, retrieves every record, and reconstructs the exact file. + self-contained multiaddress, retrieves every record, and reconstructs the + exact file. - Before acceptance, run interactive tests on current Chrome, Firefox, and Safari and add shared lookup convergence vectors with the native client. - Revisit this decision when streaming file reconstruction or production signed diff --git a/web/README.md b/web/README.md index dbbdaf20..ad07d7c0 100644 --- a/web/README.md +++ b/web/README.md @@ -14,7 +14,11 @@ proxies file bytes. - Rust 1.88 or newer for the node's optional `wtransport` dependency. - Node.js 20.19+ or 22.12+. -- A current browser implementing WebTransport certificate hashes. +- A current browser implementing WebTransport certificate hashes. The client + extracts them from node multiaddresses; users do not enter hashes separately. + +Nodes serialize these addresses from the native `saorsa_core::MultiAddr` +representation; the JavaScript parser consumes that canonical string form. ## Run the browser-enabled testnet @@ -36,7 +40,8 @@ listeners on UDP 24000-24004. It also: - self-encrypts the built-in `autonomi-browser-testnet.txt` and publishes its encrypted chunks and public DataMap through the ordinary node PUT handler using devnet-prepaid cache entries; -- exposes all direct node URLs and certificate pins at +- exposes all direct node multiaddresses, including their certificate pins and + peer IDs, at `http://127.0.0.1:25000/api/browser-manifest.json`; - includes the public DataMap address, plaintext BLAKE3 hash, resolved chunk metadata, filename, size, and replica count in that manifest. @@ -57,8 +62,9 @@ npm run dev Open `http://127.0.0.1:5173`. The page automatically loads the testnet manifest from port 25000 and fills in the default file: -1. **Load testnet** refreshes the manifest and direct endpoint catalog. -2. **Connect** performs a pinned WebTransport `HELLO` with the first node. +1. **Load testnet** refreshes the manifest and direct multiaddress catalog. +2. **Connect** parses the first node multiaddress and performs a pinned + WebTransport `HELLO`. 3. **Find closest** runs the iterative lookup in the browser. 4. **Download and save file** opens the browser save flow, fetches the public DataMap and encrypted chunks from direct closest storage nodes, reconstructs diff --git a/web/index.html b/web/index.html index 9578a3c2..e624fb5c 100644 --- a/web/index.html +++ b/web/index.html @@ -47,23 +47,15 @@

Local testnet

Direct node endpoint

- Populated from the testnet manifest. It can also be entered manually. + The certificate pins and peer identity are embedded in this address. + It is populated from the testnet manifest and can also be pasted manually.

- diff --git a/web/src/main.js b/web/src/main.js index 70536d5a..176d303d 100644 --- a/web/src/main.js +++ b/web/src/main.js @@ -17,8 +17,7 @@ const elements = { publicFileAddress: document.querySelector("#public-file-address"), publicFileChunks: document.querySelector("#public-file-chunks"), publicFileReplicas: document.querySelector("#public-file-replicas"), - endpointUrl: document.querySelector("#endpoint-url"), - certificateHash: document.querySelector("#certificate-hash"), + endpointMultiaddr: document.querySelector("#endpoint-multiaddr"), connect: document.querySelector("#connect"), connectionState: document.querySelector("#connection-state"), lookupTarget: document.querySelector("#lookup-target"), @@ -46,10 +45,7 @@ function log(message, value) { } function endpointFromForm() { - return { - url: elements.endpointUrl.value.trim(), - certificateSha256: elements.certificateHash.value.trim(), - }; + return elements.endpointMultiaddr.value.trim(); } function seedEndpoints() { @@ -65,8 +61,7 @@ async function loadManifest() { browserManifest = manifest; const first = manifest.endpoints[0]; - elements.endpointUrl.value = first.url; - elements.certificateHash.value = first.certificate_sha256; + elements.endpointMultiaddr.value = first.multiaddr; client?.close(); client = undefined; elements.connectionState.classList.remove("connected"); diff --git a/web/src/manifest.js b/web/src/manifest.js index 9bd8a5d6..214e4b89 100644 --- a/web/src/manifest.js +++ b/web/src/manifest.js @@ -1,6 +1,6 @@ -import { hexToBytes } from "./protocol.js"; +import { hexToBytes, parseWebTransportMultiaddr } from "./protocol.js"; -export const BROWSER_MANIFEST_VERSION = 2; +export const BROWSER_MANIFEST_VERSION = 3; const MAX_PUBLIC_FILE_BYTES = 64 * 1024 * 1024; const MAX_DATA_MAP_BYTES = 4 * 1024 * 1024; const MAX_FILE_CHUNKS = 1024; @@ -16,19 +16,8 @@ export function parseBrowserManifest(value) { throw new Error("Browser manifest contains no WebTransport endpoints"); } const endpoints = value.endpoints.map((endpoint) => { - if (!endpoint || typeof endpoint.url !== "string") { - throw new Error("Browser manifest endpoint has no URL"); - } - if (!endpoint.url.startsWith("https://")) { - throw new Error(`WebTransport endpoint must use HTTPS: ${endpoint.url}`); - } - hexToBytes(endpoint.peer_id ?? "", 32); - hexToBytes(endpoint.certificate_sha256 ?? "", 32); - return { - peer_id: endpoint.peer_id.toLowerCase(), - url: endpoint.url, - certificate_sha256: endpoint.certificate_sha256.toLowerCase(), - }; + const parsed = parseWebTransportMultiaddr(endpoint); + return { multiaddr: parsed.multiaddr }; }); const files = (value.files ?? []).map((file) => { diff --git a/web/src/manifest.test.js b/web/src/manifest.test.js index f9949eb3..4ff4d469 100644 --- a/web/src/manifest.test.js +++ b/web/src/manifest.test.js @@ -4,14 +4,12 @@ import { parseBrowserManifest } from "./manifest.js"; test("browser manifest validates and normalizes endpoints and files", () => { const manifest = parseBrowserManifest({ - version: 2, + version: 3, network_id: "local-test", created_at: "2026-08-03T00:00:00Z", endpoints: [ { - peer_id: "AA".repeat(32), - url: "https://127.0.0.1:22000/autonomi/webtransport/v1", - certificate_sha256: "BB".repeat(32), + multiaddr: webtransportMultiaddr("AA".repeat(32), 0xbb), }, ], files: [ @@ -32,7 +30,10 @@ test("browser manifest validates and normalizes endpoints and files", () => { ], }); - assert.equal(manifest.endpoints[0].peer_id, "aa".repeat(32)); + assert.equal( + manifest.endpoints[0].multiaddr, + webtransportMultiaddr("AA".repeat(32), 0xbb), + ); assert.equal(manifest.files[0].address, "cc".repeat(32)); assert.equal(manifest.files[0].blake3, "dd".repeat(32)); assert.deepEqual( @@ -42,24 +43,33 @@ test("browser manifest validates and normalizes endpoints and files", () => { assert.equal(manifest.files[0].replicas, 5); }); -test("browser manifest rejects missing endpoints and malformed hashes", () => { +test("browser manifest rejects missing endpoints and malformed multiaddresses", () => { assert.throws( - () => parseBrowserManifest({ version: 2, network_id: "test", endpoints: [] }), + () => parseBrowserManifest({ version: 3, network_id: "test", endpoints: [] }), /no WebTransport endpoints/, ); assert.throws( () => parseBrowserManifest({ - version: 2, + version: 3, network_id: "test", endpoints: [ { - peer_id: "wrong", - url: "https://127.0.0.1:22000/path", - certificate_sha256: "bb".repeat(32), + multiaddr: + "/ip4/127.0.0.1/udp/22000/quic-v1/webtransport/certhash/uAA/p2p/wrong", }, ], }), - /hexadecimal|Expected 32 bytes/, + /multihash|hexadecimal|Expected 32 bytes/, ); }); + +function webtransportMultiaddr(peerId, certificateByte) { + const multihash = Uint8Array.from([ + 0x12, + 0x20, + ...Array(32).fill(certificateByte), + ]); + const certhash = `u${Buffer.from(multihash).toString("base64url")}`; + return `/ip4/127.0.0.1/udp/22000/quic-v1/webtransport/certhash/${certhash}/p2p/${peerId}`; +} diff --git a/web/src/protocol.js b/web/src/protocol.js index 7779fc5e..d6ee7988 100644 --- a/web/src/protocol.js +++ b/web/src/protocol.js @@ -1,11 +1,16 @@ import { blake3 } from "@noble/hashes/blake3.js"; import { bytesToHex as nobleBytesToHex } from "@noble/hashes/utils.js"; -export const PROTOCOL_VERSION = 1; -export const PROTOCOL_NAME = "autonomi.web.poc.v1"; +export const PROTOCOL_VERSION = 2; +export const PROTOCOL_NAME = "autonomi.web.poc.v2"; +export const WEBTRANSPORT_PATH = "/autonomi/webtransport/v1"; export const MAX_CHUNK_SIZE = 4 * 1024 * 1024; export const MAX_RESPONSE_HEADER_BYTES = 64 * 1024; const MAX_RESPONSE_BYTES = 4 + MAX_RESPONSE_HEADER_BYTES + MAX_CHUNK_SIZE; +const MAX_WEBTRANSPORT_MULTIADDR_LENGTH = 2048; +const MAX_CERTIFICATE_HASHES = 2; +const SHA2_256_MULTIHASH_CODE = 0x12; +const SHA2_256_MULTIHASH_LENGTH = 32; const encoder = new TextEncoder(); const decoder = new TextDecoder("utf-8", { fatal: true }); @@ -118,20 +123,145 @@ async function readAll(readable, limit = MAX_RESPONSE_BYTES) { return result; } -function normalizeEndpoint(endpoint) { - if (!endpoint || typeof endpoint.url !== "string") { - throw new Error("Endpoint URL is required"); +function decodeBase64Url(value) { + if (!/^[A-Za-z0-9_-]+$/.test(value)) { + throw new Error("Certificate multihash is not valid unpadded base64url"); + } + const padding = "=".repeat((4 - (value.length % 4)) % 4); + let binary; + try { + binary = atob(value.replaceAll("-", "+").replaceAll("_", "/") + padding); + } catch (error) { + throw new Error("Certificate multihash is not valid unpadded base64url", { + cause: error, + }); + } + return Uint8Array.from(binary, (character) => character.charCodeAt(0)); +} + +function decodeCertificateMultihash(value) { + if (typeof value !== "string" || !value.startsWith("u")) { + throw new Error("Certificate multihash must use base64url multibase (`u`)"); + } + const decoded = decodeBase64Url(value.slice(1)); + if ( + decoded.length !== 34 || + decoded[0] !== SHA2_256_MULTIHASH_CODE || + decoded[1] !== SHA2_256_MULTIHASH_LENGTH + ) { + throw new Error("Certificate multihash must contain a 32-byte SHA-256 digest"); + } + return decoded.slice(2); +} + +function validateIpv4(value) { + const octets = value.split("."); + if ( + octets.length !== 4 || + octets.some( + (octet) => + !/^(0|[1-9][0-9]{0,2})$/.test(octet) || Number.parseInt(octet, 10) > 255, + ) + ) { + throw new Error(`Invalid IPv4 address ${value}`); + } + return value; +} + +function endpointMultiaddr(endpoint) { + if (typeof endpoint === "string") return endpoint; + if (endpoint && typeof endpoint.multiaddr === "string") return endpoint.multiaddr; + throw new Error("A WebTransport multiaddress is required"); +} + +export function parseWebTransportMultiaddr(endpoint) { + const multiaddr = endpointMultiaddr(endpoint).trim(); + if ( + multiaddr.length === 0 || + multiaddr.length > MAX_WEBTRANSPORT_MULTIADDR_LENGTH || + !multiaddr.startsWith("/") + ) { + throw new Error("Invalid WebTransport multiaddress length or prefix"); + } + const segments = multiaddr.split("/"); + if (segments.length < 9) { + throw new Error("WebTransport multiaddress is incomplete"); + } + + const hostProtocol = segments[1]; + const hostValue = segments[2]; + if (!hostValue) throw new Error("WebTransport multiaddress host is empty"); + let urlHost; + if (hostProtocol === "ip4") { + urlHost = validateIpv4(hostValue); + } else if (hostProtocol === "ip6") { + urlHost = `[${hostValue}]`; + } else if (["dns", "dns4", "dns6"].includes(hostProtocol)) { + urlHost = hostValue.toLowerCase(); + } else { + throw new Error(`Unsupported WebTransport host protocol ${hostProtocol}`); + } + if (segments[3] !== "udp") { + throw new Error("WebTransport multiaddress must use UDP"); + } + if (!/^[0-9]{1,5}$/.test(segments[4])) { + throw new Error("WebTransport multiaddress has an invalid UDP port"); + } + const port = Number.parseInt(segments[4], 10); + if (port < 1 || port > 65535) { + throw new Error("WebTransport multiaddress has an invalid UDP port"); + } + if (segments[5] !== "quic-v1" || segments[6] !== "webtransport") { + throw new Error( + "WebTransport multiaddress must contain /quic-v1/webtransport", + ); + } + + let index = 7; + const certificateHashes = []; + const certificateHashMultihashes = []; + while (segments[index] === "certhash") { + const encoded = segments[index + 1]; + if (!encoded) throw new Error("WebTransport multiaddress has an empty certhash"); + certificateHashes.push(decodeCertificateMultihash(encoded)); + certificateHashMultihashes.push(encoded); + index += 2; + } + if ( + certificateHashes.length < 1 || + certificateHashes.length > MAX_CERTIFICATE_HASHES + ) { + throw new Error( + `WebTransport multiaddress must contain between 1 and ${MAX_CERTIFICATE_HASHES} certificate hashes`, + ); + } + if (new Set(certificateHashMultihashes).size !== certificateHashes.length) { + throw new Error("WebTransport multiaddress contains duplicate certificate hashes"); + } + if (segments[index] !== "p2p" || index + 2 !== segments.length) { + throw new Error("WebTransport multiaddress must end with /p2p/"); + } + const peerId = segments[index + 1]?.toLowerCase() ?? ""; + hexToBytes(peerId, 32); + + let url; + try { + url = new URL(`https://${urlHost}:${port}${WEBTRANSPORT_PATH}`).toString(); + } catch (error) { + throw new Error("WebTransport multiaddress contains an invalid host", { + cause: error, + }); } - const certificateSha256 = - endpoint.certificate_sha256 ?? endpoint.certificateSha256; return { - url: endpoint.url, - peerId: endpoint.peer_id ?? endpoint.peerId, - certificateSha256, - certificateBytes: hexToBytes(certificateSha256 ?? "", 32), + multiaddr, + url, + peerId, + certificateHashes, }; } +const normalizeEndpoint = parseWebTransportMultiaddr; + export class BrowserNodeClient { constructor(endpoint) { this.endpoint = normalizeEndpoint(endpoint); @@ -145,9 +275,10 @@ export class BrowserNodeClient { throw new Error("This browser does not expose the WebTransport API"); } const transport = new WebTransport(this.endpoint.url, { - serverCertificateHashes: [ - { algorithm: "sha-256", value: this.endpoint.certificateBytes }, - ], + serverCertificateHashes: this.endpoint.certificateHashes.map((value) => ({ + algorithm: "sha-256", + value, + })), }); try { await transport.ready; @@ -200,15 +331,14 @@ export class BrowserNodeClient { if (header.protocol !== PROTOCOL_NAME) { throw new Error(`Unsupported browser protocol ${header.protocol}`); } + hexToBytes(header.peer_id, 32); const advertisedEndpoint = normalizeEndpoint(header.endpoint); if ( - advertisedEndpoint.url !== this.endpoint.url || - advertisedEndpoint.certificateSha256.toLowerCase() !== - this.endpoint.certificateSha256.toLowerCase() + advertisedEndpoint.multiaddr !== this.endpoint.multiaddr || + advertisedEndpoint.peerId !== header.peer_id.toLowerCase() ) { throw new Error("Node advertised a different WebTransport endpoint"); } - hexToBytes(header.peer_id, 32); if ( this.endpoint.peerId && header.peer_id.toLowerCase() !== this.endpoint.peerId.toLowerCase() @@ -231,7 +361,15 @@ export class BrowserNodeClient { if (!Array.isArray(header.nodes)) { throw new Error("Node returned an invalid node list"); } - for (const node of header.nodes) hexToBytes(node.peer_id, 32); + for (const node of header.nodes) { + hexToBytes(node.peer_id, 32); + if (node.webtransport) { + const parsed = normalizeEndpoint(node.webtransport); + if (parsed.peerId !== node.peer_id.toLowerCase()) { + throw new Error(`Node ${node.peer_id} advertised another peer's endpoint`); + } + } + } return header.nodes; } @@ -264,7 +402,7 @@ export class BrowserNodeClient { function endpointKey(endpoint) { const normalized = normalizeEndpoint(endpoint); - return `${normalized.url}|${normalized.certificateSha256}`; + return normalized.multiaddr; } export async function iterativeFindClosest( @@ -294,7 +432,8 @@ export async function iterativeFindClosest( await Promise.all( seedEndpoints.map(async (endpoint) => { - const seedName = endpoint?.peer_id ?? endpoint?.peerId ?? endpoint?.url ?? "seed"; + const seedName = + typeof endpoint === "string" ? endpoint : endpoint?.multiaddr ?? "seed"; try { const client = clientFor(endpoint); const hello = await client.hello(); @@ -333,10 +472,7 @@ export async function iterativeFindClosest( candidates.map(async (candidate) => { queried.add(candidate.peer_id); try { - const candidateClient = clientFor({ - ...candidate.webtransport, - peer_id: candidate.peer_id, - }); + const candidateClient = clientFor(candidate.webtransport); if (!candidateClient.peerId) await candidateClient.hello(); const nodes = await candidateClient.findNode(target, k); onProgress( @@ -395,7 +531,7 @@ export async function getChunkFromClosest( try { for (const node of lookup.nodes) { if (!node.webtransport) continue; - const endpoint = { ...node.webtransport, peer_id: node.peer_id }; + const endpoint = node.webtransport; const key = endpointKey(endpoint); let client = lookup.clients.get(key); if (!client) { diff --git a/web/src/protocol.test.js b/web/src/protocol.test.js index a8a48bfc..7dbfc4e2 100644 --- a/web/src/protocol.test.js +++ b/web/src/protocol.test.js @@ -6,6 +6,7 @@ import { getChunkFromClosest, hexToBytes, parseResponseFrame, + parseWebTransportMultiaddr, verifyChunk, xorDistance, } from "./protocol.js"; @@ -26,7 +27,7 @@ test("XOR distance is an unsigned 256-bit ordering value", () => { test("response framing preserves a raw binary body", () => { const header = new TextEncoder().encode( JSON.stringify({ - version: 1, + version: 2, request_id: 9, status: "ok", content_length: 3, @@ -45,6 +46,29 @@ test("response framing preserves a raw binary body", () => { assert.deepEqual([...parsed.content], [1, 2, 3]); }); +test("WebTransport multiaddresses carry current and next certificate hashes", () => { + const peerId = "ab".repeat(32); + const multiaddr = webtransportMultiaddr("ip4", "127.0.0.1", 24000, peerId, [ + 0x11, + 0x22, + ]); + const parsed = parseWebTransportMultiaddr(multiaddr); + + assert.equal(parsed.url, "https://127.0.0.1:24000/autonomi/webtransport/v1"); + assert.equal(parsed.peerId, peerId); + assert.deepEqual( + parsed.certificateHashes.map((hash) => [...hash]), + [Array(32).fill(0x11), Array(32).fill(0x22)], + ); + assert.throws( + () => + parseWebTransportMultiaddr( + `/ip4/127.0.0.1/udp/24000/quic-v1/webtransport/p2p/${peerId}`, + ), + /certificate hashes/, + ); +}); + test("BLAKE3 verification accepts the canonical empty hash", () => { const emptyHash = "af1349b9f5f9a1a6a0404dea36dcc9499bcb25c9adc112b7cc9a93cae41f3262"; @@ -57,16 +81,8 @@ test("browser lookup discovers a direct node and downloads a verified chunk", as const address = bytesToHex(blake3(content)); const seedPeer = "ff".repeat(32); const storagePeer = address; - const seed = { - peer_id: seedPeer, - url: "https://seed.test/autonomi/webtransport/v1", - certificate_sha256: "11".repeat(32), - }; - const storage = { - peer_id: storagePeer, - url: "https://storage.test/autonomi/webtransport/v1", - certificate_sha256: "22".repeat(32), - }; + const seed = endpoint("seed.test", seedPeer, 0x11); + const storage = endpoint("storage.test", storagePeer, 0x22); const calls = []; const routes = new Map([ [ @@ -114,7 +130,7 @@ test("browser lookup discovers a direct node and downloads a verified chunk", as globalThis.WebTransport = previousWebTransport; }); - const downloaded = await getChunkFromClosest([seed], address); + const downloaded = await getChunkFromClosest([seed.multiaddr], address); assert.deepEqual(downloaded.content, content); assert.equal(downloaded.hash, address); assert.equal(downloaded.node.peer_id, storagePeer); @@ -133,8 +149,7 @@ function browserNode(endpoint) { native_addresses: [], reliability: 1, webtransport: { - url: endpoint.url, - certificate_sha256: endpoint.certificate_sha256, + multiaddr: endpoint.multiaddr, }, }; } @@ -142,12 +157,11 @@ function browserNode(endpoint) { function helloResponse(request, endpoint) { return response(request, { type: "hello", - protocol: "autonomi.web.poc.v1", + protocol: "autonomi.web.poc.v2", peer_id: endpoint.peer_id, max_chunk_size: 4 * 1024 * 1024, endpoint: { - url: endpoint.url, - certificate_sha256: endpoint.certificate_sha256, + multiaddr: endpoint.multiaddr, }, capabilities: ["find_node", "get_chunk"], }); @@ -156,7 +170,7 @@ function helloResponse(request, endpoint) { function response(request, fields, content = new Uint8Array()) { return { header: { - version: 1, + version: 2, request_id: request.request_id, status: "ok", content_length: content.length, @@ -166,6 +180,26 @@ function response(request, fields, content = new Uint8Array()) { }; } +function certificateMultihash(byte) { + const multihash = Uint8Array.from([0x12, 0x20, ...Array(32).fill(byte)]); + return `u${Buffer.from(multihash).toString("base64url")}`; +} + +function webtransportMultiaddr(hostProtocol, host, port, peerId, hashBytes) { + const hashes = hashBytes + .map((byte) => `/certhash/${certificateMultihash(byte)}`) + .join(""); + return `/${hostProtocol}/${host}/udp/${port}/quic-v1/webtransport${hashes}/p2p/${peerId}`; +} + +function endpoint(host, peerId, certificateByte) { + return { + peer_id: peerId, + url: `https://${host}/autonomi/webtransport/v1`, + multiaddr: webtransportMultiaddr("dns", host, 443, peerId, [certificateByte]), + }; +} + function encodeResponse({ header, content }) { const headerBytes = new TextEncoder().encode(JSON.stringify(header)); const frame = new Uint8Array(4 + headerBytes.length + content.length); From a6c4c5a12a0d00019ddfe2ddb564c1e4e25a88d3 Mon Sep 17 00:00:00 2001 From: Mick van Dijke <12992260+mickvandijke@users.noreply.github.com> Date: Wed, 5 Aug 2026 14:17:36 +0200 Subject: [PATCH 03/31] feat(web): upload paid public files --- .../ADR-0003-direct-browser-read-client.md | 52 ++- web/README.md | 31 +- web/index.html | 40 ++- web/package-lock.json | 183 +++++++++- web/package.json | 6 +- web/src/main.js | 64 ++++ web/src/manifest.js | 29 +- web/src/manifest.test.js | 41 ++- web/src/payment.js | 306 ++++++++++++++++ web/src/payment.test.js | 235 +++++++++++++ web/src/protocol.js | 67 +++- web/src/protocol.test.js | 60 +++- web/src/style.css | 13 + web/src/upload.js | 326 ++++++++++++++++++ web/src/upload.test.js | 81 +++++ 15 files changed, 1488 insertions(+), 46 deletions(-) create mode 100644 web/src/payment.js create mode 100644 web/src/payment.test.js create mode 100644 web/src/upload.js create mode 100644 web/src/upload.test.js diff --git a/docs/adr/ADR-0003-direct-browser-read-client.md b/docs/adr/ADR-0003-direct-browser-read-client.md index 8e94e1e4..e6121b67 100644 --- a/docs/adr/ADR-0003-direct-browser-read-client.md +++ b/docs/adr/ADR-0003-direct-browser-read-client.md @@ -1,8 +1,8 @@ -# ADR-0003: Direct browser read client over WebTransport +# ADR-0003: Direct browser immutable-data client over WebTransport - **Status:** Proposed - **Date:** 2026-08-03 -- **Last amended:** 2026-08-04 +- **Last amended:** 2026-08-05 - **Decision owners:** - **Reviewers:** - **Supersedes:** none @@ -11,15 +11,16 @@ ## Context -The Autonomi web client must perform closest-node lookup and immutable-data -download itself. Sending those operations through an HTTP application gateway -would make the gateway an availability, privacy, and bandwidth chokepoint. +The Autonomi web client must perform closest-node lookup, immutable-data +download, quote verification, payment, and upload itself. Sending those +operations through an HTTP application gateway would make the gateway an +availability, privacy, and bandwidth chokepoint. Browsers cannot use the native Saorsa QUIC protocol, but they can establish WebTransport sessions with browser-compatible node listeners. This repository owns the client and UI side of that split. Nodes own transport -termination, local DHT answers, storage reads, endpoint records, and testnet -bootstrap-manifest production under ant-node ADR-0009. +termination, local DHT answers, storage reads and paid writes, endpoint +records, and testnet bootstrap-manifest production under ant-node ADR-0009. ## Decision Drivers @@ -28,6 +29,8 @@ bootstrap-manifest production under ant-node ADR-0009. - Self-signed node certificates must be authenticated through hashes embedded in self-contained node multiaddresses, without a separate client argument. - Downloaded immutable content must be verified before it is exposed to users. +- The wallet secret must be provided at runtime, used only by the local EVM + signer, and never sent to a node or persisted in bootstrap metadata. - Local testnets need a reproducible bootstrap and default-file workflow. ## Considered Options @@ -37,13 +40,13 @@ bootstrap-manifest production under ant-node ADR-0009. 2. **Compile the complete native Rust client to WebAssembly.** Deferred because the native transport, EVM, and filesystem dependency graph is not currently browser-compatible. -3. **Implement a narrow JavaScript WebTransport read client (chosen).** It maps - directly to the versioned browser node protocol and keeps the application - boundary small enough to audit. +3. **Implement a narrow JavaScript WebTransport immutable-data client + (chosen).** It maps directly to the versioned browser node protocol and + keeps the application boundary small enough to audit. ## Decision -The `web/` package will implement the direct browser read client: +The `web/` package will implement the direct browser client: - load a versioned browser bootstrap manifest containing WebTransport multiaddresses and published immutable-file metadata; @@ -55,13 +58,23 @@ The `web/` package will implement the direct browser read client: and `ALPHA = 3`; - query closest direct endpoints with `GET_CHUNK`, retrying `not_found` and unavailable nodes without routing bytes through the manifest service; +- self-encrypt selected public files with the native `self_encryption 0.36` + format and generate the public MessagePack DataMap; +- request ordinary node storage quotes, verify their ML-DSA peer/content + binding, forced price, and signed storage commitment before payment; +- construct an EVM wallet only from the runtime secret field, approve the + public vault when required, and make one batched `payForQuotes` transaction; +- upload each content-addressed encrypted record with the signed quote and + transaction hash through paid `PUT_CHUNK`; the wallet key never crosses the + WebTransport session; - fetch the public MessagePack DataMap and every resolved encrypted data chunk; - reconstruct the file with the native `self_encryption 0.36` BLAKE3 KDF, ChaCha20-Poly1305 authentication, and Brotli decompression; - verify encrypted-record addresses, per-chunk plaintext hashes and sizes, and the final whole-file BLAKE3 hash before allowing a save; - expose a small test site that loads the local testnet manifest, displays the - startup-published file, and downloads it through the browser save flow. + startup-published file, uploads paid files, and downloads either through the + browser save flow. The local browser manifest is bootstrap metadata, not a gateway. Production clients will replace its unsigned endpoint list with the ML-DSA-signed records @@ -92,7 +105,8 @@ metadata chain. ### Positive -- Lookup and data transfer remain decentralized at the application layer. +- Lookup, payment, and data transfer remain decentralized at the application + layer. - A local five-node testnet can validate multiple direct node connections, multiaddress-embedded certificate pins, lookup convergence, fallback, and content verification. @@ -101,7 +115,8 @@ metadata chain. ### Negative / Trade-offs - The current client reconstructs files in memory and the local launcher caps - public files at 64 MiB; streaming-to-disk is not yet implemented. + public files at 64 MiB; upload encryption and reconstruction are not yet + streaming. - JavaScript lookup behavior must remain aligned with native Kademlia rules. - Certificate and endpoint verification adds bootstrap-record lifecycle work. @@ -114,15 +129,18 @@ metadata chain. ## Validation -- Unit tests cover fixed-width identifiers, XOR ordering, response framing, - manifest validation, and BLAKE3 mismatch rejection. +- Unit tests cover fixed-width identifiers, XOR ordering, bidirectional binary + framing, manifest/payment validation, quote signatures and the native + Keccak-256 EVM quote-hash vector, native-format + encryption/DataMap generation, and BLAKE3 mismatch rejection. - The browser production bundle builds without Node-specific runtime APIs. - A fixed vector generated by native `self_encryption 0.36` verifies browser KDF, authenticated decryption, Brotli reconstruction, and tamper rejection. - A live node integration test starts five WebTransport-enabled nodes, publishes a public DataMap and encrypted chunks, connects with the advertised self-contained multiaddress, retrieves every record, and reconstructs the - exact file. + exact file, pays a real signed quote, accepts a paid binary PUT through the + ordinary node verifier, and reads the stored record back. - Before acceptance, run interactive tests on current Chrome, Firefox, and Safari and add shared lookup convergence vectors with the native client. - Revisit this decision when streaming file reconstruction or production signed diff --git a/web/README.md b/web/README.md index ad07d7c0..e60c42fe 100644 --- a/web/README.md +++ b/web/README.md @@ -4,7 +4,9 @@ This web application is the browser-facing client for ADR-0009. It loads a local testnet bootstrap manifest, connects directly to storage nodes over WebTransport, performs the XOR closest-node lookup in JavaScript, retrieves a public DataMap and every encrypted file chunk, reconstructs the complete file, -and verifies its whole-file BLAKE3 hash before saving it. +and verifies its whole-file BLAKE3 hash before saving it. It can also +self-encrypt a file, pay signed node quotes with a wallet held only in the +page, and upload the encrypted records directly to closest storage nodes. The node-side WebTransport listener and testnet manifest API live in the `ant-node-web-support` sibling repository. No HTTP gateway performs lookup or @@ -31,6 +33,7 @@ cargo run --features webtransport-poc --bin ant-devnet -- \ --webtransport \ --webtransport-base-port 24000 \ --serve-port 25000 \ + --enable-evm \ --enable-logging ``` @@ -45,6 +48,13 @@ listeners on UDP 24000-24004. It also: `http://127.0.0.1:25000/api/browser-manifest.json`; - includes the public DataMap address, plaintext BLAKE3 hash, resolved chunk metadata, filename, size, and replica count in that manifest. +- starts local Anvil payment contracts and prints a disposable funded wallet + private key. The browser manifest contains only public RPC and contract + addresses, never the private key. + +Confirm that `HELLO.payment.rpc_url` is a loopback Anvil URL. An +`https://arb1.arbitrum.io/rpc` value means an older devnet was started without +`--enable-evm`; its disposable local key cannot fund uploads there. Pass `--public-file /path/to/file` to publish another file instead. The built-in file is generated as 5 MiB so the demo exercises whole-file reconstruction. A @@ -66,7 +76,12 @@ manifest from port 25000 and fills in the default file: 2. **Connect** parses the first node multiaddress and performs a pinned WebTransport `HELLO`. 3. **Find closest** runs the iterative lookup in the browser. -4. **Download and save file** opens the browser save flow, fetches the public +4. Under **Paid public file upload**, choose a file, paste the funded private + key printed by ant-devnet, then select **Pay and upload file**. Encryption, + quote/commitment verification, approval, and payment happen locally. The + key field is cleared immediately and the resulting public DataMap address + is placed in the download field. +5. **Download and save file** opens the browser save flow, fetches the public DataMap and encrypted chunks from direct closest storage nodes, reconstructs the whole file, verifies BLAKE3, and retains a **Save again** link. @@ -86,11 +101,15 @@ npm test npm run build ``` -The tests cover fixed-width IDs, XOR ordering, response framing, browser -manifest validation, and a native `self_encryption 0.36` compatibility vector. +The tests cover fixed-width IDs, XOR ordering, bidirectional binary framing, +browser manifest/payment validation, signed quote verification including the +native Keccak-256 EVM quote hash, native +encryption/DataMap generation, and a `self_encryption 0.36` compatibility +vector. Cross-repository live verification additionally starts the node testnet, -downloads all public-file records through WebTransport, and reconstructs the -original bytes. +downloads all public-file records through WebTransport, reconstructs the +original bytes, pays a real quote on local Anvil, uploads a fresh record +through the ordinary node payment validator, and reads it back. ## Current boundary diff --git a/web/index.html b/web/index.html index e624fb5c..8da7ece3 100644 --- a/web/index.html +++ b/web/index.html @@ -17,7 +17,7 @@

Direct browser client

Connect directly to an ant-node, query its local closest-node view, - reconstruct a complete public file, and save it from the browser. + upload a paid public file, or reconstruct and save one from the browser.

@@ -79,6 +79,44 @@

Browser-side closest lookup

+
+

Paid public file upload

+

+ Self-encrypts the file in this page, verifies signed storage quotes, pays + the testnet vault, and sends encrypted records directly to closest nodes. +

+

+ The wallet key is used only by this page to sign the EVM transactions. It + is cleared from the form immediately and is never sent to a node or stored + in the manifest. Use the disposable funded key printed by ant-devnet. +

+ + +
+ + +
+ +
+

Direct public file download

diff --git a/web/package-lock.json b/web/package-lock.json index 8f5fd2f9..a3490305 100644 --- a/web/package-lock.json +++ b/web/package-lock.json @@ -8,14 +8,33 @@ "name": "ant-client-web", "version": "0.1.0", "dependencies": { + "@msgpack/msgpack": "^3.1.3", "@noble/ciphers": "2.2.0", "@noble/hashes": "2.2.0", - "brotli-dec-wasm": "2.3.2" + "@noble/post-quantum": "^0.6.1", + "brotli-dec-wasm": "2.3.2", + "brotli-wasm": "^3.0.1", + "ethers": "^6.17.0" }, "devDependencies": { "vite": "8.2.0" } }, + "node_modules/@adraffy/ens-normalize": { + "version": "1.11.1", + "resolved": "https://registry.npmjs.org/@adraffy/ens-normalize/-/ens-normalize-1.11.1.tgz", + "integrity": "sha512-nhCBV3quEgesuf7c7KYfperqSS14T8bYuvJ8PcLJp6znkZpFc0AuW4qBtr8eKVyPPe/8RSr7sglCWPU5eaxwKQ==", + "license": "MIT" + }, + "node_modules/@msgpack/msgpack": { + "version": "3.1.3", + "resolved": "https://registry.npmjs.org/@msgpack/msgpack/-/msgpack-3.1.3.tgz", + "integrity": "sha512-47XIizs9XZXvuJgoaJUIE2lFoID8ugvc0jzSHP+Ptfk8nTbnR8g788wv48N03Kx0UkAv559HWRQ3yzOgzlRNUA==", + "license": "ISC", + "engines": { + "node": ">= 18" + } + }, "node_modules/@noble/ciphers": { "version": "2.2.0", "resolved": "https://registry.npmjs.org/@noble/ciphers/-/ciphers-2.2.0.tgz", @@ -28,6 +47,21 @@ "url": "https://paulmillr.com/funding/" } }, + "node_modules/@noble/curves": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/@noble/curves/-/curves-2.2.0.tgz", + "integrity": "sha512-T/BoHgFXirb0ENSPBquzX0rcjXeM6Lo892a2jlYJkqk83LqZx0l1Of7DzlKJ6jkpvMrkHSnAcgb5JegL8SeIkQ==", + "license": "MIT", + "dependencies": { + "@noble/hashes": "2.2.0" + }, + "engines": { + "node": ">= 20.19.0" + }, + "funding": { + "url": "https://paulmillr.com/funding/" + } + }, "node_modules/@noble/hashes": { "version": "2.2.0", "resolved": "https://registry.npmjs.org/@noble/hashes/-/hashes-2.2.0.tgz", @@ -40,6 +74,23 @@ "url": "https://paulmillr.com/funding/" } }, + "node_modules/@noble/post-quantum": { + "version": "0.6.1", + "resolved": "https://registry.npmjs.org/@noble/post-quantum/-/post-quantum-0.6.1.tgz", + "integrity": "sha512-+pormrDZwjRw05U8ADK4JpHejo87+gBd+muRBB/ozztH5yhDLMDF4jHQWN3NQQAsu1zBNPWTG0ZwVI0CR29H0A==", + "license": "MIT", + "dependencies": { + "@noble/ciphers": "~2.2.0", + "@noble/curves": "~2.2.0", + "@noble/hashes": "~2.2.0" + }, + "engines": { + "node": ">= 20.19.0" + }, + "funding": { + "url": "https://paulmillr.com/funding/" + } + }, "node_modules/@oxc-project/types": { "version": "0.142.0", "resolved": "https://registry.npmjs.org/@oxc-project/types/-/types-0.142.0.tgz", @@ -295,12 +346,39 @@ "dev": true, "license": "MIT" }, + "node_modules/@types/node": { + "version": "26.1.2", + "resolved": "https://registry.npmjs.org/@types/node/-/node-26.1.2.tgz", + "integrity": "sha512-Vu4a5UFA9rIIFJ7rB/Vaafh9lrCQszopTCx6KjFboXTGQbPNasehVR5TEiithSDGyd1DEiUByggTZsg8jukeIg==", + "dev": true, + "license": "MIT", + "optional": true, + "peer": true, + "dependencies": { + "undici-types": "~8.3.0" + } + }, + "node_modules/aes-js": { + "version": "4.0.0-beta.5", + "resolved": "https://registry.npmjs.org/aes-js/-/aes-js-4.0.0-beta.5.tgz", + "integrity": "sha512-G965FqalsNyrPqgEGON7nIx1e/OVENSgiEIzyC63haUMuvNnwIgIjMs52hlTCKhkBny7A2ORNlfY9Zu+jmGk1Q==", + "license": "MIT" + }, "node_modules/brotli-dec-wasm": { "version": "2.3.2", "resolved": "https://registry.npmjs.org/brotli-dec-wasm/-/brotli-dec-wasm-2.3.2.tgz", "integrity": "sha512-5H+k8eVLIJY6B4olN2HP9QzJAxcplf0jV7mWnkpxvOSeUE9Npg3dQ2pgLn30a9MUFHNro1iSmtdu6VNdlb+TIw==", "license": "MIT OR Apache-2.0" }, + "node_modules/brotli-wasm": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/brotli-wasm/-/brotli-wasm-3.0.1.tgz", + "integrity": "sha512-U3K72/JAi3jITpdhZBqzSUq+DUY697tLxOuFXB+FpAE/Ug+5C3VZrv4uA674EUZHxNAuQ9wETXNqQkxZD6oL4A==", + "license": "Apache-2.0", + "engines": { + "node": ">=v18.0.0" + } + }, "node_modules/detect-libc": { "version": "2.1.2", "resolved": "https://registry.npmjs.org/detect-libc/-/detect-libc-2.1.2.tgz", @@ -311,6 +389,73 @@ "node": ">=8" } }, + "node_modules/ethers": { + "version": "6.17.0", + "resolved": "https://registry.npmjs.org/ethers/-/ethers-6.17.0.tgz", + "integrity": "sha512-BpyrpIPJ3ydEVow8zGaz1DuPS7YU8DcWxuBnY9a0UA/lvAPwrMr+EPXsfrul628SRaekPNeIM4UFh/91GWZang==", + "funding": [ + { + "type": "individual", + "url": "https://github.com/sponsors/ethers-io/" + }, + { + "type": "individual", + "url": "https://www.buymeacoffee.com/ricmoo" + } + ], + "license": "MIT", + "dependencies": { + "@adraffy/ens-normalize": "1.11.1", + "@noble/curves": "1.2.0", + "@noble/hashes": "1.3.2", + "@types/node": "22.7.5", + "aes-js": "4.0.0-beta.5", + "tslib": "2.7.0", + "ws": "8.21.0" + }, + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/ethers/node_modules/@noble/curves": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/@noble/curves/-/curves-1.2.0.tgz", + "integrity": "sha512-oYclrNgRaM9SsBUBVbb8M6DTV7ZHRTKugureoYEncY5c65HOmRzvSiTE3y5CYaPYJA/GVkrhXEoF0M3Ya9PMnw==", + "license": "MIT", + "dependencies": { + "@noble/hashes": "1.3.2" + }, + "funding": { + "url": "https://paulmillr.com/funding/" + } + }, + "node_modules/ethers/node_modules/@noble/hashes": { + "version": "1.3.2", + "resolved": "https://registry.npmjs.org/@noble/hashes/-/hashes-1.3.2.tgz", + "integrity": "sha512-MVC8EAQp7MvEcm30KWENFjgR+Mkmf+D189XJTkFIlwohU5hcBbn1ZkKq7KVTi2Hme3PMGF390DaL52beVrIihQ==", + "license": "MIT", + "engines": { + "node": ">= 16" + }, + "funding": { + "url": "https://paulmillr.com/funding/" + } + }, + "node_modules/ethers/node_modules/@types/node": { + "version": "22.7.5", + "resolved": "https://registry.npmjs.org/@types/node/-/node-22.7.5.tgz", + "integrity": "sha512-jML7s2NAzMWc//QSJ1a3prpk78cOPchGvXJsC3C6R6PSMoooztvRVQEz89gmBTBY1SPMaqo5teB4uNHPdetShQ==", + "license": "MIT", + "dependencies": { + "undici-types": "~6.19.2" + } + }, + "node_modules/ethers/node_modules/undici-types": { + "version": "6.19.8", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-6.19.8.tgz", + "integrity": "sha512-ve2KP6f/JnbPBFyobGHuerC9g1FYGn/F8n1LWTwNxCEzd6IfqTwUQcNXgEtmmQ6DlRrC1hrSrBnCZPokRrDHjw==", + "license": "MIT" + }, "node_modules/fdir": { "version": "6.5.0", "resolved": "https://registry.npmjs.org/fdir/-/fdir-6.5.0.tgz", @@ -733,6 +878,21 @@ "url": "https://github.com/sponsors/SuperchupuDev" } }, + "node_modules/tslib": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.7.0.tgz", + "integrity": "sha512-gLXCKdN1/j47AiHiOkJN69hJmcbGTHI0ImLmbYLHykhgeN0jVGola9yVjFgzCUklsZQMW55o+dW7IXv3RCXDzA==", + "license": "0BSD" + }, + "node_modules/undici-types": { + "version": "8.3.0", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-8.3.0.tgz", + "integrity": "sha512-j375ScV60dom+YkPFIfTLcOiPxkN/buHz5GobjLhixFuANaNs3C9l4GmrWqejgXWJ7BbJcFYpTEUkS1Ge8bpZQ==", + "dev": true, + "license": "MIT", + "optional": true, + "peer": true + }, "node_modules/vite": { "version": "8.2.0", "resolved": "https://registry.npmjs.org/vite/-/vite-8.2.0.tgz", @@ -810,6 +970,27 @@ "optional": true } } + }, + "node_modules/ws": { + "version": "8.21.0", + "resolved": "https://registry.npmjs.org/ws/-/ws-8.21.0.tgz", + "integrity": "sha512-Vsp28b7DRcimFQvrqu2Wek3z1iYxDCWqHYB8Qsnk/S4RfaCQzPGPyBNuVjJV3cd6UiKtUtp6sNM77gWvzcCH+g==", + "license": "MIT", + "engines": { + "node": ">=10.0.0" + }, + "peerDependencies": { + "bufferutil": "^4.0.1", + "utf-8-validate": ">=5.0.2" + }, + "peerDependenciesMeta": { + "bufferutil": { + "optional": true + }, + "utf-8-validate": { + "optional": true + } + } } } } diff --git a/web/package.json b/web/package.json index 19cad25c..6e9f616c 100644 --- a/web/package.json +++ b/web/package.json @@ -9,9 +9,13 @@ "test": "node --test" }, "dependencies": { + "@msgpack/msgpack": "^3.1.3", "@noble/ciphers": "2.2.0", "@noble/hashes": "2.2.0", - "brotli-dec-wasm": "2.3.2" + "@noble/post-quantum": "^0.6.1", + "brotli-dec-wasm": "2.3.2", + "brotli-wasm": "^3.0.1", + "ethers": "^6.17.0" }, "devDependencies": { "vite": "8.2.0" diff --git a/web/src/main.js b/web/src/main.js index 176d303d..192de1a9 100644 --- a/web/src/main.js +++ b/web/src/main.js @@ -7,6 +7,7 @@ import { } from "./protocol.js"; import { downloadPublicFile } from "./file.js"; import { fetchBrowserManifest } from "./manifest.js"; +import { uploadPublicFile } from "./upload.js"; const elements = { manifestUrl: document.querySelector("#manifest-url"), @@ -23,6 +24,15 @@ const elements = { lookupTarget: document.querySelector("#lookup-target"), randomTarget: document.querySelector("#random-target"), findClosest: document.querySelector("#find-closest"), + uploadInput: document.querySelector("#upload-file-input"), + walletSecret: document.querySelector("#wallet-secret"), + uploadFile: document.querySelector("#upload-file"), + uploadState: document.querySelector("#upload-state"), + uploadResult: document.querySelector("#upload-result"), + uploadResultName: document.querySelector("#upload-result-name"), + uploadResultAddress: document.querySelector("#upload-result-address"), + uploadResultRecords: document.querySelector("#upload-result-records"), + uploadResultPayment: document.querySelector("#upload-result-payment"), fileAddress: document.querySelector("#file-address"), downloadFile: document.querySelector("#download-file"), downloadState: document.querySelector("#download-state"), @@ -151,6 +161,60 @@ elements.findClosest.addEventListener("click", async () => { } }); +elements.uploadFile.addEventListener("click", async () => { + elements.uploadState.classList.remove("connected"); + elements.uploadState.textContent = "Preparing upload…"; + elements.uploadResult.hidden = true; + elements.uploadFile.disabled = true; + let walletSecret = elements.walletSecret.value.trim(); + elements.walletSecret.value = ""; + try { + if (!browserManifest) throw new Error("Load the browser testnet manifest first"); + const file = elements.uploadInput.files?.[0]; + if (!file) throw new Error("Choose a file to upload"); + if (!walletSecret) throw new Error("Enter the paying wallet secret key"); + + log(`Starting paid public upload for ${file.name}`); + const result = await uploadPublicFile( + seedEndpoints(), + browserManifest.payment, + file, + walletSecret, + { + onProgress: (message) => { + elements.uploadState.textContent = message; + log(message); + }, + }, + ); + + browserManifest.files = [ + ...browserManifest.files.filter( + (published) => published.address !== result.file.address, + ), + result.file, + ]; + elements.fileAddress.value = result.file.address; + elements.uploadResultName.textContent = `${result.file.name} · ${result.file.size.toLocaleString()} bytes`; + elements.uploadResultAddress.textContent = result.file.address; + elements.uploadResultRecords.textContent = `${result.records} records · at least ${result.file.replicas} replica${result.file.replicas === 1 ? "" : "s"}`; + elements.uploadResultPayment.textContent = result.transactionHash + ? `${result.transactionHash} · ${result.storageCostAtto} atto tokens` + : "No new payment was required"; + elements.uploadResult.hidden = false; + elements.uploadState.textContent = "Uploaded · ready to download"; + elements.uploadState.classList.add("connected"); + log(`Uploaded and registered ${result.file.name} for immediate download`, result); + } catch (error) { + elements.uploadState.textContent = "Upload failed"; + log(`File upload failed: ${error.message}`); + console.error(error); + } finally { + walletSecret = ""; + elements.uploadFile.disabled = false; + } +}); + elements.downloadFile.addEventListener("click", async () => { elements.downloadState.classList.remove("connected"); elements.downloadState.textContent = "Preparing save…"; diff --git a/web/src/manifest.js b/web/src/manifest.js index 214e4b89..b42edd43 100644 --- a/web/src/manifest.js +++ b/web/src/manifest.js @@ -1,6 +1,6 @@ import { hexToBytes, parseWebTransportMultiaddr } from "./protocol.js"; -export const BROWSER_MANIFEST_VERSION = 3; +export const BROWSER_MANIFEST_VERSION = 4; const MAX_PUBLIC_FILE_BYTES = 64 * 1024 * 1024; const MAX_DATA_MAP_BYTES = 4 * 1024 * 1024; const MAX_FILE_CHUNKS = 1024; @@ -19,6 +19,7 @@ export function parseBrowserManifest(value) { const parsed = parseWebTransportMultiaddr(endpoint); return { multiaddr: parsed.multiaddr }; }); + const payment = parsePaymentNetwork(value.payment); const files = (value.files ?? []).map((file) => { if (!file || typeof file.name !== "string" || file.name.length === 0) { @@ -93,10 +94,36 @@ export function parseBrowserManifest(value) { network_id: value.network_id, created_at: value.created_at, endpoints, + payment, files, }; } +export function parsePaymentNetwork(value) { + if (!value || typeof value !== "object") { + throw new Error("Browser manifest contains no payment network"); + } + let rpcUrl; + try { + rpcUrl = new URL(value.rpc_url); + } catch (error) { + throw new Error("Payment RPC URL is invalid", { cause: error }); + } + if (!["http:", "https:"].includes(rpcUrl.protocol)) { + throw new Error("Payment RPC URL must use HTTP or HTTPS"); + } + if (rpcUrl.username || rpcUrl.password) { + throw new Error("Payment RPC URL must not contain credentials"); + } + hexToBytes(value.payment_token_address ?? "", 20); + hexToBytes(value.payment_vault_address ?? "", 20); + return { + rpc_url: rpcUrl.toString(), + payment_token_address: `0x${value.payment_token_address.replace(/^0x/i, "").toLowerCase()}`, + payment_vault_address: `0x${value.payment_vault_address.replace(/^0x/i, "").toLowerCase()}`, + }; +} + export async function fetchBrowserManifest(url) { const response = await fetch(url, { cache: "no-store" }); if (!response.ok) { diff --git a/web/src/manifest.test.js b/web/src/manifest.test.js index 4ff4d469..4322ca1e 100644 --- a/web/src/manifest.test.js +++ b/web/src/manifest.test.js @@ -4,9 +4,10 @@ import { parseBrowserManifest } from "./manifest.js"; test("browser manifest validates and normalizes endpoints and files", () => { const manifest = parseBrowserManifest({ - version: 3, + version: 4, network_id: "local-test", created_at: "2026-08-03T00:00:00Z", + payment: paymentNetwork(), endpoints: [ { multiaddr: webtransportMultiaddr("AA".repeat(32), 0xbb), @@ -41,18 +42,26 @@ test("browser manifest validates and normalizes endpoints and files", () => { [0, 1, 2], ); assert.equal(manifest.files[0].replicas, 5); + assert.deepEqual(manifest.payment, paymentNetwork()); }); test("browser manifest rejects missing endpoints and malformed multiaddresses", () => { assert.throws( - () => parseBrowserManifest({ version: 3, network_id: "test", endpoints: [] }), + () => + parseBrowserManifest({ + version: 4, + network_id: "test", + payment: paymentNetwork(), + endpoints: [], + }), /no WebTransport endpoints/, ); assert.throws( () => parseBrowserManifest({ - version: 3, + version: 4, network_id: "test", + payment: paymentNetwork(), endpoints: [ { multiaddr: @@ -64,6 +73,32 @@ test("browser manifest rejects missing endpoints and malformed multiaddresses", ); }); +test("browser manifest requires public payment contract configuration", () => { + const endpoint = { multiaddr: webtransportMultiaddr("aa".repeat(32), 0xbb) }; + assert.throws( + () => parseBrowserManifest({ version: 4, network_id: "test", endpoints: [endpoint] }), + /no payment network/, + ); + assert.throws( + () => + parseBrowserManifest({ + version: 4, + network_id: "test", + endpoints: [endpoint], + payment: { ...paymentNetwork(), rpc_url: "file:///tmp/anvil" }, + }), + /HTTP or HTTPS/, + ); +}); + +function paymentNetwork() { + return { + rpc_url: "http://127.0.0.1:8545/", + payment_token_address: `0x${"11".repeat(20)}`, + payment_vault_address: `0x${"22".repeat(20)}`, + }; +} + function webtransportMultiaddr(peerId, certificateByte) { const multihash = Uint8Array.from([ 0x12, diff --git a/web/src/payment.js b/web/src/payment.js new file mode 100644 index 00000000..7f61c032 --- /dev/null +++ b/web/src/payment.js @@ -0,0 +1,306 @@ +import { blake3 } from "@noble/hashes/blake3.js"; +import { ml_dsa65 } from "@noble/post-quantum/ml-dsa.js"; +import { decode } from "@msgpack/msgpack"; +import { + Contract, + JsonRpcProvider, + keccak256, + MaxUint256, + NonceManager, + Wallet, +} from "ethers"; +import { bytesToHex, hexToBytes } from "./protocol.js"; + +const U256_MAX = (1n << 256n) - 1n; +const PAYMENT_MULTIPLIER = 3n; +const PRICE_BASELINE_WEI = 3_906_250_000_000_000n; +const PRICE_COEFFICIENT_WEI = 35_156_250_000_000_000n; +const DIVISOR_SQUARED = 6_000n * 6_000n; +const MAX_COMMITMENT_KEY_COUNT = 1_000_000; +const MAX_COMMITMENT_SIDECAR_BYTES = 8 * 1024; +const DOMAIN_COMMITMENT = new TextEncoder().encode( + "autonomi.ant.replication.storage_commitment.v1", +); +const DOMAIN_COMMITMENT_HASH = new TextEncoder().encode( + "autonomi.ant.replication.commitment_hash.v1", +); + +const TOKEN_ABI = [ + "function allowance(address owner, address spender) view returns (uint256)", + "function approve(address spender, uint256 amount) returns (bool)", +]; +const VAULT_ABI = [ + "function payForQuotes((address rewardsAddress,uint256 amount,bytes32 quoteHash)[] payments)", +]; + +function concatBytes(...parts) { + const size = parts.reduce((total, part) => total + part.length, 0); + const output = new Uint8Array(size); + let offset = 0; + for (const part of parts) { + output.set(part, offset); + offset += part.length; + } + return output; +} + +function unsignedLittleEndian(value, length) { + let remaining = BigInt(value); + if (remaining < 0n || remaining >= 1n << BigInt(length * 8)) { + throw new Error(`Unsigned integer does not fit ${length} bytes`); + } + const result = new Uint8Array(length); + for (let index = 0; index < length; index += 1) { + result[index] = Number(remaining & 0xffn); + remaining >>= 8n; + } + return result; +} + +function postcardVarint(value) { + let remaining = BigInt(value); + const result = []; + do { + let byte = Number(remaining & 0x7fn); + remaining >>= 7n; + if (remaining > 0n) byte |= 0x80; + result.push(byte); + } while (remaining > 0n); + return Uint8Array.from(result); +} + +function canonicalQuoteBytes(quote) { + const content = hexToBytes(quote.content, 32); + const rewardsAddress = hexToBytes(quote.rewards_address, 20); + const price = parseAmount(quote.price, "quote price"); + const timestamp = parseAmount(String(quote.timestamp_secs), "quote timestamp"); + const count = quote.committed_key_count; + if (!Number.isSafeInteger(count) || count < 0 || count > MAX_COMMITMENT_KEY_COUNT) { + throw new Error(`Invalid committed key count ${count}`); + } + let pin = Uint8Array.of(0); + if (quote.commitment_pin !== null && quote.commitment_pin !== undefined) { + pin = concatBytes(Uint8Array.of(1), hexToBytes(quote.commitment_pin, 32)); + } + return concatBytes( + content, + unsignedLittleEndian(timestamp, 8), + unsignedLittleEndian(price, 32), + rewardsAddress, + unsignedLittleEndian(BigInt(count), 4), + pin, + ); +} + +export function paymentQuoteHash(signedBytes, publicKey, signature) { + // This is the EVM-facing PaymentQuote hash settled by the payment vault. + // Native evmlib deliberately uses Keccak-256 here, while ANT identities, + // chunk addresses, and commitment pins use BLAKE3. + return keccak256(concatBytes(signedBytes, publicKey, signature)).slice(2); +} + +function parseAmount(value, label) { + if (typeof value !== "string" || !/^(0|[1-9][0-9]*)$/.test(value)) { + throw new Error(`Invalid ${label}`); + } + const amount = BigInt(value); + if (amount > U256_MAX) throw new Error(`${label} exceeds uint256`); + return amount; +} + +function calculatePrice(keyCount) { + const count = BigInt(keyCount); + return PRICE_BASELINE_WEI + (count * count * PRICE_COEFFICIENT_WEI) / DIVISOR_SQUARED; +} + +function equalBytes(left, right) { + if (left.length !== right.length) return false; + return left.every((byte, index) => byte === right[index]); +} + +function verifyEncodedCommitment(commitment, normalized) { + const encoded = hexToBytes(commitment.encoded); + if (encoded.length > MAX_COMMITMENT_SIDECAR_BYTES) { + throw new Error("Storage commitment sidecar exceeds the protocol limit"); + } + let fields; + try { + fields = decode(encoded); + } catch (error) { + throw new Error(`Storage commitment sidecar is not valid MessagePack: ${error.message}`, { + cause: error, + }); + } + if (!Array.isArray(fields) || fields.length !== 5) { + throw new Error("Storage commitment sidecar has an invalid native shape"); + } + const [root, keyCount, peerId, publicKey, signature] = fields; + if ( + keyCount !== normalized.keyCount || + !equalBytes(fixedCommitmentBytes(root, 32), normalized.root) || + !equalBytes(fixedCommitmentBytes(peerId, 32), normalized.peerId) || + !equalBytes(fixedCommitmentBytes(publicKey), normalized.publicKey) || + !equalBytes(fixedCommitmentBytes(signature), normalized.signature) + ) { + throw new Error("Storage commitment sidecar differs from the verified commitment"); + } +} + +function fixedCommitmentBytes(value, length) { + const bytes = value instanceof Uint8Array ? value : Uint8Array.from(value ?? []); + if (length !== undefined && bytes.length !== length) { + throw new Error(`Storage commitment field must contain ${length} bytes`); + } + return bytes; +} + +function verifyCommitment(commitment, quote) { + if (!commitment || typeof commitment !== "object") { + throw new Error("Bound quote omitted its storage commitment"); + } + const root = hexToBytes(commitment.root, 32); + const peerId = hexToBytes(commitment.sender_peer_id, 32); + const publicKey = hexToBytes(commitment.sender_public_key); + const signature = hexToBytes(commitment.signature); + const keyCount = commitment.key_count; + if ( + publicKey.length !== ml_dsa65.lengths.publicKey || + signature.length !== ml_dsa65.lengths.signature + ) { + throw new Error("Storage commitment has invalid ML-DSA-65 field lengths"); + } + if (!Number.isSafeInteger(keyCount) || keyCount !== quote.committed_key_count) { + throw new Error("Storage commitment key count does not match quote"); + } + if (bytesToHex(blake3(publicKey)) !== quote.peer_id.toLowerCase()) { + throw new Error("Storage commitment public key is not bound to quote peer"); + } + if (bytesToHex(peerId) !== quote.peer_id.toLowerCase()) { + throw new Error("Storage commitment belongs to a different peer"); + } + verifyEncodedCommitment(commitment, { + root, + keyCount, + peerId, + publicKey, + signature, + }); + const signedPayload = concatBytes( + root, + unsignedLittleEndian(BigInt(keyCount), 4), + peerId, + unsignedLittleEndian(BigInt(publicKey.length), 4), + publicKey, + ); + if ( + !ml_dsa65.verify(signature, signedPayload, publicKey, { + context: DOMAIN_COMMITMENT, + }) + ) { + throw new Error("Storage commitment has an invalid ML-DSA-65 signature"); + } + + const postcard = concatBytes( + root, + postcardVarint(keyCount), + peerId, + postcardVarint(publicKey.length), + publicKey, + postcardVarint(signature.length), + signature, + ); + const pin = bytesToHex(blake3(concatBytes(DOMAIN_COMMITMENT_HASH, postcard))); + if (pin !== quote.commitment_pin.toLowerCase()) { + throw new Error("Storage commitment does not resolve the quote pin"); + } + return true; +} + +export function verifyStorageQuote(quote, expectedAddress, expectedPeerId) { + if (!quote || typeof quote !== "object") throw new Error("Node returned no quote"); + hexToBytes(expectedAddress, 32); + hexToBytes(expectedPeerId, 32); + if (quote.content?.toLowerCase() !== expectedAddress.toLowerCase()) { + throw new Error("Storage quote is for a different chunk"); + } + if (quote.peer_id?.toLowerCase() !== expectedPeerId.toLowerCase()) { + throw new Error("Storage quote belongs to a different WebTransport peer"); + } + const publicKey = hexToBytes(quote.public_key); + const signature = hexToBytes(quote.signature); + if (publicKey.length !== ml_dsa65.lengths.publicKey) { + throw new Error(`Storage quote has a ${publicKey.length}-byte public key`); + } + if (signature.length !== ml_dsa65.lengths.signature) { + throw new Error(`Storage quote has a ${signature.length}-byte signature`); + } + if (bytesToHex(blake3(publicKey)) !== quote.peer_id.toLowerCase()) { + throw new Error("Storage quote public key is not bound to its peer ID"); + } + const signedBytes = canonicalQuoteBytes(quote); + if (!ml_dsa65.verify(signature, signedBytes, publicKey)) { + throw new Error("Storage quote has an invalid ML-DSA-65 signature"); + } + const quoteHash = paymentQuoteHash(signedBytes, publicKey, signature); + if (quoteHash !== quote.quote_hash?.toLowerCase()) { + throw new Error("Storage quote hash does not match its signed fields"); + } + const price = parseAmount(quote.price, "quote price"); + if (price !== calculatePrice(quote.committed_key_count)) { + throw new Error("Storage quote price is not bound to its committed key count"); + } + if (quote.committed_key_count === 0) { + if (quote.commitment_pin !== null || quote.commitment !== null) { + throw new Error("Baseline storage quote has an incoherent commitment"); + } + } else { + if (!quote.commitment_pin) throw new Error("Bound storage quote omitted its pin"); + verifyCommitment(quote.commitment, quote); + } + hexToBytes(quote.rewards_address, 20); + return { + quote, + quoteHash, + rewardsAddress: `0x${quote.rewards_address.replace(/^0x/i, "")}`, + amount: price * PAYMENT_MULTIPLIER, + }; +} + +export async function payForStorageQuotes( + paymentNetwork, + verifiedQuotes, + walletSecret, + { onProgress = () => {} } = {}, +) { + if (!Array.isArray(verifiedQuotes) || verifiedQuotes.length === 0) { + return { transactionHash: undefined, walletAddress: undefined, totalAmount: 0n }; + } + const provider = new JsonRpcProvider(paymentNetwork.rpc_url); + let wallet; + try { + wallet = new Wallet(walletSecret, provider); + } catch (error) { + throw new Error("Wallet secret key is invalid", { cause: error }); + } + const signer = new NonceManager(wallet); + const totalAmount = verifiedQuotes.reduce((total, quote) => total + quote.amount, 0n); + const token = new Contract(paymentNetwork.payment_token_address, TOKEN_ABI, signer); + const vault = new Contract(paymentNetwork.payment_vault_address, VAULT_ABI, signer); + const allowance = await token.allowance(wallet.address, paymentNetwork.payment_vault_address); + if (allowance < totalAmount) { + onProgress(`Approving the payment vault from wallet ${wallet.address}`); + const approval = await token.approve(paymentNetwork.payment_vault_address, MaxUint256); + await approval.wait(); + } + const payments = verifiedQuotes.map((quote) => ({ + rewardsAddress: quote.rewardsAddress, + amount: quote.amount, + quoteHash: `0x${quote.quoteHash}`, + })); + onProgress(`Submitting one payment for ${payments.length} storage quote(s)`); + const transaction = await vault.payForQuotes(payments); + const receipt = await transaction.wait(); + if (!receipt || receipt.status !== 1) throw new Error("Storage payment transaction reverted"); + onProgress(`Payment confirmed in ${transaction.hash}`); + return { transactionHash: transaction.hash, walletAddress: wallet.address, totalAmount }; +} diff --git a/web/src/payment.test.js b/web/src/payment.test.js new file mode 100644 index 00000000..bdb7d1d7 --- /dev/null +++ b/web/src/payment.test.js @@ -0,0 +1,235 @@ +import assert from "node:assert/strict"; +import test from "node:test"; +import { blake3 } from "@noble/hashes/blake3.js"; +import { ml_dsa65 } from "@noble/post-quantum/ml-dsa.js"; +import { encode } from "@msgpack/msgpack"; +import { keccak256 } from "ethers"; +import { bytesToHex } from "./protocol.js"; +import { paymentQuoteHash, verifyStorageQuote } from "./payment.js"; + +const BASELINE_PRICE = 3_906_250_000_000_000n; +const PRICE_COEFFICIENT = 35_156_250_000_000_000n; +const COMMITMENT_CONTEXT = new TextEncoder().encode( + "autonomi.ant.replication.storage_commitment.v1", +); +const COMMITMENT_HASH_DOMAIN = new TextEncoder().encode( + "autonomi.ant.replication.commitment_hash.v1", +); + +function concatBytes(...parts) { + const output = new Uint8Array(parts.reduce((total, part) => total + part.length, 0)); + let offset = 0; + for (const part of parts) { + output.set(part, offset); + offset += part.length; + } + return output; +} + +function littleEndian(value, length) { + let remaining = BigInt(value); + const result = new Uint8Array(length); + for (let index = 0; index < length; index += 1) { + result[index] = Number(remaining & 0xffn); + remaining >>= 8n; + } + return result; +} + +function postcardVarint(value) { + let remaining = BigInt(value); + const bytes = []; + do { + let byte = Number(remaining & 0x7fn); + remaining >>= 7n; + if (remaining > 0n) byte |= 0x80; + bytes.push(byte); + } while (remaining > 0n); + return Uint8Array.from(bytes); +} + +function signedBaselineQuote() { + const content = Uint8Array.from({ length: 32 }, (_, index) => index); + const rewards = new Uint8Array(20).fill(0x44); + const timestamp = 1_775_000_000n; + const { publicKey, secretKey } = ml_dsa65.keygen(new Uint8Array(32).fill(0x17)); + const payload = concatBytes( + content, + littleEndian(timestamp, 8), + littleEndian(BASELINE_PRICE, 32), + rewards, + littleEndian(0n, 4), + Uint8Array.of(0), + ); + const signature = ml_dsa65.sign(payload, secretKey); + const quoteHash = keccak256(concatBytes(payload, publicKey, signature)).slice(2); + const peerId = bytesToHex(blake3(publicKey)); + return { + peerId, + address: bytesToHex(content), + quote: { + peer_id: peerId, + content: bytesToHex(content), + timestamp_secs: Number(timestamp), + price: BASELINE_PRICE.toString(), + rewards_address: bytesToHex(rewards), + public_key: bytesToHex(publicKey), + signature: bytesToHex(signature), + committed_key_count: 0, + commitment_pin: null, + quote_hash: quoteHash, + commitment: null, + }, + }; +} + +function signedBoundQuote() { + const content = new Uint8Array(32).fill(0x31); + const rewards = new Uint8Array(20).fill(0x42); + const root = new Uint8Array(32).fill(0x53); + const keyCount = 23; + const timestamp = 1_775_000_001n; + const { publicKey, secretKey } = ml_dsa65.keygen(new Uint8Array(32).fill(0x29)); + const peerId = blake3(publicKey); + const commitmentPayload = concatBytes( + root, + littleEndian(keyCount, 4), + peerId, + littleEndian(publicKey.length, 4), + publicKey, + ); + const commitmentSignature = ml_dsa65.sign(commitmentPayload, secretKey, { + context: COMMITMENT_CONTEXT, + }); + const postcard = concatBytes( + root, + postcardVarint(keyCount), + peerId, + postcardVarint(publicKey.length), + publicKey, + postcardVarint(commitmentSignature.length), + commitmentSignature, + ); + const pin = blake3(concatBytes(COMMITMENT_HASH_DOMAIN, postcard)); + const price = + BASELINE_PRICE + + (BigInt(keyCount) * BigInt(keyCount) * PRICE_COEFFICIENT) / (6_000n * 6_000n); + const quotePayload = concatBytes( + content, + littleEndian(timestamp, 8), + littleEndian(price, 32), + rewards, + littleEndian(keyCount, 4), + Uint8Array.of(1), + pin, + ); + const quoteSignature = ml_dsa65.sign(quotePayload, secretKey); + const quoteHash = keccak256( + concatBytes(quotePayload, publicKey, quoteSignature), + ).slice(2); + const encodedCommitment = encode([ + Array.from(root), + keyCount, + Array.from(peerId), + Array.from(publicKey), + Array.from(commitmentSignature), + ]); + const commitment = { + encoded: bytesToHex(encodedCommitment), + root: bytesToHex(root), + key_count: keyCount, + sender_peer_id: bytesToHex(peerId), + sender_public_key: bytesToHex(publicKey), + signature: bytesToHex(commitmentSignature), + }; + return { + address: bytesToHex(content), + peerId: bytesToHex(peerId), + quote: { + peer_id: bytesToHex(peerId), + content: bytesToHex(content), + timestamp_secs: Number(timestamp), + price: price.toString(), + rewards_address: bytesToHex(rewards), + public_key: bytesToHex(publicKey), + signature: bytesToHex(quoteSignature), + committed_key_count: keyCount, + commitment_pin: bytesToHex(pin), + quote_hash: quoteHash, + commitment, + }, + }; +} + +test("accepts a correctly bound and signed native-shaped storage quote", () => { + const { quote, address, peerId } = signedBaselineQuote(); + const verified = verifyStorageQuote(quote, address, peerId); + assert.equal(verified.quoteHash, quote.quote_hash); + assert.equal(verified.amount, BASELINE_PRICE * 3n); + assert.equal(verified.rewardsAddress, `0x${quote.rewards_address}`); +}); + +test("uses evmlib's Keccak-256 PaymentQuote hash", () => { + // Fixed native hash-vector input split as bytes-for-signing, public key, + // and signature. PaymentQuote::hash() concatenates these exact byte slices. + assert.equal( + paymentQuoteHash(Uint8Array.of(0, 1), Uint8Array.of(2), Uint8Array.of(3)), + "d98f2e8134922f73748703c8e7084d42f13d2fa1439936ef5a3abcf5646fe83f", + ); +}); + +test("rejects quote field tampering before any payment", () => { + const fixture = signedBaselineQuote(); + assert.throws( + () => + verifyStorageQuote( + { ...fixture.quote, price: (BASELINE_PRICE + 1n).toString() }, + fixture.address, + fixture.peerId, + ), + /invalid ML-DSA-65 signature|price/, + ); + assert.throws( + () => verifyStorageQuote(fixture.quote, fixture.address, "ff".repeat(32)), + /different WebTransport peer/, + ); + const signature = Uint8Array.from(Buffer.from(fixture.quote.signature, "hex")); + signature[100] ^= 1; + assert.throws( + () => + verifyStorageQuote( + { ...fixture.quote, signature: bytesToHex(signature) }, + fixture.address, + fixture.peerId, + ), + /invalid ML-DSA-65 signature/, + ); +}); + +test("verifies a bound commitment and the exact native sidecar before payment", () => { + const fixture = signedBoundQuote(); + assert.doesNotThrow(() => + verifyStorageQuote(fixture.quote, fixture.address, fixture.peerId), + ); + + const decodedShape = [ + Array(32).fill(0x99), + fixture.quote.committed_key_count, + Array.from(Buffer.from(fixture.quote.commitment.sender_peer_id, "hex")), + Array.from(Buffer.from(fixture.quote.commitment.sender_public_key, "hex")), + Array.from(Buffer.from(fixture.quote.commitment.signature, "hex")), + ]; + const mismatchedSidecar = { + ...fixture.quote.commitment, + encoded: bytesToHex(encode(decodedShape)), + }; + assert.throws( + () => + verifyStorageQuote( + { ...fixture.quote, commitment: mismatchedSidecar }, + fixture.address, + fixture.peerId, + ), + /sidecar differs/, + ); +}); diff --git a/web/src/protocol.js b/web/src/protocol.js index d6ee7988..b439c394 100644 --- a/web/src/protocol.js +++ b/web/src/protocol.js @@ -1,8 +1,8 @@ import { blake3 } from "@noble/hashes/blake3.js"; import { bytesToHex as nobleBytesToHex } from "@noble/hashes/utils.js"; -export const PROTOCOL_VERSION = 2; -export const PROTOCOL_NAME = "autonomi.web.poc.v2"; +export const PROTOCOL_VERSION = 3; +export const PROTOCOL_NAME = "autonomi.web.poc.v3"; export const WEBTRANSPORT_PATH = "/autonomi/webtransport/v1"; export const MAX_CHUNK_SIZE = 4 * 1024 * 1024; export const MAX_RESPONSE_HEADER_BYTES = 64 * 1024; @@ -289,23 +289,33 @@ export class BrowserNodeClient { this.transport = transport; } - async request(type, fields = {}) { + async request(type, fields = {}, content = new Uint8Array()) { await this.connect(); + if (!(content instanceof Uint8Array) || content.length > MAX_CHUNK_SIZE) { + throw new Error(`Request content must be at most ${MAX_CHUNK_SIZE} bytes`); + } const requestId = nextRequestId; nextRequestId += 1; const stream = await this.transport.createBidirectionalStream(); const writer = stream.writable.getWriter(); try { - await writer.write( - encoder.encode( - JSON.stringify({ - version: PROTOCOL_VERSION, - request_id: requestId, - type, - ...fields, - }), - ), + const header = encoder.encode( + JSON.stringify({ + version: PROTOCOL_VERSION, + request_id: requestId, + content_length: content.length, + type, + ...fields, + }), ); + if (header.length === 0 || header.length > MAX_RESPONSE_HEADER_BYTES) { + throw new Error(`Request header is ${header.length} bytes`); + } + const prefix = new Uint8Array(4); + new DataView(prefix.buffer).setUint32(0, header.length, false); + await writer.write(prefix); + await writer.write(header); + if (content.length > 0) await writer.write(content); await writer.close(); } finally { writer.releaseLock(); @@ -394,6 +404,39 @@ export class BrowserNodeClient { return { content: response.content, hash }; } + async quoteChunk(address, size) { + hexToBytes(address, 32); + if (!Number.isSafeInteger(size) || size < 0 || size > MAX_CHUNK_SIZE) { + throw new Error(`Invalid chunk size ${size}`); + } + const { header } = await this.request("quote_chunk", { address, size }); + if (header.type !== "storage_quote") { + throw new Error("Expected a STORAGE_QUOTE response"); + } + if (header.address.toLowerCase() !== address.toLowerCase()) { + throw new Error("Node returned a quote for a different chunk address"); + } + return { quote: header.quote, alreadyStored: Boolean(header.already_stored) }; + } + + async putChunk(address, content, quote, transactionHash) { + hexToBytes(address, 32); + hexToBytes(transactionHash, 32); + verifyChunk(address, content); + const { header } = await this.request( + "put_chunk", + { address, quote, transaction_hash: transactionHash }, + content, + ); + if (header.type !== "chunk_stored") { + throw new Error("Expected a CHUNK_STORED response"); + } + if (header.address.toLowerCase() !== address.toLowerCase()) { + throw new Error("Node stored a different chunk address"); + } + return { address: header.address, alreadyStored: Boolean(header.already_stored) }; + } + close() { this.transport?.close({ closeCode: 0, reason: "client closed" }); this.transport = undefined; diff --git a/web/src/protocol.test.js b/web/src/protocol.test.js index 7dbfc4e2..833ca2a8 100644 --- a/web/src/protocol.test.js +++ b/web/src/protocol.test.js @@ -2,6 +2,7 @@ import assert from "node:assert/strict"; import test from "node:test"; import { blake3 } from "@noble/hashes/blake3.js"; import { + BrowserNodeClient, bytesToHex, getChunkFromClosest, hexToBytes, @@ -27,7 +28,7 @@ test("XOR distance is an unsigned 256-bit ordering value", () => { test("response framing preserves a raw binary body", () => { const header = new TextEncoder().encode( JSON.stringify({ - version: 2, + version: 3, request_id: 9, status: "ok", content_length: 3, @@ -143,6 +144,44 @@ test("browser lookup discovers a direct node and downloads a verified chunk", as ]); }); +test("paid PUT frames the encrypted chunk as a binary request body", async (t) => { + const content = new TextEncoder().encode("encrypted record"); + const address = bytesToHex(blake3(content)); + const node = endpoint("storage.test", "33".repeat(32), 0x44); + const transactionHash = "55".repeat(32); + const quote = { quote_hash: "66".repeat(32) }; + let observed; + const routes = new Map([ + [ + node.url, + (request) => { + if (request.type === "put_chunk") { + observed = request; + return response(request, { + type: "chunk_stored", + address, + already_stored: false, + }); + } + throw new Error(`Unexpected request ${request.type}`); + }, + ], + ]); + const previousWebTransport = globalThis.WebTransport; + globalThis.WebTransport = mockWebTransport(routes); + t.after(() => { + globalThis.WebTransport = previousWebTransport; + }); + + const client = new BrowserNodeClient(node.multiaddr); + const stored = await client.putChunk(address, content, quote, transactionHash); + assert.deepEqual(stored, { address, alreadyStored: false }); + assert.equal(observed.content_length, content.length); + assert.deepEqual(observed.content, content); + assert.deepEqual(observed.quote, quote); + assert.equal(observed.transaction_hash, transactionHash); +}); + function browserNode(endpoint) { return { peer_id: endpoint.peer_id, @@ -157,7 +196,7 @@ function browserNode(endpoint) { function helloResponse(request, endpoint) { return response(request, { type: "hello", - protocol: "autonomi.web.poc.v2", + protocol: "autonomi.web.poc.v3", peer_id: endpoint.peer_id, max_chunk_size: 4 * 1024 * 1024, endpoint: { @@ -170,7 +209,7 @@ function helloResponse(request, endpoint) { function response(request, fields, content = new Uint8Array()) { return { header: { - version: 2, + version: 3, request_id: request.request_id, status: "ok", content_length: content.length, @@ -240,7 +279,20 @@ function mockWebTransport(routes) { encoded.set(chunk, offset); offset += chunk.length; } - const request = JSON.parse(new TextDecoder().decode(encoded)); + if (encoded.length < 4) throw new Error("Request omitted its frame prefix"); + const headerLength = new DataView( + encoded.buffer, + encoded.byteOffset, + encoded.byteLength, + ).getUint32(0, false); + const contentOffset = 4 + headerLength; + const request = JSON.parse( + new TextDecoder().decode(encoded.subarray(4, contentOffset)), + ); + request.content = encoded.slice(contentOffset); + if (request.content.length !== request.content_length) { + throw new Error("Request content length mismatch"); + } const handler = routes.get(this.url); responseController.enqueue(encodeResponse(handler(request))); responseController.close(); diff --git a/web/src/style.css b/web/src/style.css index d65ff0ae..301225fb 100644 --- a/web/src/style.css +++ b/web/src/style.css @@ -48,6 +48,14 @@ h2 { font-size: 0.88rem; } +.secret-note { + padding: 0.75rem 0.85rem; + border-left: 3px solid #a76b16; + color: #71501e; + background: #fff8e9; + font-size: 0.84rem; +} + .eyebrow { color: #25764a; font-size: 0.78rem; @@ -209,6 +217,11 @@ a { color: #a6b5aa; } + .secret-note { + color: #efd19e; + background: #2c2519; + } + .file-card { background: #111713; } diff --git a/web/src/upload.js b/web/src/upload.js new file mode 100644 index 00000000..5cc234b6 --- /dev/null +++ b/web/src/upload.js @@ -0,0 +1,326 @@ +import { chacha20poly1305 } from "@noble/ciphers/chacha.js"; +import { blake3 } from "@noble/hashes/blake3.js"; +import { decode, encode } from "@msgpack/msgpack"; +import { deriveChunkMaterial } from "./file.js"; +import { + BrowserNodeClient, + bytesToHex, + hexToBytes, + iterativeFindClosest, +} from "./protocol.js"; +import { payForStorageQuotes, verifyStorageQuote } from "./payment.js"; + +export const MAX_BROWSER_UPLOAD_BYTES = 64 * 1024 * 1024; +const SELF_ENCRYPTION_MAX_CHUNK_SIZE = 4_190_208; +const MAX_STORE_TARGETS = 7; + +function numberOfChunks(fileSize) { + if (fileSize < 3) return 0; + if (fileSize < 3 * SELF_ENCRYPTION_MAX_CHUNK_SIZE) return 3; + return Math.ceil(fileSize / SELF_ENCRYPTION_MAX_CHUNK_SIZE); +} + +function chunkSize(fileSize, index) { + if (fileSize < 3 * SELF_ENCRYPTION_MAX_CHUNK_SIZE) { + return index < 2 ? Math.floor(fileSize / 3) : fileSize - 2 * Math.floor(fileSize / 3); + } + const count = numberOfChunks(fileSize); + const remainder = fileSize % SELF_ENCRYPTION_MAX_CHUNK_SIZE; + if (index < count - 2 || remainder === 0) return SELF_ENCRYPTION_MAX_CHUNK_SIZE; + return index === count - 2 ? SELF_ENCRYPTION_MAX_CHUNK_SIZE : remainder; +} + +function chunkStart(fileSize, index) { + const count = numberOfChunks(fileSize); + if (index === count - 1) { + return chunkSize(fileSize, 0) * (index - 1) + chunkSize(fileSize, index - 1); + } + return chunkSize(fileSize, 0) * index; +} + +async function defaultCompress(input) { + const { default: brotliPromise } = await import("brotli-wasm"); + const brotli = await brotliPromise; + return brotli.compress(input, { quality: 6 }); +} + +function xorPad(content, pad) { + const output = new Uint8Array(content.length); + for (let index = 0; index < content.length; index += 1) { + output[index] = content[index] ^ pad[index % pad.length]; + } + return output; +} + +export function encodePublicDataMap(chunks) { + const compact = [ + 1, + chunks.map((chunk) => [ + chunk.index, + Array.from(hexToBytes(chunk.dst_hash, 32)), + Array.from(hexToBytes(chunk.src_hash, 32)), + chunk.src_size, + ]), + null, + ]; + return encode(compact, { sortKeys: false }); +} + +function fixedBytes(value, length, label) { + const bytes = value instanceof Uint8Array ? value : Uint8Array.from(value ?? []); + if (bytes.length !== length) throw new Error(`${label} must contain ${length} bytes`); + return bytes; +} + +export function decodePublicDataMap(content) { + let dataMap; + try { + dataMap = decode(content); + } catch (error) { + throw new Error(`Public DataMap is not valid MessagePack: ${error.message}`, { + cause: error, + }); + } + if (!Array.isArray(dataMap) || dataMap.length !== 3 || dataMap[0] !== 1) { + throw new Error("Public DataMap does not use self_encryption version 1"); + } + if (dataMap[2] !== null) { + throw new Error("Nested DataMaps are not yet supported by the browser uploader"); + } + if (!Array.isArray(dataMap[1]) || dataMap[1].length < 3) { + throw new Error("Public DataMap has fewer than three chunks"); + } + const chunks = dataMap[1] + .map((chunk) => { + if (!Array.isArray(chunk) || chunk.length !== 4) { + throw new Error("Public DataMap contains an invalid chunk descriptor"); + } + const [index, dstHash, srcHash, srcSize] = chunk; + if (!Number.isSafeInteger(index) || index < 0) { + throw new Error(`Invalid DataMap chunk index ${index}`); + } + if (!Number.isSafeInteger(srcSize) || srcSize < 1) { + throw new Error(`Invalid DataMap plaintext chunk size ${srcSize}`); + } + return { + index, + dst_hash: bytesToHex(fixedBytes(dstHash, 32, "DataMap destination hash")), + src_hash: bytesToHex(fixedBytes(srcHash, 32, "DataMap source hash")), + src_size: srcSize, + }; + }) + .sort((left, right) => left.index - right.index); + chunks.forEach((chunk, index) => { + if (chunk.index !== index) throw new Error("DataMap chunk indices are not contiguous"); + }); + return chunks; +} + +export async function encryptPublicFile( + content, + { name = "upload.bin", contentType = "application/octet-stream", compress = defaultCompress } = {}, +) { + if (!(content instanceof Uint8Array)) throw new Error("Upload content must be bytes"); + if (content.length < 3) throw new Error("Self-encryption requires a file of at least 3 bytes"); + if (content.length > MAX_BROWSER_UPLOAD_BYTES) { + throw new Error(`Browser uploads are limited to ${MAX_BROWSER_UPLOAD_BYTES} bytes`); + } + const count = numberOfChunks(content.length); + const plaintextChunks = Array.from({ length: count }, (_, index) => { + const start = chunkStart(content.length, index); + return content.slice(start, start + chunkSize(content.length, index)); + }); + const sourceHashes = plaintextChunks.map((chunk) => blake3(chunk)); + const encrypted = await Promise.all( + plaintextChunks.map(async (plaintext, index) => { + const descriptor = { index }; + const { pad, key, nonce } = deriveChunkMaterial(descriptor, sourceHashes, 0); + const compressed = await compress(plaintext); + const ciphertext = chacha20poly1305(key, nonce).encrypt(compressed); + const bytes = xorPad(ciphertext, pad); + return { + content: bytes, + info: { + index, + dst_hash: bytesToHex(blake3(bytes)), + src_hash: bytesToHex(sourceHashes[index]), + src_size: plaintext.length, + }, + }; + }), + ); + const chunks = encrypted.map(({ info }) => info); + const dataMap = encodePublicDataMap(chunks); + const address = bytesToHex(blake3(dataMap)); + const records = encrypted.map(({ content: bytes, info }) => ({ + address: info.dst_hash, + content: bytes, + })); + records.push({ address, content: dataMap }); + return { + descriptor: { + name, + address, + size: content.length, + content_type: contentType || "application/octet-stream", + blake3: bytesToHex(blake3(content)), + data_map_size: dataMap.length, + chunks, + replicas: 0, + }, + records, + }; +} + +function closeClients(clients) { + for (const client of clients.values()) client.close(); +} + +function assertUploadNode(hello, paymentNetwork) { + if ( + !Array.isArray(hello.capabilities) || + !hello.capabilities.includes("quote_chunk") || + !hello.capabilities.includes("put_chunk") + ) { + throw new Error("Node does not advertise paid browser uploads"); + } + const advertised = hello.payment; + if ( + !advertised || + new URL(advertised.rpc_url).toString() !== new URL(paymentNetwork.rpc_url).toString() || + advertised.payment_token_address?.toLowerCase() !== + paymentNetwork.payment_token_address.toLowerCase() || + advertised.payment_vault_address?.toLowerCase() !== + paymentNetwork.payment_vault_address.toLowerCase() + ) { + throw new Error("Node advertises a different payment network than the manifest"); + } +} + +async function prepareRecord(seedEndpoints, paymentNetwork, record, onProgress) { + onProgress(`Finding closest nodes for ${record.address}`); + const lookup = await iterativeFindClosest(seedEndpoints, record.address, { onProgress }); + const endpoints = lookup.nodes + .filter((node) => node.webtransport) + .slice(0, MAX_STORE_TARGETS) + .map((node) => ({ peerId: node.peer_id, endpoint: node.webtransport })); + try { + const failures = []; + for (const target of endpoints) { + const client = new BrowserNodeClient(target.endpoint); + try { + assertUploadNode(await client.hello(), paymentNetwork); + const response = await client.quoteChunk(record.address, record.content.length); + const verified = verifyStorageQuote( + response.quote, + record.address, + target.peerId, + ); + if (response.alreadyStored) { + onProgress(`Chunk ${record.address} is already stored; skipping payment`); + return { record, alreadyStored: true, targets: endpoints }; + } + onProgress(`Verified storage quote ${verified.quoteHash} from ${target.peerId}`); + return { + record, + alreadyStored: false, + targets: [target, ...endpoints.filter((candidate) => candidate !== target)], + verified, + }; + } catch (error) { + failures.push(`${target.peerId}: ${error.message}`); + } finally { + client.close(); + } + } + throw new Error(`No closest node supplied a valid quote (${failures.join("; ")})`); + } finally { + closeClients(lookup.clients); + } +} + +async function storePrepared(prepared, paymentNetwork, transactionHash, onProgress) { + if (prepared.alreadyStored) return 1; + const attempts = await Promise.allSettled( + prepared.targets.map(async (target) => { + const client = new BrowserNodeClient(target.endpoint); + try { + assertUploadNode(await client.hello(), paymentNetwork); + const result = await client.putChunk( + prepared.record.address, + prepared.record.content, + prepared.verified.quote, + transactionHash, + ); + onProgress( + `${result.alreadyStored ? "Confirmed" : "Stored"} ${prepared.record.address} on ${target.peerId}`, + ); + return result; + } finally { + client.close(); + } + }), + ); + const stored = attempts.filter((attempt) => attempt.status === "fulfilled").length; + if (stored === 0) { + const failures = attempts + .filter((attempt) => attempt.status === "rejected") + .map((attempt) => attempt.reason?.message ?? String(attempt.reason)); + throw new Error(`Paid chunk was rejected by every closest node: ${failures.join("; ")}`); + } + return stored; +} + +export async function uploadPublicFile( + seedEndpoints, + paymentNetwork, + file, + walletSecret, + { onProgress = () => {}, compress = defaultCompress } = {}, +) { + const content = new Uint8Array(await file.arrayBuffer()); + onProgress(`Self-encrypting ${file.name} (${content.length.toLocaleString()} bytes)`); + const encrypted = await encryptPublicFile(content, { + name: file.name, + contentType: file.type, + compress, + }); + const prepared = []; + for (let index = 0; index < encrypted.records.length; index += 1) { + onProgress(`Preparing record ${index + 1}/${encrypted.records.length}`); + prepared.push( + await prepareRecord(seedEndpoints, paymentNetwork, encrypted.records[index], onProgress), + ); + } + const payable = prepared.filter((record) => !record.alreadyStored); + let transactionHash; + let totalAmount = 0n; + if (payable.length > 0) { + const payment = await payForStorageQuotes( + paymentNetwork, + payable.map((record) => record.verified), + walletSecret, + { onProgress }, + ); + transactionHash = payment.transactionHash; + totalAmount = payment.totalAmount; + } + let replicas = Number.POSITIVE_INFINITY; + for (let index = 0; index < prepared.length; index += 1) { + onProgress(`Storing record ${index + 1}/${prepared.length}`); + const stored = await storePrepared( + prepared[index], + paymentNetwork, + transactionHash, + onProgress, + ); + replicas = Math.min(replicas, stored); + } + encrypted.descriptor.replicas = Number.isFinite(replicas) ? replicas : 0; + return { + file: encrypted.descriptor, + transactionHash, + storageCostAtto: totalAmount.toString(), + records: encrypted.records.length, + }; +} diff --git a/web/src/upload.test.js b/web/src/upload.test.js new file mode 100644 index 00000000..37b12425 --- /dev/null +++ b/web/src/upload.test.js @@ -0,0 +1,81 @@ +import assert from "node:assert/strict"; +import test from "node:test"; +import { brotliCompressSync, brotliDecompressSync, constants } from "node:zlib"; +import { blake3 } from "@noble/hashes/blake3.js"; +import { decryptSelfEncryptedChunk } from "./file.js"; +import { bytesToHex } from "./protocol.js"; +import { + decodePublicDataMap, + encodePublicDataMap, + encryptPublicFile, +} from "./upload.js"; + +const compress = (input) => + new Uint8Array( + brotliCompressSync(input, { + params: { [constants.BROTLI_PARAM_QUALITY]: 6 }, + }), + ); +const decompress = (input) => new Uint8Array(brotliDecompressSync(input)); + +test("browser encryption reproduces the native self_encryption 0.36 vector", async () => { + const content = new TextEncoder().encode("browser whole-file fixture\n".repeat(160)); + const encrypted = await encryptPublicFile(content, { + name: "fixture.txt", + contentType: "text/plain", + compress, + }); + + assert.deepEqual( + encrypted.descriptor.chunks.map((chunk) => chunk.dst_hash), + [ + "c024c6884a2f39be7ba07c3d9636efedeb94df7397fcd38bac5ae904643c5cc9", + "350a88e6eb0b2a3e774107a212a272b4191af69ca4366a4b91f5a1e5872c459a", + "d73db5a8b0be3b571b40d2b80ff490fe45e135f1992c5863ecb78e25d00ceddb", + ], + ); + assert.equal( + encrypted.descriptor.address, + "0d3636dd504d04a236f7e104909234766f077fa7e1ca4a18293d3d168d5f169b", + ); + assert.deepEqual( + decodePublicDataMap(encrypted.records.at(-1).content), + encrypted.descriptor.chunks, + ); + + const sourceHashes = encrypted.descriptor.chunks.map((chunk) => chunk.src_hash); + const plaintext = await Promise.all( + encrypted.descriptor.chunks.map((chunk, index) => + decryptSelfEncryptedChunk( + chunk, + encrypted.records[index].content, + sourceHashes.map((hash) => Uint8Array.from(Buffer.from(hash, "hex"))), + 0, + decompress, + ), + ), + ); + const reconstructed = new Uint8Array( + plaintext.reduce((total, chunk) => total + chunk.length, 0), + ); + let offset = 0; + for (const chunk of plaintext) { + reconstructed.set(chunk, offset); + offset += chunk.length; + } + assert.deepEqual(reconstructed, content); + assert.equal(bytesToHex(blake3(reconstructed)), encrypted.descriptor.blake3); +}); + +test("public DataMap encoder matches rmp-serde's compact native representation", () => { + const chunks = [0, 1, 2].map((index) => ({ + index, + dst_hash: (11 + index).toString(16).padStart(2, "0").repeat(32), + src_hash: (21 + index).toString(16).padStart(2, "0").repeat(32), + src_size: 100 + index, + })); + const expected = + "9301939400dc00200b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0bdc00201515151515151515151515151515151515151515151515151515151515151515649401dc00200c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0cdc00201616161616161616161616161616161616161616161616161616161616161616659402dc00200d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0ddc0020171717171717171717171717171717171717171717171717171717171717171766c0"; + assert.equal(bytesToHex(encodePublicDataMap(chunks)), expected); + assert.deepEqual(decodePublicDataMap(Uint8Array.from(Buffer.from(expected, "hex"))), chunks); +}); From 716bf7eba901ab99c784dd95868f02d93d3b3a47 Mon Sep 17 00:00:00 2001 From: Mick van Dijke <12992260+mickvandijke@users.noreply.github.com> Date: Wed, 5 Aug 2026 16:26:38 +0200 Subject: [PATCH 04/31] feat(web): move file processing to Rust WASM --- .github/workflows/ci.yml | 32 ++ .gitignore | 1 + Cargo.lock | 39 ++- README.md | 10 +- ant-core/Cargo.toml | 127 +++++-- ant-core/src/browser.rs | 323 ++++++++++++++++++ ant-core/src/lib.rs | 12 + .../ADR-0003-direct-browser-read-client.md | 95 ++++-- web/README.md | 52 ++- web/package-lock.json | 18 - web/package.json | 10 +- web/setup-wasm.js | 5 + web/src/file.js | 137 ++------ web/src/file.test.js | 98 ++---- web/src/main.js | 3 + web/src/payment.js | 8 +- web/src/protocol.js | 8 +- web/src/upload.js | 170 +-------- web/src/upload.test.js | 101 ++---- web/src/wasm.test.js | 50 +++ 20 files changed, 772 insertions(+), 527 deletions(-) create mode 100644 ant-core/src/browser.rs create mode 100644 web/setup-wasm.js create mode 100644 web/src/wasm.test.js diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2382b67c..9c2ed5d4 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -33,6 +33,38 @@ jobs: - uses: Swatinem/rust-cache@v2 - run: cargo clippy --all-targets --all-features -- -D warnings + browser-wasm: + name: Browser WASM + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: dtolnay/rust-toolchain@stable + with: + targets: wasm32-unknown-unknown + - uses: Swatinem/rust-cache@v2 + - uses: actions/setup-node@v4 + with: + node-version: 22 + cache: npm + cache-dependency-path: web/package-lock.json + - name: Check the portable Rust library + run: >- + cargo check -p ant-core + --target wasm32-unknown-unknown + --no-default-features + --features browser-wasm + - name: Install wasm-pack + run: cargo install wasm-pack --version 0.15.0 --locked + - name: Install browser dependencies + working-directory: web + run: npm ci + - name: Test browser client against Rust WASM + working-directory: web + run: npm test + - name: Build browser bundle + working-directory: web + run: npm run build + test-unit: name: Unit Tests (${{ matrix.os }}) runs-on: ${{ matrix.os }} diff --git a/.gitignore b/.gitignore index b71f648b..0b6b5d47 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,7 @@ /target /web/node_modules/ /web/dist/ +/web/pkg/ .cargo/config.toml .claude/plans/ .claude/scheduled_tasks.lock diff --git a/Cargo.lock b/Cargo.lock index b61d5377..35e81e4e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -845,12 +845,15 @@ dependencies = [ "axum", "blake3", "bytes", + "console_error_panic_hook", "flate2", "fs2", "futures", "futures-core", "futures-util", + "getrandom 0.2.17", "hex", + "js-sys", "libc", "lru", "openssl", @@ -862,6 +865,8 @@ dependencies = [ "self_encryption", "semver 1.0.28", "serde", + "serde-wasm-bindgen", + "serde_bytes", "serde_json", "serial_test", "sysinfo", @@ -876,6 +881,7 @@ dependencies = [ "tracing", "tracing-subscriber", "utoipa", + "wasm-bindgen", "windows-sys 0.61.2", "xor_name", "zip", @@ -1849,6 +1855,16 @@ dependencies = [ "windows-sys 0.59.0", ] +[[package]] +name = "console_error_panic_hook" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a06aeb73f470f66dcdbf7223caeebb85984942f22f1adb2a088cf9668146bbbc" +dependencies = [ + "cfg-if", + "wasm-bindgen", +] + [[package]] name = "const-hex" version = "1.19.1" @@ -3255,7 +3271,7 @@ dependencies = [ "js-sys", "log", "wasm-bindgen", - "windows-core 0.57.0", + "windows-core 0.58.0", ] [[package]] @@ -5556,6 +5572,27 @@ dependencies = [ "serde_derive", ] +[[package]] +name = "serde-wasm-bindgen" +version = "0.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8302e169f0eddcc139c70f139d19d6467353af16f9fce27e8c30158036a1e16b" +dependencies = [ + "js-sys", + "serde", + "wasm-bindgen", +] + +[[package]] +name = "serde_bytes" +version = "0.11.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a5d440709e79d88e51ac01c4b72fc6cb7314017bb7da9eeff678aa94c10e3ea8" +dependencies = [ + "serde", + "serde_core", +] + [[package]] name = "serde_core" version = "1.0.228" diff --git a/README.md b/README.md index 4725a7fe..8ffc044f 100644 --- a/README.md +++ b/README.md @@ -6,11 +6,11 @@ A unified CLI and Rust library for storing data on the Autonomi decentralized ne This project provides two Rust crates and a browser client: -- **ant-core** — A headless Rust library containing all business logic: data storage/retrieval with self-encryption and EVM payments, node lifecycle management, and local devnet tooling. Designed to be consumed by any frontend (CLI, GUI, AI agents, REST clients). +- **ant-core** — A headless Rust library containing all business logic: data storage/retrieval with self-encryption and EVM payments, node lifecycle management, and local devnet tooling. Its portable immutable-data core also builds for browsers with the `browser-wasm` feature. - **ant-cli** — A thin CLI binary (`ant`) built on `ant-core`. - **web** — A direct WebTransport client and test site. It performs browser-side - closest-node lookup, reconstructs complete public self-encrypted files, and - saves them without a data gateway. + closest-node lookup and uses `ant-core` through WASM to self-encrypt and + reconstruct complete public files without a data gateway. Data on Autonomi is **content-addressed**. Files are split into encrypted chunks (via [self-encryption](https://en.wikipedia.org/wiki/Convergent_encryption)), each stored at an XOR address derived from its content. A `DataMap` tracks which chunks belong to a file. Payments for storage are made on an EVM-compatible blockchain (Arbitrum). @@ -24,7 +24,7 @@ the browser-enabled node devnet, then run the site: cargo run --features webtransport-poc --bin ant-devnet -- \ --preset minimal --base-port 23000 \ --webtransport --webtransport-base-port 24000 \ - --serve-port 25000 --enable-logging + --serve-port 25000 --enable-evm --enable-logging # In ant-client-web-support/web npm ci @@ -33,7 +33,7 @@ npm run dev Open `http://127.0.0.1:5173`; the default public test file and direct node endpoints are loaded from the local browser manifest. See [web/README.md](web/README.md) -for the protocol flow and LAN configuration. +for the one-time `wasm-pack` setup, protocol flow, and LAN configuration. ## Installation diff --git a/ant-core/Cargo.toml b/ant-core/Cargo.toml index 3fcff457..11c3c93e 100644 --- a/ant-core/Cargo.toml +++ b/ant-core/Cargo.toml @@ -7,29 +7,41 @@ license = "MIT OR Apache-2.0" repository = "https://github.com/WithAutonomi/ant-client" readme = "../README.md" +[lib] +crate-type = ["rlib", "cdylib"] + [dependencies] -# Node management -async-stream = "0.3" -axum = "0.8" -flate2 = "1" +# Cross-platform data primitives. These are the only dependencies compiled for +# the browser-WASM target; native networking and node management stay behind +# the `native` feature below. blake3 = "1" -fs2 = "0.4" -futures-core = "0.3" -futures-util = "0.3" -self-replace = "1" -semver = "1" +bytes = "1" +hex = "0.4" +rmp-serde = "1" serde = { version = "1", features = ["derive"] } -serde_json = "1" -reqwest = { version = "0.12", features = ["json", "stream"] } -tar = "0.4" -tempfile = "3" -toml = "0.8" +serde_bytes = "0.11" +self_encryption = "0.36" thiserror = "2" -tokio = { version = "1", features = ["full"] } -tokio-util = { version = "0.7", features = ["rt"] } -utoipa = { version = "5", features = ["axum_extras"] } -zip = "2" -tower-http = { version = "0.6.8", features = ["cors"] } + +# Node management +async-stream = { version = "0.3", optional = true } +axum = { version = "0.8", optional = true } +flate2 = { version = "1", optional = true } +fs2 = { version = "0.4", optional = true } +futures-core = { version = "0.3", optional = true } +futures-util = { version = "0.3", optional = true } +self-replace = { version = "1", optional = true } +semver = { version = "1", optional = true } +serde_json = { version = "1", optional = true } +reqwest = { version = "0.12", features = ["json", "stream"], optional = true } +tar = { version = "0.4", optional = true } +tempfile = { version = "3", optional = true } +toml = { version = "0.8", optional = true } +tokio = { version = "1", features = ["full"], optional = true } +tokio-util = { version = "0.7", features = ["rt"], optional = true } +utoipa = { version = "5", features = ["axum_extras"], optional = true } +zip = { version = "2", optional = true } +tower-http = { version = "0.6.8", features = ["cors"], optional = true } # Data operations # Wire protocol crate: gives us `ant_protocol::{chunk, payment, …}` plus @@ -37,23 +49,19 @@ tower-http = { version = "0.6.8", features = ["cors"] } # under `ant_protocol::{evm, transport, pqc}`. This is the ONE pin for # those three deps — do not add direct evmlib/saorsa-core/saorsa-pqc # deps here or the version can skew between ant-client and ant-node. -ant-protocol = "2.3.1" -xor_name = "5" -self_encryption = "0.36" -futures = "0.3" -postcard = { version = "1.1.3", features = ["use-std"] } -rmp-serde = "1" -hex = "0.4" -tracing = "0.1" -bytes = "1" -lru = "0.16" -rand = "0.8" +ant-protocol = { version = "2.3.1", optional = true } +xor_name = { version = "5", optional = true } +futures = { version = "0.3", optional = true } +postcard = { version = "1.1.3", features = ["use-std"], optional = true } +tracing = { version = "0.1", optional = true } +lru = { version = "0.16", optional = true } +rand = { version = "0.8", optional = true } # Used by the daemon supervisor to scan the OS process table when adopting # running nodes whose pid file is missing (e.g. nodes spawned by a pre-adoption # daemon). Happy-path adoption reads the pid file directly and doesn't touch # sysinfo, so the crate is loaded lazily and the cost is bounded to first-time # upgrade scenarios. -sysinfo = { version = "0.32", default-features = false, features = ["system"] } +sysinfo = { version = "0.32", default-features = false, features = ["system"], optional = true } # ant-node is optional. It is only linked for the `LocalDevnet` wrapper # that spawns a local in-process network for development and testing. # Enable with `--features devnet`. @@ -66,7 +74,19 @@ sysinfo = { version = "0.32", default-features = false, features = ["system"] } # track the matching released version carrying the same saorsa-core / # ant-protocol lineage. ant-node = { version = "0.16.0", optional = true } -tracing-subscriber = { version = "0.3", features = ["env-filter"] } +tracing-subscriber = { version = "0.3", features = ["env-filter"], optional = true } + +# Browser bindings are optional so ordinary native consumers do not pull the +# wasm-bindgen toolchain into their dependency graph. +js-sys = { version = "0.3", optional = true } +serde-wasm-bindgen = { version = "0.6", optional = true } +wasm-bindgen = { version = "0.2", optional = true } + +[target.'cfg(target_arch = "wasm32")'.dependencies] +# self_encryption/rand use getrandom 0.2. Browser entropy is supplied by the +# Web Crypto API through its `js` feature. +getrandom = { version = "0.2", features = ["js"] } +console_error_panic_hook = { version = "0.1", optional = true } [target.'cfg(unix)'.dependencies] libc = "0.2" @@ -79,12 +99,47 @@ openssl = { version = "0.10", features = ["vendored"] } windows-sys = { version = "0.61", features = ["Win32_Foundation", "Win32_System_Console", "Win32_System_Threading"] } [features] -# No features enabled by default — consumers that want to spawn a local -# devnet opt in with `features = ["devnet"]`. -default = [] +# Preserve the existing native ant-core API for ordinary consumers. WASM +# consumers build with `--no-default-features --features browser-wasm`. +default = ["native"] +native = [ + "dep:ant-protocol", + "dep:async-stream", + "dep:axum", + "dep:flate2", + "dep:fs2", + "dep:futures", + "dep:futures-core", + "dep:futures-util", + "dep:lru", + "dep:postcard", + "dep:rand", + "dep:reqwest", + "dep:self-replace", + "dep:semver", + "dep:serde_json", + "dep:sysinfo", + "dep:tar", + "dep:tempfile", + "dep:tokio", + "dep:tokio-util", + "dep:toml", + "dep:tower-http", + "dep:tracing", + "dep:tracing-subscriber", + "dep:utoipa", + "dep:xor_name", + "dep:zip", +] +browser-wasm = [ + "dep:console_error_panic_hook", + "dep:js-sys", + "dep:serde-wasm-bindgen", + "dep:wasm-bindgen", +] # Enable `LocalDevnet` (ant-core/src/node/devnet.rs) which wraps # `ant_node::devnet::Devnet` and an Anvil EVM testnet. -devnet = ["dep:ant-node"] +devnet = ["native", "dep:ant-node"] # Expose test-only client seams (e.g. forcing the ADR-0002 extended PUT # fallback) used by the e2e/integration test suite. test-utils = [] diff --git a/ant-core/src/browser.rs b/ant-core/src/browser.rs new file mode 100644 index 00000000..9efe5dfe --- /dev/null +++ b/ant-core/src/browser.rs @@ -0,0 +1,323 @@ +//! Browser-safe immutable-data primitives. +//! +//! This module deliberately contains no transport, filesystem, Tokio runtime, +//! or EVM provider. It is the shared compatibility-sensitive core used by the +//! native client and by the browser WASM package: native self-encryption, +//! public DataMap encoding, content addressing, and reconstruction. + +use bytes::Bytes; +use self_encryption::{DataMap, EncryptedChunk}; +use serde::{Deserialize, Serialize}; + +/// Maximum file size accepted by the in-memory browser demo. +pub const MAX_BROWSER_FILE_BYTES: usize = 64 * 1024 * 1024; + +/// One native self-encryption chunk descriptor exposed to the browser. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct BrowserChunkInfo { + /// Zero-based chunk index. + pub index: usize, + /// BLAKE3 address of the encrypted record. + pub dst_hash: String, + /// BLAKE3 hash of the plaintext chunk. + pub src_hash: String, + /// Plaintext chunk size. + pub src_size: usize, +} + +/// One content-addressed record ready for network upload. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct BrowserRecord { + /// Lowercase hexadecimal BLAKE3 record address. + pub address: String, + /// Raw record bytes. `serde_bytes` maps this to `Uint8Array` in WASM. + #[serde(with = "serde_bytes")] + pub content: Vec, +} + +/// Result of native public-file self-encryption for a browser upload. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct BrowserEncryptedFile { + /// Public DataMap record address. + pub address: String, + /// Whole-file plaintext BLAKE3 hash. + pub blake3: String, + /// Serialized public DataMap size. + pub data_map_size: usize, + /// Native root DataMap chunk descriptors. + pub chunks: Vec, + /// Encrypted data records followed by the public DataMap record. + pub records: Vec, +} + +/// Browser immutable-data processing error. +#[derive(Debug, thiserror::Error)] +pub enum BrowserError { + /// Input did not satisfy the browser demo limits. + #[error("invalid browser data: {0}")] + Invalid(String), + /// Native self-encryption failed. + #[error("self-encryption failed: {0}")] + SelfEncryption(String), + /// Public DataMap encoding or decoding failed. + #[error("DataMap serialization failed: {0}")] + DataMap(String), +} + +/// BLAKE3-address bytes using the same lowercase hexadecimal representation as +/// the native chunk protocol. +#[must_use] +pub fn content_address(content: &[u8]) -> String { + blake3::hash(content).to_hex().to_string() +} + +/// Verify raw record bytes against a lowercase or uppercase hexadecimal BLAKE3 +/// address. +pub fn verify_record(address: &str, content: &[u8]) -> Result<(), BrowserError> { + let expected = address.strip_prefix("0x").unwrap_or(address); + if expected.len() != 64 || !expected.bytes().all(|byte| byte.is_ascii_hexdigit()) { + return Err(BrowserError::Invalid( + "record address must be 32 hexadecimal bytes".to_string(), + )); + } + let actual = content_address(content); + if !actual.eq_ignore_ascii_case(expected) { + return Err(BrowserError::Invalid(format!( + "BLAKE3 mismatch: expected {}, received {actual}", + expected.to_ascii_lowercase() + ))); + } + Ok(()) +} + +/// Encrypt a complete public file with the native `self_encryption 0.36` +/// implementation and append its MessagePack DataMap as a public record. +pub fn encrypt_public_file(content: &[u8]) -> Result { + if content.len() < self_encryption::MIN_ENCRYPTABLE_BYTES { + return Err(BrowserError::Invalid(format!( + "self-encryption requires at least {} bytes", + self_encryption::MIN_ENCRYPTABLE_BYTES + ))); + } + if content.len() > MAX_BROWSER_FILE_BYTES { + return Err(BrowserError::Invalid(format!( + "browser files are limited to {MAX_BROWSER_FILE_BYTES} bytes" + ))); + } + + let whole_file_hash = content_address(content); + let (data_map, encrypted_chunks) = self_encryption::encrypt(Bytes::copy_from_slice(content)) + .map_err(|error| BrowserError::SelfEncryption(error.to_string()))?; + if data_map.is_child() { + return Err(BrowserError::Invalid( + "nested DataMaps are not supported by the browser client".to_string(), + )); + } + let chunks = chunk_infos(&data_map); + if encrypted_chunks.len() != chunks.len() { + return Err(BrowserError::SelfEncryption(format!( + "native encryptor returned {} records for {} DataMap entries", + encrypted_chunks.len(), + chunks.len() + ))); + } + + let mut records: Vec = encrypted_chunks + .into_iter() + .zip(&chunks) + .map(|(chunk, info)| BrowserRecord { + address: info.dst_hash.clone(), + content: chunk.content.to_vec(), + }) + .collect(); + let encoded_data_map = + rmp_serde::to_vec(&data_map).map_err(|error| BrowserError::DataMap(error.to_string()))?; + let address = content_address(&encoded_data_map); + let data_map_size = encoded_data_map.len(); + records.push(BrowserRecord { + address: address.clone(), + content: encoded_data_map, + }); + + Ok(BrowserEncryptedFile { + address, + blake3: whole_file_hash, + data_map_size, + chunks, + records, + }) +} + +/// Decode and normalize a native public DataMap. +pub fn decode_public_data_map(content: &[u8]) -> Result, BrowserError> { + let data_map: DataMap = + rmp_serde::from_slice(content).map_err(|error| BrowserError::DataMap(error.to_string()))?; + if data_map.is_child() { + return Err(BrowserError::Invalid( + "nested DataMaps are not supported by the browser client".to_string(), + )); + } + Ok(chunk_infos(&data_map)) +} + +/// Reconstruct a public file with native self-encryption after verifying every +/// encrypted record against its DataMap destination address. +pub fn decrypt_public_file( + data_map_content: &[u8], + encrypted_contents: &[Vec], +) -> Result, BrowserError> { + let data_map: DataMap = rmp_serde::from_slice(data_map_content) + .map_err(|error| BrowserError::DataMap(error.to_string()))?; + if data_map.is_child() { + return Err(BrowserError::Invalid( + "nested DataMaps are not supported by the browser client".to_string(), + )); + } + if encrypted_contents.len() != data_map.infos().len() { + return Err(BrowserError::Invalid(format!( + "received {} encrypted records for {} DataMap entries", + encrypted_contents.len(), + data_map.infos().len() + ))); + } + + let encrypted_chunks = data_map + .infos() + .iter() + .zip(encrypted_contents) + .map(|(info, content)| { + verify_record(&hex::encode(info.dst_hash.0), content)?; + Ok(EncryptedChunk { + content: Bytes::copy_from_slice(content), + }) + }) + .collect::, BrowserError>>()?; + self_encryption::decrypt(&data_map, &encrypted_chunks) + .map(|bytes| bytes.to_vec()) + .map_err(|error| BrowserError::SelfEncryption(error.to_string())) +} + +fn chunk_infos(data_map: &DataMap) -> Vec { + data_map + .infos() + .iter() + .map(|info| BrowserChunkInfo { + index: info.index, + dst_hash: hex::encode(info.dst_hash.0), + src_hash: hex::encode(info.src_hash.0), + src_size: info.src_size, + }) + .collect() +} + +#[cfg(all(target_arch = "wasm32", feature = "browser-wasm"))] +mod wasm { + use super::{content_address, decrypt_public_file, encrypt_public_file, verify_record}; + use js_sys::{Array, Uint8Array}; + use wasm_bindgen::prelude::*; + + /// Install a readable panic hook for browser developer tools. + #[wasm_bindgen(start)] + pub fn start() { + console_error_panic_hook::set_once(); + } + + /// Native `self_encryption` plus public DataMap generation. + #[wasm_bindgen(js_name = encryptPublicFile)] + pub fn encrypt_public_file_wasm(content: &[u8]) -> Result { + let encrypted = + encrypt_public_file(content).map_err(|error| JsValue::from_str(&error.to_string()))?; + serde_wasm_bindgen::to_value(&encrypted) + .map_err(|error| JsValue::from_str(&error.to_string())) + } + + /// Native BLAKE3 content address. + #[wasm_bindgen(js_name = contentAddress)] + #[must_use] + pub fn content_address_wasm(content: &[u8]) -> String { + content_address(content) + } + + /// Verify one content-addressed record with native BLAKE3. + #[wasm_bindgen(js_name = verifyRecord)] + pub fn verify_record_wasm(address: &str, content: &[u8]) -> Result { + verify_record(address, content).map_err(|error| JsValue::from_str(&error.to_string()))?; + Ok(content_address(content)) + } + + /// Decode a native public DataMap for browser-side record retrieval. + #[wasm_bindgen(js_name = decodePublicDataMap)] + pub fn decode_public_data_map_wasm(content: &[u8]) -> Result { + let chunks = super::decode_public_data_map(content) + .map_err(|error| JsValue::from_str(&error.to_string()))?; + serde_wasm_bindgen::to_value(&chunks).map_err(|error| JsValue::from_str(&error.to_string())) + } + + /// Native public DataMap decoding and whole-file reconstruction. + #[wasm_bindgen(js_name = decryptPublicFile)] + pub fn decrypt_public_file_wasm( + data_map_content: &[u8], + encrypted_contents: Array, + ) -> Result { + let encrypted_contents = encrypted_contents + .iter() + .map(|value| Uint8Array::new(&value).to_vec()) + .collect::>(); + let plaintext = decrypt_public_file(data_map_content, &encrypted_contents) + .map_err(|error| JsValue::from_str(&error.to_string()))?; + Ok(Uint8Array::from(plaintext.as_slice())) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn fixture() -> Vec { + "browser whole-file fixture\n".repeat(160).into_bytes() + } + + #[test] + fn native_browser_encrypt_matches_existing_wire_vector() { + let encrypted = encrypt_public_file(&fixture()).expect("encrypt fixture"); + assert_eq!( + encrypted + .chunks + .iter() + .map(|chunk| chunk.dst_hash.as_str()) + .collect::>(), + vec![ + "c024c6884a2f39be7ba07c3d9636efedeb94df7397fcd38bac5ae904643c5cc9", + "350a88e6eb0b2a3e774107a212a272b4191af69ca4366a4b91f5a1e5872c459a", + "d73db5a8b0be3b571b40d2b80ff490fe45e135f1992c5863ecb78e25d00ceddb", + ] + ); + assert_eq!( + encrypted.address, + "0d3636dd504d04a236f7e104909234766f077fa7e1ca4a18293d3d168d5f169b" + ); + assert_eq!( + encrypted.blake3, + "e0e422267ac59c56bf032d6d830035d343369d20147dd5f6b63351a29b015f22" + ); + } + + #[test] + fn native_browser_round_trip_and_tamper_rejection() { + let content = fixture(); + let encrypted = encrypt_public_file(&content).expect("encrypt fixture"); + let data_map = &encrypted.records.last().expect("DataMap record").content; + let chunks = encrypted.records[..encrypted.records.len() - 1] + .iter() + .map(|record| record.content.clone()) + .collect::>(); + assert_eq!( + decrypt_public_file(data_map, &chunks).expect("decrypt fixture"), + content + ); + + let mut tampered = chunks; + tampered[0][0] ^= 1; + assert!(decrypt_public_file(data_map, &tampered).is_err()); + } +} diff --git a/ant-core/src/lib.rs b/ant-core/src/lib.rs index eab436cd..c39dea3b 100644 --- a/ant-core/src/lib.rs +++ b/ant-core/src/lib.rs @@ -1,6 +1,18 @@ +/// Browser-safe immutable-data primitives shared by native and WASM clients. +pub mod browser; + +#[cfg(feature = "native")] pub mod config; +#[cfg(feature = "native")] pub mod data; +#[cfg(feature = "native")] pub mod datamap_file; +#[cfg(feature = "native")] pub mod error; +#[cfg(feature = "native")] pub mod node; +#[cfg(feature = "native")] pub mod update; + +#[cfg(all(target_arch = "wasm32", not(feature = "browser-wasm")))] +compile_error!("WASM builds of ant-core require the `browser-wasm` feature"); diff --git a/docs/adr/ADR-0003-direct-browser-read-client.md b/docs/adr/ADR-0003-direct-browser-read-client.md index e6121b67..b9410205 100644 --- a/docs/adr/ADR-0003-direct-browser-read-client.md +++ b/docs/adr/ADR-0003-direct-browser-read-client.md @@ -32,21 +32,39 @@ records, and testnet bootstrap-manifest production under ant-node ADR-0009. - The wallet secret must be provided at runtime, used only by the local EVM signer, and never sent to a node or persisted in bootstrap metadata. - Local testnets need a reproducible bootstrap and default-file workflow. +- Compatibility-sensitive file processing should be shared with the Rust + client instead of being independently reimplemented in JavaScript. ## Considered Options 1. **Use the daemon REST API as a data gateway.** Rejected for lookup and file bytes because it would not exercise a full browser client. 2. **Compile the complete native Rust client to WebAssembly.** Deferred because - the native transport, EVM, and filesystem dependency graph is not currently - browser-compatible. -3. **Implement a narrow JavaScript WebTransport immutable-data client - (chosen).** It maps directly to the versioned browser node protocol and - keeps the application boundary small enough to audit. + the native transport, EVM provider, Tokio, and filesystem dependency graph + is not currently browser-compatible. +3. **Use a Rust/WASM data core with thin JavaScript browser adapters + (chosen).** Compile the portable immutable-data part of `ant-core` to WASM + while retaining JavaScript only where browser APIs or currently + native-only dependencies require it. ## Decision -The `web/` package will implement the direct browser client: +`ant-core` has two build surfaces. Its default `native` feature preserves the +existing native library. A `wasm32-unknown-unknown` build with +`--no-default-features --features browser-wasm` excludes node management, +native transport, Tokio, filesystem, and native EVM-provider dependencies and +exports browser-safe immutable-data operations through `wasm-bindgen`. + +The Rust/WASM core will: + +- self-encrypt complete public files with the same `self_encryption 0.36` + implementation used by the Rust client; +- encode and decode the native MessagePack `DataMap` representation; +- calculate and verify BLAKE3 content addresses; +- verify every encrypted record against its DataMap destination address; and +- authenticate, decompress, and reconstruct complete public files. + +The `web/` package will remain responsible for browser-specific orchestration: - load a versioned browser bootstrap manifest containing WebTransport multiaddresses and published immutable-file metadata; @@ -58,8 +76,8 @@ The `web/` package will implement the direct browser client: and `ALPHA = 3`; - query closest direct endpoints with `GET_CHUNK`, retrying `not_found` and unavailable nodes without routing bytes through the manifest service; -- self-encrypt selected public files with the native `self_encryption 0.36` - format and generate the public MessagePack DataMap; +- call the Rust/WASM core to self-encrypt selected public files and generate + their public MessagePack DataMaps; - request ordinary node storage quotes, verify their ML-DSA peer/content binding, forced price, and signed storage commitment before payment; - construct an EVM wallet only from the runtime secret field, approve the @@ -67,14 +85,13 @@ The `web/` package will implement the direct browser client: - upload each content-addressed encrypted record with the signed quote and transaction hash through paid `PUT_CHUNK`; the wallet key never crosses the WebTransport session; -- fetch the public MessagePack DataMap and every resolved encrypted data chunk; -- reconstruct the file with the native `self_encryption 0.36` BLAKE3 KDF, - ChaCha20-Poly1305 authentication, and Brotli decompression; -- verify encrypted-record addresses, per-chunk plaintext hashes and sizes, and - the final whole-file BLAKE3 hash before allowing a save; +- fetch the public MessagePack DataMap and every resolved encrypted data chunk, + then pass those records to the Rust/WASM reconstruction API; +- verify the final file size and use the Rust/WASM BLAKE3 verifier before + allowing a save; - expose a small test site that loads the local testnet manifest, displays the - startup-published file, uploads paid files, and downloads either through the - browser save flow. + startup-published file, uploads paid files, and downloads through the browser + save flow. The local browser manifest is bootstrap metadata, not a gateway. Production clients will replace its unsigned endpoint list with the ML-DSA-signed records @@ -94,11 +111,21 @@ Rust nodes construct and validate this syntax through JavaScript parser is the browser implementation of that same canonical wire format and is covered by matching current/next-pin fixtures. -For the local vertical slice, the bootstrap manifest carries a resolved JSON -view of the public root DataMap alongside its ordinary on-network DataMap -address. The browser still fetches and verifies that public DataMap record and -all file bytes directly from nodes. Production discovery must replace this -unsigned resolved view with parsing and validation of the signed/on-network +Quote and storage-commitment verification remains JavaScript for now. +`ant-protocol 2.3.1` unconditionally reaches the native Saorsa transport and +EVM dependency graph, including Tokio networking and `mio`, and therefore +cannot be linked into a browser WASM target. A future transport-free +`ant-protocol` feature should expose the pure multiaddress, quote, commitment, +and ML-DSA verification types without enabling native networking. At that +point those compatibility-sensitive operations should also move behind the +Rust/WASM boundary. + +For compatibility with the local launcher, the bootstrap manifest still +carries a resolved JSON view of the public root DataMap alongside its ordinary +on-network DataMap address. The download path does not use that copy to select +records: it fetches the public DataMap from a node and uses the Rust/WASM +decoder to derive the encrypted-record addresses. Production discovery must +still replace the unsigned manifest with validation of the signed/on-network metadata chain. ## Consequences @@ -110,14 +137,20 @@ metadata chain. - A local five-node testnet can validate multiple direct node connections, multiaddress-embedded certificate pins, lookup convergence, fallback, and content verification. -- The browser protocol is independent of native Rust serialization details. +- Self-encryption, DataMap serialization, reconstruction, and content + addressing have one Rust implementation across native and browser clients. +- The browser application keeps direct control of WebTransport, wallet, and + DOM APIs without pulling native runtime dependencies into WASM. ### Negative / Trade-offs - The current client reconstructs files in memory and the local launcher caps public files at 64 MiB; upload encryption and reconstruction are not yet streaming. -- JavaScript lookup behavior must remain aligned with native Kademlia rules. +- JavaScript lookup and quote-verification behavior must remain aligned with + native Kademlia and protocol rules until transport-free Rust APIs exist. +- The initial WASM module is approximately 1.4 MiB uncompressed and browser + file processing is still in memory. - Certificate and endpoint verification adds bootstrap-record lifecycle work. ### Neutral / Operational @@ -129,13 +162,19 @@ metadata chain. ## Validation -- Unit tests cover fixed-width identifiers, XOR ordering, bidirectional binary - framing, manifest/payment validation, quote signatures and the native - Keccak-256 EVM quote-hash vector, native-format - encryption/DataMap generation, and BLAKE3 mismatch rejection. +- Rust unit tests cover an exact native `self_encryption 0.36` wire vector, + public DataMap generation, round-trip reconstruction, and tamper rejection. +- JavaScript unit tests cover fixed-width identifiers, XOR ordering, + bidirectional binary framing, manifest/payment validation, quote signatures, + the native Keccak-256 EVM quote-hash vector, and the browser orchestration + around the Rust/WASM boundary. +- CI compiles `ant-core` for `wasm32-unknown-unknown` with native features + disabled, builds the generated `wasm-pack` package, runs browser-client + tests against it, and produces the Vite bundle. - The browser production bundle builds without Node-specific runtime APIs. -- A fixed vector generated by native `self_encryption 0.36` verifies browser - KDF, authenticated decryption, Brotli reconstruction, and tamper rejection. +- The generated WASM package encrypts and reconstructs the same fixed vector + as the native Rust test, covering the native KDF, authenticated decryption, + Brotli reconstruction, MessagePack DataMap, and BLAKE3 addresses. - A live node integration test starts five WebTransport-enabled nodes, publishes a public DataMap and encrypted chunks, connects with the advertised self-contained multiaddress, retrieves every record, and reconstructs the diff --git a/web/README.md b/web/README.md index e60c42fe..0964bbd4 100644 --- a/web/README.md +++ b/web/README.md @@ -2,11 +2,11 @@ This web application is the browser-facing client for ADR-0009. It loads a local testnet bootstrap manifest, connects directly to storage nodes over -WebTransport, performs the XOR closest-node lookup in JavaScript, retrieves a -public DataMap and every encrypted file chunk, reconstructs the complete file, -and verifies its whole-file BLAKE3 hash before saving it. It can also -self-encrypt a file, pay signed node quotes with a wallet held only in the -page, and upload the encrypted records directly to closest storage nodes. +WebTransport, and performs the XOR closest-node lookup in the browser. The +portable part of the Rust `ant-core` library is compiled to WASM and performs +native self-encryption, public DataMap serialization, reconstruction, and +BLAKE3 content verification. The thin JavaScript layer retrieves and uploads +records, uses the browser wallet/payment APIs, and drives the page. The node-side WebTransport listener and testnet manifest API live in the `ant-node-web-support` sibling repository. No HTTP gateway performs lookup or @@ -15,6 +15,8 @@ proxies file bytes. ## Requirements - Rust 1.88 or newer for the node's optional `wtransport` dependency. +- The `wasm32-unknown-unknown` Rust target. +- `wasm-pack` 0.15. - Node.js 20.19+ or 22.12+. - A current browser implementing WebTransport certificate hashes. The client extracts them from node multiaddresses; users do not enter hashes separately. @@ -22,6 +24,13 @@ proxies file bytes. Nodes serialize these addresses from the native `saorsa_core::MultiAddr` representation; the JavaScript parser consumes that canonical string form. +Install the WASM build tools once if needed: + +```bash +rustup target add wasm32-unknown-unknown +cargo install wasm-pack --version 0.15.0 --locked +``` + ## Run the browser-enabled testnet From `ant-node-web-support`: @@ -69,6 +78,10 @@ npm ci npm run dev ``` +The `predev` hook builds `ant-core` with `--no-default-features --features +browser-wasm` into the ignored `web/pkg/` directory before Vite starts. No +published or hand-maintained JavaScript copy of the Rust algorithms is used. + Open `http://127.0.0.1:5173`. The page automatically loads the testnet manifest from port 25000 and fills in the default file: @@ -77,10 +90,11 @@ manifest from port 25000 and fills in the default file: WebTransport `HELLO`. 3. **Find closest** runs the iterative lookup in the browser. 4. Under **Paid public file upload**, choose a file, paste the funded private - key printed by ant-devnet, then select **Pay and upload file**. Encryption, - quote/commitment verification, approval, and payment happen locally. The - key field is cleared immediately and the resulting public DataMap address - is placed in the download field. + key printed by ant-devnet, then select **Pay and upload file**. Rust/WASM + performs encryption and DataMap generation; quote/commitment verification, + approval, and payment happen locally in the browser. The key field is + cleared immediately and the resulting public DataMap address is placed in + the download field. 5. **Download and save file** opens the browser save flow, fetches the public DataMap and encrypted chunks from direct closest storage nodes, reconstructs the whole file, verifies BLAKE3, and retains a **Save again** link. @@ -101,11 +115,19 @@ npm test npm run build ``` +Both commands build the WASM package automatically. To validate the Rust +boundary directly: + +```bash +cargo check -p ant-core --target wasm32-unknown-unknown \ + --no-default-features --features browser-wasm +cargo test -p ant-core --lib browser::tests +``` + The tests cover fixed-width IDs, XOR ordering, bidirectional binary framing, browser manifest/payment validation, signed quote verification including the -native Keccak-256 EVM quote hash, native -encryption/DataMap generation, and a `self_encryption 0.36` compatibility -vector. +native Keccak-256 EVM quote hash, and a Rust `self_encryption 0.36` wire vector +with native/WASM round-trip and tamper verification. Cross-repository live verification additionally starts the node testnet, downloads all public-file records through WebTransport, reconstructs the original bytes, pays a real quote on local Anvil, uploads a fresh record @@ -113,6 +135,12 @@ through the ordinary node payment validator, and reads it back. ## Current boundary +JavaScript currently owns WebTransport stream handling, iterative lookup, +Ethers payment submission, ML-DSA quote verification, and DOM/save APIs. +`ant-protocol 2.3.1` still enables native Saorsa/Tokio networking and cannot be +linked into a browser WASM build; a future transport-free feature can move the +remaining quote and multiaddress verification into Rust. + The local testnet manifest is intentionally unsigned bootstrap material. A production deployment still needs ML-DSA-signed endpoint records, certificate overlap/rotation, network dissemination, relayed WebTransport, and production diff --git a/web/package-lock.json b/web/package-lock.json index a3490305..17d41a4d 100644 --- a/web/package-lock.json +++ b/web/package-lock.json @@ -9,11 +9,8 @@ "version": "0.1.0", "dependencies": { "@msgpack/msgpack": "^3.1.3", - "@noble/ciphers": "2.2.0", "@noble/hashes": "2.2.0", "@noble/post-quantum": "^0.6.1", - "brotli-dec-wasm": "2.3.2", - "brotli-wasm": "^3.0.1", "ethers": "^6.17.0" }, "devDependencies": { @@ -364,21 +361,6 @@ "integrity": "sha512-G965FqalsNyrPqgEGON7nIx1e/OVENSgiEIzyC63haUMuvNnwIgIjMs52hlTCKhkBny7A2ORNlfY9Zu+jmGk1Q==", "license": "MIT" }, - "node_modules/brotli-dec-wasm": { - "version": "2.3.2", - "resolved": "https://registry.npmjs.org/brotli-dec-wasm/-/brotli-dec-wasm-2.3.2.tgz", - "integrity": "sha512-5H+k8eVLIJY6B4olN2HP9QzJAxcplf0jV7mWnkpxvOSeUE9Npg3dQ2pgLn30a9MUFHNro1iSmtdu6VNdlb+TIw==", - "license": "MIT OR Apache-2.0" - }, - "node_modules/brotli-wasm": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/brotli-wasm/-/brotli-wasm-3.0.1.tgz", - "integrity": "sha512-U3K72/JAi3jITpdhZBqzSUq+DUY697tLxOuFXB+FpAE/Ug+5C3VZrv4uA674EUZHxNAuQ9wETXNqQkxZD6oL4A==", - "license": "Apache-2.0", - "engines": { - "node": ">=v18.0.0" - } - }, "node_modules/detect-libc": { "version": "2.1.2", "resolved": "https://registry.npmjs.org/detect-libc/-/detect-libc-2.1.2.tgz", diff --git a/web/package.json b/web/package.json index 6e9f616c..4979f5d5 100644 --- a/web/package.json +++ b/web/package.json @@ -4,17 +4,19 @@ "private": true, "type": "module", "scripts": { + "wasm:dev": "wasm-pack build --target web --out-dir ../web/pkg --dev ../ant-core --no-default-features --features browser-wasm", + "wasm:release": "wasm-pack build --target web --out-dir ../web/pkg --release ../ant-core --no-default-features --features browser-wasm", + "predev": "npm run wasm:dev", "dev": "vite --host 127.0.0.1 --port 5173", + "prebuild": "npm run wasm:release", "build": "vite build", - "test": "node --test" + "pretest": "npm run wasm:dev", + "test": "node --import ./setup-wasm.js --test" }, "dependencies": { "@msgpack/msgpack": "^3.1.3", - "@noble/ciphers": "2.2.0", "@noble/hashes": "2.2.0", "@noble/post-quantum": "^0.6.1", - "brotli-dec-wasm": "2.3.2", - "brotli-wasm": "^3.0.1", "ethers": "^6.17.0" }, "devDependencies": { diff --git a/web/setup-wasm.js b/web/setup-wasm.js new file mode 100644 index 00000000..ee494ea8 --- /dev/null +++ b/web/setup-wasm.js @@ -0,0 +1,5 @@ +import { readFile } from "node:fs/promises"; +import initAntCore from "./pkg/ant_core.js"; + +const wasm = await readFile(new URL("./pkg/ant_core_bg.wasm", import.meta.url)); +await initAntCore({ module_or_path: wasm }); diff --git a/web/src/file.js b/web/src/file.js index a857f567..76d774e8 100644 --- a/web/src/file.js +++ b/web/src/file.js @@ -1,95 +1,11 @@ -import { chacha20poly1305 } from "@noble/ciphers/chacha.js"; -import { blake3 } from "@noble/hashes/blake3.js"; -import { bytesToHex, getChunkFromClosest, hexToBytes } from "./protocol.js"; +import { + decodePublicDataMap as decodePublicDataMapNative, + decryptPublicFile as decryptPublicFileNative, +} from "../pkg/ant_core.js"; +import { getChunkFromClosest, hexToBytes, verifyChunk } from "./protocol.js"; -const KDF_CONTEXT = new TextEncoder().encode("self_encryption/chunk/v2"); -const PAD_SIZE = 52; -const KEY_SIZE = 32; -const NONCE_SIZE = 12; -const DERIVED_SIZE = PAD_SIZE + KEY_SIZE + NONCE_SIZE; const MAX_DOWNLOAD_CONCURRENCY = 6; -function writeU64LittleEndian(target, offset, value) { - const numeric = BigInt(value); - new DataView(target.buffer, target.byteOffset, target.byteLength).setBigUint64( - offset, - numeric, - true, - ); -} - -function predecessorIndices(index, count) { - if (!Number.isSafeInteger(index) || index < 0 || index >= count || count < 3) { - throw new Error(`Invalid self-encryption chunk index ${index}/${count}`); - } - if (index === 0) return [count - 1, count - 2]; - if (index === 1) return [0, count - 1]; - return [index - 1, index - 2]; -} - -export function deriveChunkMaterial(chunk, sourceHashes, childLevel = 0) { - const [previous, previousPrevious] = predecessorIndices(chunk.index, sourceHashes.length); - const context = new Uint8Array(32 * 3 + 8 * 2); - context.set(sourceHashes[chunk.index], 0); - context.set(sourceHashes[previous], 32); - context.set(sourceHashes[previousPrevious], 64); - writeU64LittleEndian(context, 96, chunk.index); - writeU64LittleEndian(context, 104, childLevel); - - const derived = blake3(context, { context: KDF_CONTEXT, dkLen: DERIVED_SIZE }); - return { - pad: derived.slice(0, PAD_SIZE), - key: derived.slice(PAD_SIZE, PAD_SIZE + KEY_SIZE), - nonce: derived.slice(PAD_SIZE + KEY_SIZE), - }; -} - -export async function decryptSelfEncryptedChunk( - chunk, - encryptedContent, - sourceHashes, - childLevel = 0, - decompress = decompressBrotli, -) { - const { pad, key, nonce } = deriveChunkMaterial(chunk, sourceHashes, childLevel); - const ciphertext = new Uint8Array(encryptedContent.length); - for (let index = 0; index < encryptedContent.length; index += 1) { - ciphertext[index] = encryptedContent[index] ^ pad[index % pad.length]; - } - - let compressed; - try { - compressed = chacha20poly1305(key, nonce).decrypt(ciphertext); - } catch (error) { - throw new Error(`Chunk ${chunk.index} authentication failed`, { cause: error }); - } - - let plaintext; - try { - plaintext = await decompress(compressed); - } catch (error) { - throw new Error(`Chunk ${chunk.index} Brotli decompression failed`, { cause: error }); - } - if (plaintext.length !== chunk.src_size) { - throw new Error( - `Chunk ${chunk.index} reconstructed ${plaintext.length} bytes, expected ${chunk.src_size}`, - ); - } - const sourceHash = bytesToHex(blake3(plaintext)); - if (sourceHash !== chunk.src_hash) { - throw new Error( - `Chunk ${chunk.index} plaintext hash mismatch: expected ${chunk.src_hash}, received ${sourceHash}`, - ); - } - return plaintext; -} - -async function decompressBrotli(compressed) { - const { default: brotliPromise } = await import("brotli-dec-wasm"); - const brotli = await brotliPromise; - return brotli.decompress(compressed); -} - async function mapWithConcurrency(items, concurrency, operation) { const results = new Array(items.length); let next = 0; @@ -113,7 +29,8 @@ export async function downloadPublicFile( concurrency = 3, onProgress = () => {}, downloadChunk = getChunkFromClosest, - decompress = decompressBrotli, + decodeDataMap = decodePublicDataMapNative, + decrypt = decryptPublicFileNative, } = {}, ) { hexToBytes(file.address, 32); @@ -131,44 +48,34 @@ export async function downloadPublicFile( ); } onProgress(`Verified public DataMap (${dataMap.content.length} bytes)`); + const chunks = decodeDataMap(dataMap.content); + if (!Array.isArray(chunks) || chunks.length < 3) { + throw new Error("ant-core WASM returned an invalid public DataMap"); + } - const sourceHashes = file.chunks.map((chunk) => hexToBytes(chunk.src_hash, 32)); - const plaintextChunks = await mapWithConcurrency( - file.chunks, + const encryptedChunks = await mapWithConcurrency( + chunks, boundedConcurrency, async (chunk) => { onProgress( - `Fetching encrypted file chunk ${chunk.index + 1}/${file.chunks.length} (${chunk.dst_hash})`, + `Fetching encrypted file chunk ${chunk.index + 1}/${chunks.length} (${chunk.dst_hash})`, ); const downloaded = await downloadChunk(seedEndpoints, chunk.dst_hash, { onProgress, }); - const plaintext = await decryptSelfEncryptedChunk( - chunk, - downloaded.content, - sourceHashes, - 0, - decompress, - ); - onProgress(`Reconstructed file chunk ${chunk.index + 1}/${file.chunks.length}`); - return plaintext; + return downloaded.content; }, ); - const totalSize = plaintextChunks.reduce((total, chunk) => total + chunk.length, 0); - if (totalSize !== file.size) { - throw new Error(`Reconstructed file has ${totalSize} bytes, expected ${file.size}`); - } - const content = new Uint8Array(totalSize); - let offset = 0; - for (const chunk of plaintextChunks) { - content.set(chunk, offset); - offset += chunk.length; + onProgress(`Reconstructing ${file.name} with native ant-core WASM`); + const content = decrypt(dataMap.content, encryptedChunks); + if (!(content instanceof Uint8Array)) { + throw new Error("ant-core WASM returned non-byte file content"); } - const hash = bytesToHex(blake3(content)); - if (hash !== file.blake3) { - throw new Error(`Whole-file BLAKE3 mismatch: expected ${file.blake3}, received ${hash}`); + if (content.length !== file.size) { + throw new Error(`Reconstructed file has ${content.length} bytes, expected ${file.size}`); } + const hash = verifyChunk(file.blake3, content); onProgress(`Verified complete ${file.name} as ${hash}`); return { content, hash, dataMapNode: dataMap.node }; } diff --git a/web/src/file.test.js b/web/src/file.test.js index fbe46329..86200dab 100644 --- a/web/src/file.test.js +++ b/web/src/file.test.js @@ -1,81 +1,21 @@ import assert from "node:assert/strict"; import test from "node:test"; -import { brotliDecompressSync } from "node:zlib"; import { blake3 } from "@noble/hashes/blake3.js"; -import { decryptSelfEncryptedChunk, downloadPublicFile } from "./file.js"; -import { bytesToHex, hexToBytes } from "./protocol.js"; +import { downloadPublicFile } from "./file.js"; +import { bytesToHex } from "./protocol.js"; -// Generated by self_encryption 0.36 from -// "browser whole-file fixture\n" repeated 160 times. This is a compatibility -// vector, not a JavaScript-generated round trip. -const chunks = [ - { - index: 0, - dst_hash: "c024c6884a2f39be7ba07c3d9636efedeb94df7397fcd38bac5ae904643c5cc9", - src_hash: "a049deadbb9eb8ef8102a99698ec8484ba4ba9f1d82089af28b31d70783aae8b", - src_size: 1440, - encrypted: - "7508d7886030b903dce0178555d3df9115240fb3efe6029c4b57c05f84d82b85aedb898c09345c65a55dc32ff125e6af96ceab", - }, - { - index: 1, - dst_hash: "350a88e6eb0b2a3e774107a212a272b4191af69ca4366a4b91f5a1e5872c459a", - src_hash: "a15219e6c26233304a4e733946b97282c2d7ac270e6bd31b7ebdd22ee428cbab", - src_size: 1440, - encrypted: - "5244d1869387ef32f8bec83ff7227fac29d3963a198f271f4ec74b84c1f364bdf2f7964850ecb51c36e30deef5e96c440782e17401", - }, - { - index: 2, - dst_hash: "d73db5a8b0be3b571b40d2b80ff490fe45e135f1992c5863ecb78e25d00ceddb", - src_hash: "ab7ff601a29bd95610dea57dad100e54d53a837db06604afa3c2eaf482d2a82e", - src_size: 1440, - encrypted: - "601efca604bda1cdcde9e79722e5e71d9fcd5764c0fd52a9d0dbf6aeaec53fc462cbc01a6a372a2247ab7a847538ed8e623609", - }, -]; - -const decompress = (input) => new Uint8Array(brotliDecompressSync(input)); - -test("decrypts a native self_encryption 0.36 whole-file vector", async () => { - const sourceHashes = chunks.map((chunk) => hexToBytes(chunk.src_hash, 32)); - const plaintextChunks = await Promise.all( - chunks.map((chunk) => - decryptSelfEncryptedChunk(chunk, hexToBytes(chunk.encrypted), sourceHashes, 0, decompress), - ), - ); - const plaintext = new Uint8Array( - plaintextChunks.reduce((total, chunk) => total + chunk.length, 0), - ); - let offset = 0; - for (const chunk of plaintextChunks) { - plaintext.set(chunk, offset); - offset += chunk.length; - } - - assert.equal(new TextDecoder().decode(plaintext), "browser whole-file fixture\n".repeat(160)); - assert.equal( - bytesToHex(blake3(plaintext)), - "e0e422267ac59c56bf032d6d830035d343369d20147dd5f6b63351a29b015f22", - ); -}); - -test("rejects an authenticated self-encryption chunk after tampering", async () => { - const sourceHashes = chunks.map((chunk) => hexToBytes(chunk.src_hash, 32)); - const tampered = hexToBytes(chunks[0].encrypted); - tampered[0] ^= 1; - await assert.rejects( - decryptSelfEncryptedChunk(chunks[0], tampered, sourceHashes, 0, decompress), - /authentication failed/, - ); -}); - -test("downloads a public DataMap and reconstructs a complete saveable file", async () => { +test("downloads records and delegates reconstruction to ant-core WASM", async () => { const content = new TextEncoder().encode("browser whole-file fixture\n".repeat(160)); - const dataMapContent = new TextEncoder().encode("native public DataMap fixture"); + const dataMapContent = Uint8Array.of(9, 8, 7); const dataMapAddress = bytesToHex(blake3(dataMapContent)); + const chunks = [0, 1, 2].map((index) => ({ + index, + dst_hash: (index + 1).toString(16).padStart(2, "0").repeat(32), + src_hash: (index + 4).toString(16).padStart(2, "0").repeat(32), + src_size: content.length / 3, + })); const encryptedByAddress = new Map( - chunks.map((chunk) => [chunk.dst_hash, hexToBytes(chunk.encrypted)]), + chunks.map((chunk, index) => [chunk.dst_hash, Uint8Array.of(index)]), ); encryptedByAddress.set(dataMapAddress, dataMapContent); const requested = []; @@ -97,7 +37,21 @@ test("downloads a public DataMap and reconstructs a complete saveable file", asy data_map_size: dataMapContent.length, chunks, }, - { downloadChunk, decompress }, + { + downloadChunk, + decodeDataMap: (receivedDataMap) => { + assert.deepEqual(receivedDataMap, dataMapContent); + return chunks; + }, + decrypt: (receivedDataMap, encryptedContents) => { + assert.deepEqual(receivedDataMap, dataMapContent); + assert.deepEqual( + encryptedContents, + chunks.map((_, index) => Uint8Array.of(index)), + ); + return content; + }, + }, ); assert.deepEqual(result.content, content); diff --git a/web/src/main.js b/web/src/main.js index 192de1a9..dc2c27f5 100644 --- a/web/src/main.js +++ b/web/src/main.js @@ -1,4 +1,5 @@ import "./style.css"; +import initAntCore from "../pkg/ant_core.js"; import { BrowserNodeClient, bytesToHex, @@ -9,6 +10,8 @@ import { downloadPublicFile } from "./file.js"; import { fetchBrowserManifest } from "./manifest.js"; import { uploadPublicFile } from "./upload.js"; +await initAntCore(); + const elements = { manifestUrl: document.querySelector("#manifest-url"), loadManifest: document.querySelector("#load-manifest"), diff --git a/web/src/payment.js b/web/src/payment.js index 7f61c032..f5ce0e94 100644 --- a/web/src/payment.js +++ b/web/src/payment.js @@ -1,4 +1,3 @@ -import { blake3 } from "@noble/hashes/blake3.js"; import { ml_dsa65 } from "@noble/post-quantum/ml-dsa.js"; import { decode } from "@msgpack/msgpack"; import { @@ -9,6 +8,7 @@ import { NonceManager, Wallet, } from "ethers"; +import { contentAddress as contentAddressNative } from "../pkg/ant_core.js"; import { bytesToHex, hexToBytes } from "./protocol.js"; const U256_MAX = (1n << 256n) - 1n; @@ -172,7 +172,7 @@ function verifyCommitment(commitment, quote) { if (!Number.isSafeInteger(keyCount) || keyCount !== quote.committed_key_count) { throw new Error("Storage commitment key count does not match quote"); } - if (bytesToHex(blake3(publicKey)) !== quote.peer_id.toLowerCase()) { + if (contentAddressNative(publicKey) !== quote.peer_id.toLowerCase()) { throw new Error("Storage commitment public key is not bound to quote peer"); } if (bytesToHex(peerId) !== quote.peer_id.toLowerCase()) { @@ -209,7 +209,7 @@ function verifyCommitment(commitment, quote) { postcardVarint(signature.length), signature, ); - const pin = bytesToHex(blake3(concatBytes(DOMAIN_COMMITMENT_HASH, postcard))); + const pin = contentAddressNative(concatBytes(DOMAIN_COMMITMENT_HASH, postcard)); if (pin !== quote.commitment_pin.toLowerCase()) { throw new Error("Storage commitment does not resolve the quote pin"); } @@ -234,7 +234,7 @@ export function verifyStorageQuote(quote, expectedAddress, expectedPeerId) { if (signature.length !== ml_dsa65.lengths.signature) { throw new Error(`Storage quote has a ${signature.length}-byte signature`); } - if (bytesToHex(blake3(publicKey)) !== quote.peer_id.toLowerCase()) { + if (contentAddressNative(publicKey) !== quote.peer_id.toLowerCase()) { throw new Error("Storage quote public key is not bound to its peer ID"); } const signedBytes = canonicalQuoteBytes(quote); diff --git a/web/src/protocol.js b/web/src/protocol.js index b439c394..153ce6a0 100644 --- a/web/src/protocol.js +++ b/web/src/protocol.js @@ -1,5 +1,5 @@ -import { blake3 } from "@noble/hashes/blake3.js"; import { bytesToHex as nobleBytesToHex } from "@noble/hashes/utils.js"; +import { verifyRecord as verifyRecordNative } from "../pkg/ant_core.js"; export const PROTOCOL_VERSION = 3; export const PROTOCOL_NAME = "autonomi.web.poc.v3"; @@ -47,11 +47,7 @@ export function xorDistance(peerId, target) { export function verifyChunk(address, content) { const expected = address.trim().replace(/^0x/i, "").toLowerCase(); hexToBytes(expected, 32); - const actual = nobleBytesToHex(blake3(content)); - if (actual !== expected) { - throw new Error(`BLAKE3 mismatch: expected ${expected}, received ${actual}`); - } - return actual; + return verifyRecordNative(expected, content); } export function parseResponseFrame(frame) { diff --git a/web/src/upload.js b/web/src/upload.js index 5cc234b6..f5bff9ce 100644 --- a/web/src/upload.js +++ b/web/src/upload.js @@ -1,174 +1,36 @@ -import { chacha20poly1305 } from "@noble/ciphers/chacha.js"; -import { blake3 } from "@noble/hashes/blake3.js"; -import { decode, encode } from "@msgpack/msgpack"; -import { deriveChunkMaterial } from "./file.js"; -import { - BrowserNodeClient, - bytesToHex, - hexToBytes, - iterativeFindClosest, -} from "./protocol.js"; +import { encryptPublicFile as encryptPublicFileNative } from "../pkg/ant_core.js"; +import { BrowserNodeClient, iterativeFindClosest } from "./protocol.js"; import { payForStorageQuotes, verifyStorageQuote } from "./payment.js"; export const MAX_BROWSER_UPLOAD_BYTES = 64 * 1024 * 1024; -const SELF_ENCRYPTION_MAX_CHUNK_SIZE = 4_190_208; const MAX_STORE_TARGETS = 7; -function numberOfChunks(fileSize) { - if (fileSize < 3) return 0; - if (fileSize < 3 * SELF_ENCRYPTION_MAX_CHUNK_SIZE) return 3; - return Math.ceil(fileSize / SELF_ENCRYPTION_MAX_CHUNK_SIZE); -} - -function chunkSize(fileSize, index) { - if (fileSize < 3 * SELF_ENCRYPTION_MAX_CHUNK_SIZE) { - return index < 2 ? Math.floor(fileSize / 3) : fileSize - 2 * Math.floor(fileSize / 3); - } - const count = numberOfChunks(fileSize); - const remainder = fileSize % SELF_ENCRYPTION_MAX_CHUNK_SIZE; - if (index < count - 2 || remainder === 0) return SELF_ENCRYPTION_MAX_CHUNK_SIZE; - return index === count - 2 ? SELF_ENCRYPTION_MAX_CHUNK_SIZE : remainder; -} - -function chunkStart(fileSize, index) { - const count = numberOfChunks(fileSize); - if (index === count - 1) { - return chunkSize(fileSize, 0) * (index - 1) + chunkSize(fileSize, index - 1); - } - return chunkSize(fileSize, 0) * index; -} - -async function defaultCompress(input) { - const { default: brotliPromise } = await import("brotli-wasm"); - const brotli = await brotliPromise; - return brotli.compress(input, { quality: 6 }); -} - -function xorPad(content, pad) { - const output = new Uint8Array(content.length); - for (let index = 0; index < content.length; index += 1) { - output[index] = content[index] ^ pad[index % pad.length]; - } - return output; -} - -export function encodePublicDataMap(chunks) { - const compact = [ - 1, - chunks.map((chunk) => [ - chunk.index, - Array.from(hexToBytes(chunk.dst_hash, 32)), - Array.from(hexToBytes(chunk.src_hash, 32)), - chunk.src_size, - ]), - null, - ]; - return encode(compact, { sortKeys: false }); -} - -function fixedBytes(value, length, label) { - const bytes = value instanceof Uint8Array ? value : Uint8Array.from(value ?? []); - if (bytes.length !== length) throw new Error(`${label} must contain ${length} bytes`); - return bytes; -} - -export function decodePublicDataMap(content) { - let dataMap; - try { - dataMap = decode(content); - } catch (error) { - throw new Error(`Public DataMap is not valid MessagePack: ${error.message}`, { - cause: error, - }); - } - if (!Array.isArray(dataMap) || dataMap.length !== 3 || dataMap[0] !== 1) { - throw new Error("Public DataMap does not use self_encryption version 1"); - } - if (dataMap[2] !== null) { - throw new Error("Nested DataMaps are not yet supported by the browser uploader"); - } - if (!Array.isArray(dataMap[1]) || dataMap[1].length < 3) { - throw new Error("Public DataMap has fewer than three chunks"); - } - const chunks = dataMap[1] - .map((chunk) => { - if (!Array.isArray(chunk) || chunk.length !== 4) { - throw new Error("Public DataMap contains an invalid chunk descriptor"); - } - const [index, dstHash, srcHash, srcSize] = chunk; - if (!Number.isSafeInteger(index) || index < 0) { - throw new Error(`Invalid DataMap chunk index ${index}`); - } - if (!Number.isSafeInteger(srcSize) || srcSize < 1) { - throw new Error(`Invalid DataMap plaintext chunk size ${srcSize}`); - } - return { - index, - dst_hash: bytesToHex(fixedBytes(dstHash, 32, "DataMap destination hash")), - src_hash: bytesToHex(fixedBytes(srcHash, 32, "DataMap source hash")), - src_size: srcSize, - }; - }) - .sort((left, right) => left.index - right.index); - chunks.forEach((chunk, index) => { - if (chunk.index !== index) throw new Error("DataMap chunk indices are not contiguous"); - }); - return chunks; -} - export async function encryptPublicFile( content, - { name = "upload.bin", contentType = "application/octet-stream", compress = defaultCompress } = {}, + { + name = "upload.bin", + contentType = "application/octet-stream", + encrypt = encryptPublicFileNative, + } = {}, ) { if (!(content instanceof Uint8Array)) throw new Error("Upload content must be bytes"); if (content.length < 3) throw new Error("Self-encryption requires a file of at least 3 bytes"); if (content.length > MAX_BROWSER_UPLOAD_BYTES) { throw new Error(`Browser uploads are limited to ${MAX_BROWSER_UPLOAD_BYTES} bytes`); } - const count = numberOfChunks(content.length); - const plaintextChunks = Array.from({ length: count }, (_, index) => { - const start = chunkStart(content.length, index); - return content.slice(start, start + chunkSize(content.length, index)); - }); - const sourceHashes = plaintextChunks.map((chunk) => blake3(chunk)); - const encrypted = await Promise.all( - plaintextChunks.map(async (plaintext, index) => { - const descriptor = { index }; - const { pad, key, nonce } = deriveChunkMaterial(descriptor, sourceHashes, 0); - const compressed = await compress(plaintext); - const ciphertext = chacha20poly1305(key, nonce).encrypt(compressed); - const bytes = xorPad(ciphertext, pad); - return { - content: bytes, - info: { - index, - dst_hash: bytesToHex(blake3(bytes)), - src_hash: bytesToHex(sourceHashes[index]), - src_size: plaintext.length, - }, - }; - }), - ); - const chunks = encrypted.map(({ info }) => info); - const dataMap = encodePublicDataMap(chunks); - const address = bytesToHex(blake3(dataMap)); - const records = encrypted.map(({ content: bytes, info }) => ({ - address: info.dst_hash, - content: bytes, - })); - records.push({ address, content: dataMap }); + const encrypted = encrypt(content); return { descriptor: { name, - address, + address: encrypted.address, size: content.length, content_type: contentType || "application/octet-stream", - blake3: bytesToHex(blake3(content)), - data_map_size: dataMap.length, - chunks, + blake3: encrypted.blake3, + data_map_size: encrypted.data_map_size, + chunks: encrypted.chunks, replicas: 0, }, - records, + records: encrypted.records, }; } @@ -276,14 +138,14 @@ export async function uploadPublicFile( paymentNetwork, file, walletSecret, - { onProgress = () => {}, compress = defaultCompress } = {}, + { onProgress = () => {}, encrypt = encryptPublicFileNative } = {}, ) { const content = new Uint8Array(await file.arrayBuffer()); - onProgress(`Self-encrypting ${file.name} (${content.length.toLocaleString()} bytes)`); + onProgress(`Self-encrypting ${file.name} with native ant-core WASM (${content.length.toLocaleString()} bytes)`); const encrypted = await encryptPublicFile(content, { name: file.name, contentType: file.type, - compress, + encrypt, }); const prepared = []; for (let index = 0; index < encrypted.records.length; index += 1) { diff --git a/web/src/upload.test.js b/web/src/upload.test.js index 37b12425..e53ec6ae 100644 --- a/web/src/upload.test.js +++ b/web/src/upload.test.js @@ -1,81 +1,38 @@ import assert from "node:assert/strict"; import test from "node:test"; -import { brotliCompressSync, brotliDecompressSync, constants } from "node:zlib"; -import { blake3 } from "@noble/hashes/blake3.js"; -import { decryptSelfEncryptedChunk } from "./file.js"; -import { bytesToHex } from "./protocol.js"; -import { - decodePublicDataMap, - encodePublicDataMap, - encryptPublicFile, -} from "./upload.js"; +import { encryptPublicFile } from "./upload.js"; -const compress = (input) => - new Uint8Array( - brotliCompressSync(input, { - params: { [constants.BROTLI_PARAM_QUALITY]: 6 }, - }), - ); -const decompress = (input) => new Uint8Array(brotliDecompressSync(input)); - -test("browser encryption reproduces the native self_encryption 0.36 vector", async () => { - const content = new TextEncoder().encode("browser whole-file fixture\n".repeat(160)); +test("adapts native ant-core WASM output to browser file metadata", async () => { + const content = new TextEncoder().encode("browser fixture"); + const dataMap = Uint8Array.of(7, 8, 9); + const encryptedRecord = Uint8Array.of(1, 2, 3); const encrypted = await encryptPublicFile(content, { name: "fixture.txt", contentType: "text/plain", - compress, + encrypt: () => ({ + address: "44".repeat(32), + blake3: "55".repeat(32), + data_map_size: dataMap.length, + chunks: [ + { + index: 0, + dst_hash: "66".repeat(32), + src_hash: "77".repeat(32), + src_size: content.length, + }, + ], + records: [ + { address: "66".repeat(32), content: encryptedRecord }, + { address: "44".repeat(32), content: dataMap }, + ], + }), }); - assert.deepEqual( - encrypted.descriptor.chunks.map((chunk) => chunk.dst_hash), - [ - "c024c6884a2f39be7ba07c3d9636efedeb94df7397fcd38bac5ae904643c5cc9", - "350a88e6eb0b2a3e774107a212a272b4191af69ca4366a4b91f5a1e5872c459a", - "d73db5a8b0be3b571b40d2b80ff490fe45e135f1992c5863ecb78e25d00ceddb", - ], - ); - assert.equal( - encrypted.descriptor.address, - "0d3636dd504d04a236f7e104909234766f077fa7e1ca4a18293d3d168d5f169b", - ); - assert.deepEqual( - decodePublicDataMap(encrypted.records.at(-1).content), - encrypted.descriptor.chunks, - ); - - const sourceHashes = encrypted.descriptor.chunks.map((chunk) => chunk.src_hash); - const plaintext = await Promise.all( - encrypted.descriptor.chunks.map((chunk, index) => - decryptSelfEncryptedChunk( - chunk, - encrypted.records[index].content, - sourceHashes.map((hash) => Uint8Array.from(Buffer.from(hash, "hex"))), - 0, - decompress, - ), - ), - ); - const reconstructed = new Uint8Array( - plaintext.reduce((total, chunk) => total + chunk.length, 0), - ); - let offset = 0; - for (const chunk of plaintext) { - reconstructed.set(chunk, offset); - offset += chunk.length; - } - assert.deepEqual(reconstructed, content); - assert.equal(bytesToHex(blake3(reconstructed)), encrypted.descriptor.blake3); -}); - -test("public DataMap encoder matches rmp-serde's compact native representation", () => { - const chunks = [0, 1, 2].map((index) => ({ - index, - dst_hash: (11 + index).toString(16).padStart(2, "0").repeat(32), - src_hash: (21 + index).toString(16).padStart(2, "0").repeat(32), - src_size: 100 + index, - })); - const expected = - "9301939400dc00200b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0bdc00201515151515151515151515151515151515151515151515151515151515151515649401dc00200c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0cdc00201616161616161616161616161616161616161616161616161616161616161616659402dc00200d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0ddc0020171717171717171717171717171717171717171717171717171717171717171766c0"; - assert.equal(bytesToHex(encodePublicDataMap(chunks)), expected); - assert.deepEqual(decodePublicDataMap(Uint8Array.from(Buffer.from(expected, "hex"))), chunks); + assert.equal(encrypted.descriptor.name, "fixture.txt"); + assert.equal(encrypted.descriptor.address, "44".repeat(32)); + assert.equal(encrypted.descriptor.content_type, "text/plain"); + assert.equal(encrypted.descriptor.size, content.length); + assert.equal(encrypted.records.length, 2); + assert.deepEqual(encrypted.records[0].content, encryptedRecord); + assert.deepEqual(encrypted.records[1].content, dataMap); }); diff --git a/web/src/wasm.test.js b/web/src/wasm.test.js new file mode 100644 index 00000000..9e3fd206 --- /dev/null +++ b/web/src/wasm.test.js @@ -0,0 +1,50 @@ +import assert from "node:assert/strict"; +import test from "node:test"; +import { + decodePublicDataMap, + decryptPublicFile, + encryptPublicFile, + verifyRecord, +} from "../pkg/ant_core.js"; + +const EXPECTED_CHUNK_ADDRESSES = [ + "c024c6884a2f39be7ba07c3d9636efedeb94df7397fcd38bac5ae904643c5cc9", + "350a88e6eb0b2a3e774107a212a272b4191af69ca4366a4b91f5a1e5872c459a", + "d73db5a8b0be3b571b40d2b80ff490fe45e135f1992c5863ecb78e25d00ceddb", +]; + +test("generated ant-core WASM matches the native self-encryption vector", () => { + const content = new TextEncoder().encode("browser whole-file fixture\n".repeat(160)); + const encrypted = encryptPublicFile(content); + + assert.equal( + encrypted.address, + "0d3636dd504d04a236f7e104909234766f077fa7e1ca4a18293d3d168d5f169b", + ); + assert.equal( + encrypted.blake3, + "e0e422267ac59c56bf032d6d830035d343369d20147dd5f6b63351a29b015f22", + ); + assert.deepEqual( + encrypted.chunks.map((chunk) => chunk.dst_hash), + EXPECTED_CHUNK_ADDRESSES, + ); + assert.equal(encrypted.records.length, 4); + + for (const record of encrypted.records) { + assert(record.content instanceof Uint8Array); + assert.equal(verifyRecord(record.address, record.content), record.address); + } + + const dataMap = encrypted.records.at(-1).content; + const chunks = encrypted.records.slice(0, -1).map((record) => record.content); + assert.deepEqual( + decodePublicDataMap(dataMap).map((chunk) => chunk.dst_hash), + EXPECTED_CHUNK_ADDRESSES, + ); + assert.deepEqual(decryptPublicFile(dataMap, chunks), content); + + const tampered = chunks.map((chunk) => chunk.slice()); + tampered[0][0] ^= 1; + assert.throws(() => decryptPublicFile(dataMap, tampered), /BLAKE3 mismatch/); +}); From ca21f0d53d8d0dc4a1cbab179d30981b1f3b719c Mon Sep 17 00:00:00 2001 From: Mick van Dijke <12992260+mickvandijke@users.noreply.github.com> Date: Wed, 5 Aug 2026 18:00:03 +0200 Subject: [PATCH 05/31] feat(web): use shared Rust DHT lookup runner --- Cargo.lock | 12 +- Cargo.toml | 4 + README.md | 6 +- ant-core/Cargo.toml | 3 + ant-core/src/browser.rs | 274 +++++++++++++++++- .../ADR-0003-direct-browser-read-client.md | 23 +- web/README.md | 29 +- web/src/protocol.js | 83 ++---- web/src/protocol.test.js | 7 - web/src/wasm.test.js | 51 ++++ 10 files changed, 399 insertions(+), 93 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 35e81e4e..2a43938a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -861,6 +861,7 @@ dependencies = [ "rand 0.8.6", "reqwest 0.12.28", "rmp-serde", + "saorsa-dht-lookup", "self-replace", "self_encryption", "semver 1.0.28", @@ -882,6 +883,7 @@ dependencies = [ "tracing-subscriber", "utoipa", "wasm-bindgen", + "wasm-bindgen-futures", "windows-sys 0.61.2", "xor_name", "zip", @@ -5215,8 +5217,6 @@ dependencies = [ [[package]] name = "saorsa-core" version = "0.26.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "454529f8a72b4cf22f7d9c3b009ad9d4ba78520e11f6444ae460795d55c002da" dependencies = [ "anyhow", "async-trait", @@ -5231,6 +5231,7 @@ dependencies = [ "parking_lot", "postcard", "rand 0.8.6", + "saorsa-dht-lookup", "saorsa-pqc 0.5.1", "saorsa-transport", "serde", @@ -5244,6 +5245,10 @@ dependencies = [ "wyz", ] +[[package]] +name = "saorsa-dht-lookup" +version = "0.1.0" + [[package]] name = "saorsa-pqc" version = "0.4.2" @@ -5330,12 +5335,11 @@ dependencies = [ [[package]] name = "saorsa-transport" version = "0.35.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e3284026c300f642077315b782462b22558e24d621265a8deeae23287b1c5542" dependencies = [ "anyhow", "async-trait", "aws-lc-rs", + "base64", "blake3", "bytes", "chrono", diff --git a/Cargo.toml b/Cargo.toml index f3978b22..07d8c6e7 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,3 +1,7 @@ [workspace] members = ["ant-core", "ant-cli"] resolver = "2" + +[patch.crates-io] +saorsa-core = { path = "../saorsa-core-web-support" } +saorsa-transport = { path = "../saorsa-transport-web-support" } diff --git a/README.md b/README.md index 8ffc044f..e48d81ad 100644 --- a/README.md +++ b/README.md @@ -8,9 +8,9 @@ This project provides two Rust crates and a browser client: - **ant-core** — A headless Rust library containing all business logic: data storage/retrieval with self-encryption and EVM payments, node lifecycle management, and local devnet tooling. Its portable immutable-data core also builds for browsers with the `browser-wasm` feature. - **ant-cli** — A thin CLI binary (`ant`) built on `ant-core`. -- **web** — A direct WebTransport client and test site. It performs browser-side - closest-node lookup and uses `ant-core` through WASM to self-encrypt and - reconstruct complete public files without a data gateway. +- **web** — A direct WebTransport client and test site. It drives Saorsa's + shared iterative lookup engine and uses `ant-core` through WASM to + self-encrypt and reconstruct complete public files without a data gateway. Data on Autonomi is **content-addressed**. Files are split into encrypted chunks (via [self-encryption](https://en.wikipedia.org/wiki/Convergent_encryption)), each stored at an XOR address derived from its content. A `DataMap` tracks which chunks belong to a file. Payments for storage are made on an EVM-compatible blockchain (Arbitrum). diff --git a/ant-core/Cargo.toml b/ant-core/Cargo.toml index 11c3c93e..7ebfd32a 100644 --- a/ant-core/Cargo.toml +++ b/ant-core/Cargo.toml @@ -21,6 +21,7 @@ rmp-serde = "1" serde = { version = "1", features = ["derive"] } serde_bytes = "0.11" self_encryption = "0.36" +saorsa-dht-lookup = { version = "0.1.0", path = "../../saorsa-core-web-support/crates/saorsa-dht-lookup" } thiserror = "2" # Node management @@ -81,6 +82,7 @@ tracing-subscriber = { version = "0.3", features = ["env-filter"], optional = tr js-sys = { version = "0.3", optional = true } serde-wasm-bindgen = { version = "0.6", optional = true } wasm-bindgen = { version = "0.2", optional = true } +wasm-bindgen-futures = { version = "0.4", optional = true } [target.'cfg(target_arch = "wasm32")'.dependencies] # self_encryption/rand use getrandom 0.2. Browser entropy is supplied by the @@ -136,6 +138,7 @@ browser-wasm = [ "dep:js-sys", "dep:serde-wasm-bindgen", "dep:wasm-bindgen", + "dep:wasm-bindgen-futures", ] # Enable `LocalDevnet` (ant-core/src/node/devnet.rs) which wraps # `ant_node::devnet::Devnet` and an Anvil EVM testnet. diff --git a/ant-core/src/browser.rs b/ant-core/src/browser.rs index 9efe5dfe..0e9f1ed8 100644 --- a/ant-core/src/browser.rs +++ b/ant-core/src/browser.rs @@ -213,8 +213,280 @@ fn chunk_infos(data_map: &DataMap) -> Vec { #[cfg(all(target_arch = "wasm32", feature = "browser-wasm"))] mod wasm { use super::{content_address, decrypt_public_file, encrypt_public_file, verify_record}; - use js_sys::{Array, Uint8Array}; + use js_sys::{Array, Function, Promise, Uint8Array}; + use saorsa_dht_lookup::{ + run_iterative_lookup, IterativeLookup, LookupConfig, LookupKey, LookupNode, LookupQuery, + LookupQueryOutcome, + }; + use serde::{Deserialize, Serialize}; + use std::collections::HashMap; use wasm_bindgen::prelude::*; + use wasm_bindgen_futures::JsFuture; + + #[derive(Debug, Clone, Serialize, Deserialize)] + #[serde(untagged)] + enum BrowserLookupEndpoint { + Structured { multiaddr: String }, + Multiaddr(String), + } + + #[derive(Debug, Clone, Serialize, Deserialize)] + struct BrowserLookupNode { + peer_id: String, + #[serde(default)] + native_addresses: Vec, + #[serde(default)] + reliability: f64, + #[serde(default)] + webtransport: Option, + } + + #[derive(Debug, Serialize)] + struct BrowserLookupBatch { + target: String, + count: usize, + iteration: usize, + candidates: Vec, + } + + #[derive(Debug, Deserialize)] + #[serde(tag = "status", rename_all = "snake_case")] + enum BrowserLookupQueryOutcome { + Succeeded { + responder: String, + #[serde(default)] + candidates: Vec, + }, + Failed { + responder: String, + }, + Unresponsive { + responder: String, + }, + } + + #[derive(Debug, Clone)] + struct BrowserLookupCandidate { + peer_id: LookupKey, + wire: BrowserLookupNode, + } + + impl LookupNode for BrowserLookupCandidate { + fn lookup_peer_id(&self) -> LookupKey { + self.peer_id + } + } + + impl BrowserLookupCandidate { + fn parse(mut wire: BrowserLookupNode) -> Result { + let peer_id = parse_lookup_key(&wire.peer_id, "peer ID")?; + wire.peer_id = hex::encode(peer_id); + Ok(Self { peer_id, wire }) + } + } + + /// Shared Saorsa iterative lookup state driven by browser WebTransport. + #[wasm_bindgen(js_name = BrowserIterativeLookup)] + pub struct BrowserIterativeLookup { + lookup: IterativeLookup, + known_endpoints: HashMap, + } + + #[wasm_bindgen(js_class = BrowserIterativeLookup)] + impl BrowserIterativeLookup { + /// Construct a browser lookup using the same scheduler as native QUIC. + #[wasm_bindgen(constructor)] + pub fn new( + target: &str, + count: usize, + alpha: usize, + max_iterations: usize, + ) -> Result { + let target = parse_lookup_key(target, "lookup target")?; + let config = LookupConfig { + count, + alpha, + max_iterations, + ..LookupConfig::saorsa(count) + }; + let lookup = IterativeLookup::new(target, config) + .map_err(|error| JsValue::from_str(&error.to_string()))?; + Ok(Self { + lookup, + known_endpoints: HashMap::new(), + }) + } + + /// Add validated bootstrap or FIND_NODE candidates. + #[wasm_bindgen(js_name = addCandidates)] + pub fn add_candidates(&mut self, nodes: JsValue) -> Result<(), JsValue> { + for candidate in parse_lookup_nodes(nodes)? { + self.add_candidate(candidate); + } + Ok(()) + } + + /// Run the complete shared Saorsa walk through a WebTransport batch callback. + #[wasm_bindgen(js_name = run)] + pub async fn run(&mut self, query_batch: Function) -> Result { + let mut query = BrowserLookupQuery { + callback: query_batch, + known_endpoints: &mut self.known_endpoints, + }; + run_iterative_lookup(&mut self.lookup, &mut query) + .await + .map(|termination| format!("{termination:?}")) + .map_err(|error| JsValue::from_str(&error.to_string())) + } + + /// Successful responders in final closest-first order. + #[wasm_bindgen(js_name = results)] + pub fn results(&self) -> Result { + let nodes = self + .lookup + .results() + .into_iter() + .map(|candidate| candidate.wire) + .collect::>(); + serde_wasm_bindgen::to_value(&nodes) + .map_err(|error| JsValue::from_str(&error.to_string())) + } + + /// Peer IDs selected for network queries, in query order. + #[wasm_bindgen(js_name = queriedPeers)] + pub fn queried_peers(&self) -> Result { + let peers = self + .lookup + .queried_peers() + .iter() + .map(hex::encode) + .collect::>(); + serde_wasm_bindgen::to_value(&peers) + .map_err(|error| JsValue::from_str(&error.to_string())) + } + } + + impl BrowserIterativeLookup { + fn add_candidate(&mut self, candidate: BrowserLookupCandidate) { + if let Some(candidate) = + resolve_candidate_endpoint(&mut self.known_endpoints, candidate) + { + let _ = self.lookup.add_candidate(candidate); + } + } + } + + struct BrowserLookupQuery<'a> { + callback: Function, + known_endpoints: &'a mut HashMap, + } + + impl LookupQuery for BrowserLookupQuery<'_> { + type Error = String; + + async fn query_batch( + &mut self, + target: LookupKey, + count: usize, + iteration: usize, + batch: Vec, + ) -> Result>, Self::Error> { + let request = BrowserLookupBatch { + target: hex::encode(target), + count, + iteration, + candidates: batch.into_iter().map(|candidate| candidate.wire).collect(), + }; + let request = serde_wasm_bindgen::to_value(&request) + .map_err(|error| format!("could not encode lookup batch: {error}"))?; + let returned = self + .callback + .call1(&JsValue::NULL, &request) + .map_err(js_error_message)?; + let returned = JsFuture::from(Promise::resolve(&returned)) + .await + .map_err(js_error_message)?; + let outcomes: Vec = serde_wasm_bindgen::from_value(returned) + .map_err(|error| format!("invalid lookup batch response: {error}"))?; + + outcomes + .into_iter() + .map(|outcome| match outcome { + BrowserLookupQueryOutcome::Succeeded { + responder, + candidates, + } => { + let responder = parse_lookup_key(&responder, "lookup responder") + .map_err(js_error_message)?; + let candidates = candidates + .into_iter() + .map(BrowserLookupCandidate::parse) + .collect::, _>>() + .map_err(js_error_message)? + .into_iter() + .filter_map(|candidate| { + resolve_candidate_endpoint(self.known_endpoints, candidate) + }) + .collect(); + Ok(LookupQueryOutcome::Succeeded { + responder, + candidates, + }) + } + BrowserLookupQueryOutcome::Failed { responder } => { + parse_lookup_key(&responder, "lookup responder") + .map(|responder| LookupQueryOutcome::Failed { responder }) + .map_err(js_error_message) + } + BrowserLookupQueryOutcome::Unresponsive { responder } => { + parse_lookup_key(&responder, "lookup responder") + .map(|responder| LookupQueryOutcome::Unresponsive { responder }) + .map_err(js_error_message) + } + }) + .collect() + } + } + + fn resolve_candidate_endpoint( + known_endpoints: &mut HashMap, + mut candidate: BrowserLookupCandidate, + ) -> Option { + if let Some(endpoint) = candidate.wire.webtransport.clone() { + known_endpoints.insert(candidate.peer_id, endpoint); + } else if let Some(endpoint) = known_endpoints.get(&candidate.peer_id) { + candidate.wire.webtransport = Some(endpoint.clone()); + } + candidate.wire.webtransport.as_ref()?; + Some(candidate) + } + + fn js_error_message(value: JsValue) -> String { + value + .as_string() + .unwrap_or_else(|| format!("JavaScript lookup callback failed: {value:?}")) + } + + fn parse_lookup_nodes(value: JsValue) -> Result, JsValue> { + let nodes: Vec = serde_wasm_bindgen::from_value(value) + .map_err(|error| JsValue::from_str(&format!("invalid lookup nodes: {error}")))?; + nodes + .into_iter() + .map(BrowserLookupCandidate::parse) + .collect() + } + + fn parse_lookup_key(value: &str, label: &str) -> Result { + let value = value.strip_prefix("0x").unwrap_or(value); + let bytes = hex::decode(value) + .map_err(|error| JsValue::from_str(&format!("invalid {label}: {error}")))?; + bytes.try_into().map_err(|bytes: Vec| { + JsValue::from_str(&format!( + "invalid {label}: expected 32 bytes, received {}", + bytes.len() + )) + }) + } /// Install a readable panic hook for browser developer tools. #[wasm_bindgen(start)] diff --git a/docs/adr/ADR-0003-direct-browser-read-client.md b/docs/adr/ADR-0003-direct-browser-read-client.md index b9410205..0d7206a4 100644 --- a/docs/adr/ADR-0003-direct-browser-read-client.md +++ b/docs/adr/ADR-0003-direct-browser-read-client.md @@ -57,6 +57,8 @@ exports browser-safe immutable-data operations through `wasm-bindgen`. The Rust/WASM core will: +- run iterative closest-node lookup through Saorsa's transport-independent + `run_iterative_lookup` driver and `LookupQuery` interface; - self-encrypt complete public files with the same `self_encryption 0.36` implementation used by the Rust client; - encode and decode the native MessagePack `DataMap` representation; @@ -72,8 +74,8 @@ The `web/` package will remain responsible for browser-specific orchestration: `/certhash` SHA-256 multihashes internally, and pass them to WebTransport; - require `/p2p/` in every address, verify its `HELLO` peer ID, and reject discovered endpoint/peer mismatches; -- perform iterative `FIND_NODE` queries using 256-bit XOR ordering, `K = 20`, - and `ALPHA = 3`; +- execute each WebTransport `FIND_NODE` request selected by the Rust/WASM + lookup engine and return its response or failure to that engine; - query closest direct endpoints with `GET_CHUNK`, retrying `not_found` and unavailable nodes without routing bytes through the manifest service; - call the Rust/WASM core to self-encrypt selected public files and generate @@ -120,6 +122,14 @@ and ML-DSA verification types without enabling native networking. At that point those compatibility-sensitive operations should also move behind the Rust/WASM boundary. +The shared lookup engine is intentionally a small Saorsa crate rather than a +second implementation inside `ant-core`. Native `DhtNetworkManager` and the +browser WASM adapter both drive the same candidate queue, peer-state, α-batch, +and convergence implementation. Native QUIC retains transport authentication, +address-report consensus, failure-cache integration, and trust updates; the +browser adapter retains WebTransport session establishment and endpoint +validation. + For compatibility with the local launcher, the bootstrap manifest still carries a resolved JSON view of the public root DataMap alongside its ordinary on-network DataMap address. The download path does not use that copy to select @@ -147,8 +157,8 @@ metadata chain. - The current client reconstructs files in memory and the local launcher caps public files at 64 MiB; upload encryption and reconstruction are not yet streaming. -- JavaScript lookup and quote-verification behavior must remain aligned with - native Kademlia and protocol rules until transport-free Rust APIs exist. +- JavaScript quote-verification behavior must remain aligned with native + protocol rules until transport-free `ant-protocol` APIs exist. - The initial WASM module is approximately 1.4 MiB uncompressed and browser file processing is still in memory. - Certificate and endpoint verification adds bootstrap-record lifecycle work. @@ -164,7 +174,10 @@ metadata chain. - Rust unit tests cover an exact native `self_encryption 0.36` wire vector, public DataMap generation, round-trip reconstruction, and tamper rejection. -- JavaScript unit tests cover fixed-width identifiers, XOR ordering, +- Saorsa engine tests cover XOR ordering, α limits, final peer states, bounded + candidate eviction, multi-round discovery, and top-K convergence; the native + DHT manager and browser WASM adapter both invoke the same generic runner. +- JavaScript unit tests cover fixed-width identifiers, bidirectional binary framing, manifest/payment validation, quote signatures, the native Keccak-256 EVM quote-hash vector, and the browser orchestration around the Rust/WASM boundary. diff --git a/web/README.md b/web/README.md index 0964bbd4..8f156ea3 100644 --- a/web/README.md +++ b/web/README.md @@ -2,11 +2,12 @@ This web application is the browser-facing client for ADR-0009. It loads a local testnet bootstrap manifest, connects directly to storage nodes over -WebTransport, and performs the XOR closest-node lookup in the browser. The -portable part of the Rust `ant-core` library is compiled to WASM and performs -native self-encryption, public DataMap serialization, reconstruction, and -BLAKE3 content verification. The thin JavaScript layer retrieves and uploads -records, uses the browser wallet/payment APIs, and drives the page. +WebTransport, and performs closest-node lookup through Saorsa's shared Rust +lookup engine. The portable part of `ant-core` is compiled to WASM and drives +the Kademlia walk, native self-encryption, public DataMap serialization, +reconstruction, and BLAKE3 content verification. The thin JavaScript layer +executes WebTransport requests, retrieves and uploads records, uses the browser +wallet/payment APIs, and drives the page. The node-side WebTransport listener and testnet manifest API live in the `ant-node-web-support` sibling repository. No HTTP gateway performs lookup or @@ -88,7 +89,8 @@ manifest from port 25000 and fills in the default file: 1. **Load testnet** refreshes the manifest and direct multiaddress catalog. 2. **Connect** parses the first node multiaddress and performs a pinned WebTransport `HELLO`. -3. **Find closest** runs the iterative lookup in the browser. +3. **Find closest** runs Saorsa's iterative lookup engine in WASM, with + JavaScript executing its WebTransport query batches. 4. Under **Paid public file upload**, choose a file, paste the funded private key printed by ant-devnet, then select **Pay and upload file**. Rust/WASM performs encryption and DataMap generation; quote/commitment verification, @@ -124,10 +126,11 @@ cargo check -p ant-core --target wasm32-unknown-unknown \ cargo test -p ant-core --lib browser::tests ``` -The tests cover fixed-width IDs, XOR ordering, bidirectional binary framing, -browser manifest/payment validation, signed quote verification including the -native Keccak-256 EVM quote hash, and a Rust `self_encryption 0.36` wire vector -with native/WASM round-trip and tamper verification. +The tests cover Saorsa's shared lookup engine and generic query driver, fixed-width IDs, +bidirectional binary framing, browser manifest/payment validation, signed +quote verification including the native Keccak-256 EVM quote hash, and a Rust +`self_encryption 0.36` wire vector with native/WASM round-trip and tamper +verification. Cross-repository live verification additionally starts the node testnet, downloads all public-file records through WebTransport, reconstructs the original bytes, pays a real quote on local Anvil, uploads a fresh record @@ -135,8 +138,10 @@ through the ordinary node payment validator, and reads it back. ## Current boundary -JavaScript currently owns WebTransport stream handling, iterative lookup, -Ethers payment submission, ML-DSA quote verification, and DOM/save APIs. +JavaScript currently owns WebTransport stream handling, Ethers payment +submission, ML-DSA quote verification, and DOM/save APIs. Rust/WASM owns the +complete iterative lookup loop; JavaScript implements only its transport batch +callback. `ant-protocol 2.3.1` still enables native Saorsa/Tokio networking and cannot be linked into a browser WASM build; a future transport-free feature can move the remaining quote and multiaddress verification into Rust. diff --git a/web/src/protocol.js b/web/src/protocol.js index 153ce6a0..af8b2205 100644 --- a/web/src/protocol.js +++ b/web/src/protocol.js @@ -1,5 +1,8 @@ import { bytesToHex as nobleBytesToHex } from "@noble/hashes/utils.js"; -import { verifyRecord as verifyRecordNative } from "../pkg/ant_core.js"; +import { + BrowserIterativeLookup as BrowserIterativeLookupNative, + verifyRecord as verifyRecordNative, +} from "../pkg/ant_core.js"; export const PROTOCOL_VERSION = 3; export const PROTOCOL_NAME = "autonomi.web.poc.v3"; @@ -34,16 +37,6 @@ export function bytesToHex(bytes) { return nobleBytesToHex(bytes); } -export function xorDistance(peerId, target) { - const peer = hexToBytes(peerId, 32); - const key = hexToBytes(target, 32); - let distance = 0n; - for (let index = 0; index < peer.length; index += 1) { - distance = (distance << 8n) | BigInt(peer[index] ^ key[index]); - } - return distance; -} - export function verifyChunk(address, content) { const expected = address.trim().replace(/^0x/i, "").toLowerCase(); hexToBytes(expected, 32); @@ -455,9 +448,8 @@ export async function iterativeFindClosest( } const clients = new Map(); - const known = new Map(); - const queried = new Set(); const failures = []; + const seedNodes = []; const clientFor = (endpoint) => { const key = endpointKey(endpoint); @@ -476,7 +468,7 @@ export async function iterativeFindClosest( try { const client = clientFor(endpoint); const hello = await client.hello(); - known.set(hello.peer_id, { + seedNodes.push({ peer_id: hello.peer_id, native_addresses: [], reliability: 1, @@ -489,70 +481,39 @@ export async function iterativeFindClosest( } }), ); - if (known.size === 0) { + if (seedNodes.length === 0) { for (const client of clients.values()) client.close(); const detail = failures.map(({ error }) => error.message).join("; "); throw new Error(`Could not connect to any WebTransport seed: ${detail}`); } - for (let iteration = 0; iteration < maxIterations; iteration += 1) { - const ordered = [...known.values()].sort((left, right) => { - const a = xorDistance(left.peer_id, target); - const b = xorDistance(right.peer_id, target); - return a < b ? -1 : a > b ? 1 : 0; - }); - const candidates = ordered - .filter((node) => node.webtransport && !queried.has(node.peer_id)) - .slice(0, alpha); - if (candidates.length === 0) break; - - let discovered = 0; - await Promise.all( + const lookup = new BrowserIterativeLookupNative(target, k, alpha, maxIterations); + lookup.addCandidates(seedNodes); + await lookup.run(async ({ target: lookupTarget, count, iteration, candidates }) => + Promise.all( candidates.map(async (candidate) => { - queried.add(candidate.peer_id); try { const candidateClient = clientFor(candidate.webtransport); if (!candidateClient.peerId) await candidateClient.hello(); - const nodes = await candidateClient.findNode(target, k); + const nodes = await candidateClient.findNode(lookupTarget, count); onProgress( - `Iteration ${iteration + 1}: ${candidate.peer_id} returned ${nodes.length} nodes`, + `Iteration ${iteration}: ${candidate.peer_id} returned ${nodes.length} nodes`, ); - for (const node of nodes) { - const existing = known.get(node.peer_id); - if (!existing) discovered += 1; - known.set(node.peer_id, { - ...existing, - ...node, - webtransport: node.webtransport ?? existing?.webtransport, - }); - } + return { + status: "succeeded", + responder: candidate.peer_id, + candidates: nodes, + }; } catch (error) { failures.push({ peerId: candidate.peer_id, error }); onProgress(`Query ${candidate.peer_id} failed: ${error.message}`); + return { status: "failed", responder: candidate.peer_id }; } }), - ); + ), + ); - const remainingQueryable = [...known.values()] - .sort((left, right) => { - const a = xorDistance(left.peer_id, target); - const b = xorDistance(right.peer_id, target); - return a < b ? -1 : a > b ? 1 : 0; - }) - .slice(0, k) - .some((node) => node.webtransport && !queried.has(node.peer_id)); - if (discovered === 0 && !remainingQueryable) break; - } - - const nodes = [...known.values()] - .sort((left, right) => { - const a = xorDistance(left.peer_id, target); - const b = xorDistance(right.peer_id, target); - return a < b ? -1 : a > b ? 1 : 0; - }) - .slice(0, k); - - return { nodes, queried: [...queried], failures, clients }; + return { nodes: lookup.results(), queried: lookup.queriedPeers(), failures, clients }; } export async function getChunkFromClosest( diff --git a/web/src/protocol.test.js b/web/src/protocol.test.js index 833ca2a8..5959317d 100644 --- a/web/src/protocol.test.js +++ b/web/src/protocol.test.js @@ -9,7 +9,6 @@ import { parseResponseFrame, parseWebTransportMultiaddr, verifyChunk, - xorDistance, } from "./protocol.js"; test("hex conversion enforces fixed widths", () => { @@ -19,12 +18,6 @@ test("hex conversion enforces fixed widths", () => { assert.throws(() => hexToBytes("zz", 1), /hexadecimal/); }); -test("XOR distance is an unsigned 256-bit ordering value", () => { - assert.equal(xorDistance("00".repeat(32), "00".repeat(32)), 0n); - assert.equal(xorDistance("00".repeat(31) + "01", "00".repeat(32)), 1n); - assert(xorDistance("80" + "00".repeat(31), "00".repeat(32)) > 1n); -}); - test("response framing preserves a raw binary body", () => { const header = new TextEncoder().encode( JSON.stringify({ diff --git a/web/src/wasm.test.js b/web/src/wasm.test.js index 9e3fd206..8f376039 100644 --- a/web/src/wasm.test.js +++ b/web/src/wasm.test.js @@ -1,6 +1,7 @@ import assert from "node:assert/strict"; import test from "node:test"; import { + BrowserIterativeLookup, decodePublicDataMap, decryptPublicFile, encryptPublicFile, @@ -13,6 +14,56 @@ const EXPECTED_CHUNK_ADDRESSES = [ "d73db5a8b0be3b571b40d2b80ff490fe45e135f1992c5863ecb78e25d00ceddb", ]; +function lookupNode(lastByte, stringEndpoint = false) { + return { + peer_id: `${"00".repeat(31)}${lastByte.toString(16).padStart(2, "0")}`, + native_addresses: [], + reliability: 1, + webtransport: stringEndpoint ? `/test/${lastByte}` : { multiaddr: `/test/${lastByte}` }, + }; +} + +test("generated WASM drives Saorsa's complete shared iterative lookup", async () => { + const lookup = new BrowserIterativeLookup("00".repeat(32), 2, 2, 20); + lookup.addCandidates([lookupNode(3), lookupNode(1), lookupNode(2, true)]); + const batches = []; + const termination = await lookup.run(async ({ iteration, candidates }) => { + batches.push(candidates.map((node) => node.peer_id)); + if (iteration === 1) { + assert.equal(candidates[1].webtransport, "/test/2"); + return [ + { + status: "succeeded", + responder: candidates[0].peer_id, + candidates: [lookupNode(0)], + }, + { status: "failed", responder: candidates[1].peer_id }, + ]; + } + return candidates.map((candidate) => ({ + status: "succeeded", + responder: candidate.peer_id, + candidates: [], + })); + }); + + assert.equal(termination, "Exhausted"); + assert.deepEqual(batches, [ + [lookupNode(1).peer_id, lookupNode(2).peer_id], + [lookupNode(0).peer_id, lookupNode(3).peer_id], + ]); + assert.deepEqual( + lookup.results().map((node) => node.peer_id), + [lookupNode(0).peer_id, lookupNode(1).peer_id], + ); + assert.deepEqual(lookup.queriedPeers(), [ + lookupNode(1).peer_id, + lookupNode(2).peer_id, + lookupNode(0).peer_id, + lookupNode(3).peer_id, + ]); +}); + test("generated ant-core WASM matches the native self-encryption vector", () => { const content = new TextEncoder().encode("browser whole-file fixture\n".repeat(160)); const encrypted = encryptPublicFile(content); From b01971ea1cf01d848f6862ed001234a995085c82 Mon Sep 17 00:00:00 2001 From: Mick van Dijke <12992260+mickvandijke@users.noreply.github.com> Date: Wed, 26 Aug 2026 12:39:09 +0200 Subject: [PATCH 06/31] feat(web): use Saorsa WebRTC Direct transport --- README.md | 6 +- ant-core/src/browser.rs | 12 +- .../ADR-0003-direct-browser-read-client.md | 77 +- web/README.md | 41 +- web/index.html | 7 +- web/package-lock.json | 139 +++ web/src/file.js | 86 +- web/src/main.js | 32 +- web/src/manifest.js | 8 +- web/src/manifest.test.js | 24 +- web/src/payment.js | 2 +- web/src/payment.test.js | 2 +- web/src/protocol.js | 877 +++++++++++++----- web/src/protocol.test.js | 461 ++++----- web/src/upload.js | 224 +++-- web/src/wasm.test.js | 4 +- 16 files changed, 1352 insertions(+), 650 deletions(-) diff --git a/README.md b/README.md index e48d81ad..75bac8fc 100644 --- a/README.md +++ b/README.md @@ -8,7 +8,7 @@ This project provides two Rust crates and a browser client: - **ant-core** — A headless Rust library containing all business logic: data storage/retrieval with self-encryption and EVM payments, node lifecycle management, and local devnet tooling. Its portable immutable-data core also builds for browsers with the `browser-wasm` feature. - **ant-cli** — A thin CLI binary (`ant`) built on `ant-core`. -- **web** — A direct WebTransport client and test site. It drives Saorsa's +- **web** — A direct WebRTC Direct client and test site. It drives Saorsa's shared iterative lookup engine and uses `ant-core` through WASM to self-encrypt and reconstruct complete public files without a data gateway. @@ -21,9 +21,9 @@ the browser-enabled node devnet, then run the site: ```bash # In ant-node-web-support -cargo run --features webtransport-poc --bin ant-devnet -- \ +cargo run --features webrtc-direct --bin ant-devnet -- \ --preset minimal --base-port 23000 \ - --webtransport --webtransport-base-port 24000 \ + --webrtc-direct --webrtc-direct-base-port 24000 \ --serve-port 25000 --enable-evm --enable-logging # In ant-client-web-support/web diff --git a/ant-core/src/browser.rs b/ant-core/src/browser.rs index 0e9f1ed8..899b7b53 100644 --- a/ant-core/src/browser.rs +++ b/ant-core/src/browser.rs @@ -238,7 +238,7 @@ mod wasm { #[serde(default)] reliability: f64, #[serde(default)] - webtransport: Option, + webrtc_direct: Option, } #[derive(Debug, Serialize)] @@ -285,7 +285,7 @@ mod wasm { } } - /// Shared Saorsa iterative lookup state driven by browser WebTransport. + /// Shared Saorsa iterative lookup state driven by browser WebRtcDirect. #[wasm_bindgen(js_name = BrowserIterativeLookup)] pub struct BrowserIterativeLookup { lookup: IterativeLookup, @@ -326,7 +326,7 @@ mod wasm { Ok(()) } - /// Run the complete shared Saorsa walk through a WebTransport batch callback. + /// Run the complete shared Saorsa walk through a WebRtcDirect batch callback. #[wasm_bindgen(js_name = run)] pub async fn run(&mut self, query_batch: Function) -> Result { let mut query = BrowserLookupQuery { @@ -452,12 +452,12 @@ mod wasm { known_endpoints: &mut HashMap, mut candidate: BrowserLookupCandidate, ) -> Option { - if let Some(endpoint) = candidate.wire.webtransport.clone() { + if let Some(endpoint) = candidate.wire.webrtc_direct.clone() { known_endpoints.insert(candidate.peer_id, endpoint); } else if let Some(endpoint) = known_endpoints.get(&candidate.peer_id) { - candidate.wire.webtransport = Some(endpoint.clone()); + candidate.wire.webrtc_direct = Some(endpoint.clone()); } - candidate.wire.webtransport.as_ref()?; + candidate.wire.webrtc_direct.as_ref()?; Some(candidate) } diff --git a/docs/adr/ADR-0003-direct-browser-read-client.md b/docs/adr/ADR-0003-direct-browser-read-client.md index 0d7206a4..32e2bceb 100644 --- a/docs/adr/ADR-0003-direct-browser-read-client.md +++ b/docs/adr/ADR-0003-direct-browser-read-client.md @@ -1,13 +1,13 @@ -# ADR-0003: Direct browser immutable-data client over WebTransport +# ADR-0003: Direct browser immutable-data client over WebRTC Direct - **Status:** Proposed - **Date:** 2026-08-03 -- **Last amended:** 2026-08-05 +- **Last amended:** 2026-08-25 - **Decision owners:** - **Reviewers:** - **Supersedes:** none - **Superseded by:** none -- **Related:** ant-node ADR-0009; W3C WebTransport +- **Related:** ant-node ADR-0009; Saorsa WebRTC Direct transport ## Context @@ -15,8 +15,9 @@ The Autonomi web client must perform closest-node lookup, immutable-data download, quote verification, payment, and upload itself. Sending those operations through an HTTP application gateway would make the gateway an availability, privacy, and bandwidth chokepoint. -Browsers cannot use the native Saorsa QUIC protocol, but they can establish -WebTransport sessions with browser-compatible node listeners. +Browsers cannot use the native Saorsa QUIC protocol, but they can dial +Saorsa-owned WebRTC Direct listeners over ICE-lite, DTLS, SCTP, and +DataChannels without a signaling server or a libp2p layer. This repository owns the client and UI side of that split. Nodes own transport termination, local DHT answers, storage reads and paid writes, endpoint @@ -26,8 +27,12 @@ records, and testnet bootstrap-manifest production under ant-node ADR-0009. - File bytes must travel directly from a storage node to the browser. - The browser must own iterative XOR lookup rather than ask a gateway to do it. -- Self-signed node certificates must be authenticated through hashes embedded - in self-contained node multiaddresses, without a separate client argument. +- A bootstrap list must remain usable for months: each node therefore persists + its self-signed DTLS certificate and embeds one stable fingerprint in its + self-contained multiaddress. +- Bootstrap endpoints must use literal IP addresses. Running a node must not + require DNS, a public CA certificate, an HTTPS endpoint, or a signaling + service. - Downloaded immutable content must be verified before it is exposed to users. - The wallet secret must be provided at runtime, used only by the local EVM signer, and never sent to a node or persisted in bootstrap metadata. @@ -68,13 +73,20 @@ The Rust/WASM core will: The `web/` package will remain responsible for browser-specific orchestration: -- load a versioned browser bootstrap manifest containing WebTransport +- load a versioned browser bootstrap manifest containing WebRTC Direct multiaddresses and published immutable-file metadata; -- accept a single multiaddress per seed, extract its one or two - `/certhash` SHA-256 multihashes internally, and pass them to WebTransport; -- require `/p2p/` in every address, verify its `HELLO` peer ID, and - reject discovered endpoint/peer mismatches; -- execute each WebTransport `FIND_NODE` request selected by the Rust/WASM +- accept one canonical multiaddress per seed, extract its single `/certhash` + SHA-256 multihash internally, and construct a native browser + `RTCPeerConnection` description from its literal IP and UDP port; +- open one persistent reliable ordered `RTCDataChannel` per association and + exchange bounded, declared-length request/response frames directly, without + Noise, multistream negotiation, or libp2p stream envelopes; +- reuse authenticated associations through a bounded client pool for the + duration of a complete upload or download instead of opening a new peer + connection for every encrypted record; +- require `/p2p/` in every address, verify the challenge-bound ML-DSA + `HELLO` signature, and reject certificate, endpoint, or peer mismatches; +- execute each WebRTC Direct `FIND_NODE` request selected by the Rust/WASM lookup engine and return its response or failure to that engine; - query closest direct endpoints with `GET_CHUNK`, retrying `not_found` and unavailable nodes without routing bytes through the manifest service; @@ -86,7 +98,7 @@ The `web/` package will remain responsible for browser-specific orchestration: public vault when required, and make one batched `payForQuotes` transaction; - upload each content-addressed encrypted record with the signed quote and transaction hash through paid `PUT_CHUNK`; the wallet key never crosses the - WebTransport session; + WebRTC Direct connection; - fetch the public MessagePack DataMap and every resolved encrypted data chunk, then pass those records to the Rust/WASM reconstruction API; - verify the final file size and use the Rust/WASM BLAKE3 verifier before @@ -101,17 +113,17 @@ defined by ant-node ADR-0009. The JavaScript API and demo never accept a certificate hash separately from an endpoint. Their sole dialing input is a canonical address of the form -`/.../quic-v1/webtransport/certhash//p2p/`. Repeated -`/certhash` components permit current/next certificate overlap. Keeping the -transport location, accepted TLS key, and expected peer identity in one value -prevents callers from accidentally combining fields belonging to different -nodes. +`/ip4|ip6//udp//webrtc-direct/certhash//p2p/`. +DNS protocols, port zero, and repeated `/certhash` components are rejected. +Keeping the transport location, accepted DTLS key, and expected ANT peer +identity in one value prevents callers from accidentally combining fields +belonging to different nodes. Rust nodes construct and validate this syntax through -`saorsa_transport::TransportAddr::WebTransport` wrapped by +`saorsa_transport::TransportAddr::WebRtcDirect` wrapped by `saorsa_core::MultiAddr`; it is not a browser-specific string type. The JavaScript parser is the browser implementation of that same canonical wire -format and is covered by matching current/next-pin fixtures. +format and is covered by matching stable-pin fixtures. Quote and storage-commitment verification remains JavaScript for now. `ant-protocol 2.3.1` unconditionally reaches the native Saorsa transport and @@ -127,7 +139,7 @@ second implementation inside `ant-core`. Native `DhtNetworkManager` and the browser WASM adapter both drive the same candidate queue, peer-state, α-batch, and convergence implementation. Native QUIC retains transport authentication, address-report consensus, failure-cache integration, and trust updates; the -browser adapter retains WebTransport session establishment and endpoint +browser adapter retains WebRTC Direct connection establishment and endpoint validation. For compatibility with the local launcher, the bootstrap manifest still @@ -149,7 +161,7 @@ metadata chain. content verification. - Self-encryption, DataMap serialization, reconstruction, and content addressing have one Rust implementation across native and browser clients. -- The browser application keeps direct control of WebTransport, wallet, and +- The browser application keeps direct control of WebRTC, wallet, and DOM APIs without pulling native runtime dependencies into WASM. ### Negative / Trade-offs @@ -161,13 +173,24 @@ metadata chain. protocol rules until transport-free `ant-protocol` APIs exist. - The initial WASM module is approximately 1.4 MiB uncompressed and browser file processing is still in memory. -- Certificate and endpoint verification adds bootstrap-record lifecycle work. +- The node must preserve its DTLS certificate file; deleting it changes the + certhash and invalidates old bootstrap entries. +- WebRTC DataChannels need explicit 16-KiB message fragmentation, declared- + length reassembly, and `bufferedAmount` backpressure for large chunk + transfers. +- The Safari PoC observed later DataChannels timing out after repeatedly + replacing all seed associations between records, even though callers closed + earlier connections. Bounded association reuse avoids relying on prompt + browser resource reclamation and is required for multi-record operations. +- A browser may reuse a local UDP source port for a replacement association. + Node-side UDP multiplexing must route STUN binding requests by ICE credential + before consulting a possibly stale source-address mapping. ### Neutral / Operational - The manifest HTTP service carries only small bootstrap metadata. -- WebTransport still requires a secure browser context; localhost qualifies - for development. +- Browser WebRTC APIs still require a secure browser context; localhost + qualifies for development. - Node and client repositories must run compatible browser protocol versions. ## Validation @@ -188,7 +211,7 @@ metadata chain. - The generated WASM package encrypts and reconstructs the same fixed vector as the native Rust test, covering the native KDF, authenticated decryption, Brotli reconstruction, MessagePack DataMap, and BLAKE3 addresses. -- A live node integration test starts five WebTransport-enabled nodes, +- A live node integration test starts five WebRTC Direct-enabled nodes, publishes a public DataMap and encrypted chunks, connects with the advertised self-contained multiaddress, retrieves every record, and reconstructs the exact file, pays a real signed quote, accepts a paid binary PUT through the diff --git a/web/README.md b/web/README.md index 8f156ea3..43d0cb92 100644 --- a/web/README.md +++ b/web/README.md @@ -2,25 +2,25 @@ This web application is the browser-facing client for ADR-0009. It loads a local testnet bootstrap manifest, connects directly to storage nodes over -WebTransport, and performs closest-node lookup through Saorsa's shared Rust +WebRTC Direct, and performs closest-node lookup through Saorsa's shared Rust lookup engine. The portable part of `ant-core` is compiled to WASM and drives the Kademlia walk, native self-encryption, public DataMap serialization, reconstruction, and BLAKE3 content verification. The thin JavaScript layer -executes WebTransport requests, retrieves and uploads records, uses the browser +executes WebRTC Direct requests, retrieves and uploads records, uses the browser wallet/payment APIs, and drives the page. -The node-side WebTransport listener and testnet manifest API live in the +The node-side WebRTC Direct listener and testnet manifest API live in the `ant-node-web-support` sibling repository. No HTTP gateway performs lookup or proxies file bytes. ## Requirements -- Rust 1.88 or newer for the node's optional `wtransport` dependency. +- Rust 1.88 or newer for the node's optional Saorsa WebRTC transport. - The `wasm32-unknown-unknown` Rust target. - `wasm-pack` 0.15. - Node.js 20.19+ or 22.12+. -- A current browser implementing WebTransport certificate hashes. The client - extracts them from node multiaddresses; users do not enter hashes separately. +- A current browser implementing `RTCPeerConnection`. The WebRTC Direct dialer + extracts the stable certificate fingerprint from each node multiaddress. Nodes serialize these addresses from the native `saorsa_core::MultiAddr` representation; the JavaScript parser consumes that canonical string form. @@ -37,11 +37,11 @@ cargo install wasm-pack --version 0.15.0 --locked From `ant-node-web-support`: ```bash -cargo run --features webtransport-poc --bin ant-devnet -- \ +cargo run --features webrtc-direct --bin ant-devnet -- \ --preset minimal \ --base-port 23000 \ - --webtransport \ - --webtransport-base-port 24000 \ + --webrtc-direct \ + --webrtc-direct-base-port 24000 \ --serve-port 25000 \ --enable-evm \ --enable-logging @@ -88,9 +88,9 @@ manifest from port 25000 and fills in the default file: 1. **Load testnet** refreshes the manifest and direct multiaddress catalog. 2. **Connect** parses the first node multiaddress and performs a pinned - WebTransport `HELLO`. + WebRTC Direct `HELLO` and verifies its ML-DSA identity signature. 3. **Find closest** runs Saorsa's iterative lookup engine in WASM, with - JavaScript executing its WebTransport query batches. + JavaScript executing its WebRTC Direct query batches. 4. Under **Paid public file upload**, choose a file, paste the funded private key printed by ant-devnet, then select **Pay and upload file**. Rust/WASM performs encryption and DataMap generation; quote/commitment verification, @@ -102,12 +102,11 @@ manifest from port 25000 and fills in the default file: the whole file, verifies BLAKE3, and retains a **Save again** link. The browser receives the DataMap and all encrypted file bytes from UDP -24000-24004 over HTTP/3, not from the manifest server on TCP 25000. The -manifest server is bootstrap metadata only. +24000-24004 over ICE, DTLS, SCTP, and data channels, not from the manifest +server on TCP 25000. The manifest server is bootstrap metadata only. -For a LAN test, start the node devnet with `--host `, serve Vite with -`npm run dev -- --host 0.0.0.0`, and add the exact site Origin with -`--webtransport-origin http://:5173`. Change the manifest URL in the +For a LAN test, start the node devnet with `--host ` and serve Vite with +`npm run dev -- --host 0.0.0.0`. Change the manifest URL in the page to `http://:25000/api/browser-manifest.json`. ## Verify the client @@ -132,13 +131,13 @@ quote verification including the native Keccak-256 EVM quote hash, and a Rust `self_encryption 0.36` wire vector with native/WASM round-trip and tamper verification. Cross-repository live verification additionally starts the node testnet, -downloads all public-file records through WebTransport, reconstructs the +downloads all public-file records through WebRTC Direct, reconstructs the original bytes, pays a real quote on local Anvil, uploads a fresh record through the ordinary node payment validator, and reads it back. ## Current boundary -JavaScript currently owns WebTransport stream handling, Ethers payment +JavaScript currently owns native `RTCPeerConnection`/`RTCDataChannel` handling, Ethers payment submission, ML-DSA quote verification, and DOM/save APIs. Rust/WASM owns the complete iterative lookup loop; JavaScript implements only its transport batch callback. @@ -147,6 +146,6 @@ linked into a browser WASM build; a future transport-free feature can move the remaining quote and multiaddress verification into Rust. The local testnet manifest is intentionally unsigned bootstrap material. A -production deployment still needs ML-DSA-signed endpoint records, certificate -overlap/rotation, network dissemination, relayed WebTransport, and production -traffic quotas as specified by ADR-0009. +production deployment still needs ML-DSA-signed endpoint records, exceptional +certificate-rotation recovery, network dissemination, relayed WebRTC, and +production traffic quotas as specified by ADR-0009. diff --git a/web/index.html b/web/index.html index 8da7ece3..c1f4db1f 100644 --- a/web/index.html +++ b/web/index.html @@ -7,7 +7,7 @@ name="description" content="ADR-0009 direct browser client interoperability proof" /> - Autonomi WebTransport PoC + Autonomi WebRTC Direct PoC @@ -51,11 +51,11 @@

Direct node endpoint

It is populated from the testnet manifest and can also be pasted manually.

@@ -99,6 +99,7 @@

Paid public file upload

=6.0.0" + } + }, + "node_modules/@jridgewell/source-map": { + "version": "0.3.11", + "resolved": "https://registry.npmjs.org/@jridgewell/source-map/-/source-map-0.3.11.tgz", + "integrity": "sha512-ZMp1V8ZFcPG5dIWnQLr3NSI1MiCU7UETdS/A0G8V/XWHvJv3ZsFqutJn1Y5RPmAPX6F3BiE397OqveU/9NCuIA==", + "dev": true, + "license": "MIT", + "optional": true, + "peer": true, + "dependencies": { + "@jridgewell/gen-mapping": "^0.3.5", + "@jridgewell/trace-mapping": "^0.3.25" + } + }, + "node_modules/@jridgewell/sourcemap-codec": { + "version": "1.5.5", + "resolved": "https://registry.npmjs.org/@jridgewell/sourcemap-codec/-/sourcemap-codec-1.5.5.tgz", + "integrity": "sha512-cYQ9310grqxueWbl+WuIUIaiUaDcj7WOq5fVhEljNVgRfOUhY9fy2zTvfoqWsnebh8Sl70VScFbICvJnLKB0Og==", + "dev": true, + "license": "MIT", + "optional": true, + "peer": true + }, + "node_modules/@jridgewell/trace-mapping": { + "version": "0.3.31", + "resolved": "https://registry.npmjs.org/@jridgewell/trace-mapping/-/trace-mapping-0.3.31.tgz", + "integrity": "sha512-zzNR+SdQSDJzc8joaeP8QQoCQr8NuYx2dIIytl1QeBEZHJ9uW6hebsrYgbz8hJwUQao3TWCMtmfV8Nu1twOLAw==", + "dev": true, + "license": "MIT", + "optional": true, + "peer": true, + "dependencies": { + "@jridgewell/resolve-uri": "^3.1.0", + "@jridgewell/sourcemap-codec": "^1.4.14" + } + }, "node_modules/@msgpack/msgpack": { "version": "3.1.3", "resolved": "https://registry.npmjs.org/@msgpack/msgpack/-/msgpack-3.1.3.tgz", @@ -355,12 +415,36 @@ "undici-types": "~8.3.0" } }, + "node_modules/acorn": { + "version": "8.18.0", + "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.18.0.tgz", + "integrity": "sha512-lGq+9yr1/GuAWaVYIHRjvvySG5/4VfKIvC8EWxStPdcDh/Ka7FG3twP6v4d5BkravUilhIAsG4Qj83t02LWUPQ==", + "dev": true, + "license": "MIT", + "optional": true, + "peer": true, + "bin": { + "acorn": "bin/acorn" + }, + "engines": { + "node": ">=0.4.0" + } + }, "node_modules/aes-js": { "version": "4.0.0-beta.5", "resolved": "https://registry.npmjs.org/aes-js/-/aes-js-4.0.0-beta.5.tgz", "integrity": "sha512-G965FqalsNyrPqgEGON7nIx1e/OVENSgiEIzyC63haUMuvNnwIgIjMs52hlTCKhkBny7A2ORNlfY9Zu+jmGk1Q==", "license": "MIT" }, + "node_modules/buffer-from": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/buffer-from/-/buffer-from-1.1.2.tgz", + "integrity": "sha512-E+XQCRwSbaaiChtv6k6Dwgc+bx+Bs6vuKJHHl5kox/BaKbhiXzqQOwK4cO22yElGp2OCmjwVhT3HmxgyPGnJfQ==", + "dev": true, + "license": "MIT", + "optional": true, + "peer": true + }, "node_modules/detect-libc": { "version": "2.1.2", "resolved": "https://registry.npmjs.org/detect-libc/-/detect-libc-2.1.2.tgz", @@ -843,6 +927,61 @@ "node": ">=0.10.0" } }, + "node_modules/source-map-support": { + "version": "0.5.21", + "resolved": "https://registry.npmjs.org/source-map-support/-/source-map-support-0.5.21.tgz", + "integrity": "sha512-uBHU3L3czsIyYXKX88fdrGovxdSCoTGDRZ6SYXtSRxLZUzHg5P/66Ht6uoUlHu9EZod+inXhKo3qQgwXUT/y1w==", + "dev": true, + "license": "MIT", + "optional": true, + "peer": true, + "dependencies": { + "buffer-from": "^1.0.0", + "source-map": "^0.6.0" + } + }, + "node_modules/source-map-support/node_modules/source-map": { + "version": "0.6.1", + "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz", + "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==", + "dev": true, + "license": "BSD-3-Clause", + "optional": true, + "peer": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/terser": { + "version": "5.50.0", + "resolved": "https://registry.npmjs.org/terser/-/terser-5.50.0.tgz", + "integrity": "sha512-CN9BVxWhgS/hRxtUMjtC2uRWSTcSfQFHMDWma6sKKfIivCD91sM+FOPfvwoaRMqCSrUpe1nv3jDamd9eEQ4y+w==", + "dev": true, + "license": "BSD-2-Clause", + "optional": true, + "peer": true, + "dependencies": { + "@jridgewell/source-map": "^0.3.3", + "acorn": "^8.15.0", + "commander": "^2.20.0", + "source-map-support": "~0.5.20" + }, + "bin": { + "terser": "bin/terser" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/terser/node_modules/commander": { + "version": "2.20.3", + "resolved": "https://registry.npmjs.org/commander/-/commander-2.20.3.tgz", + "integrity": "sha512-GpVkmM8vF2vQUkj2LvZmD35JxeJOLCwJ9cUkugyk2nuhbv3+mJvpLYYt+0+USMxE+oj+ey/lJEnhZw75x/OMcQ==", + "dev": true, + "license": "MIT", + "optional": true, + "peer": true + }, "node_modules/tinyglobby": { "version": "0.2.17", "resolved": "https://registry.npmjs.org/tinyglobby/-/tinyglobby-0.2.17.tgz", diff --git a/web/src/file.js b/web/src/file.js index 76d774e8..bcb5b871 100644 --- a/web/src/file.js +++ b/web/src/file.js @@ -2,7 +2,12 @@ import { decodePublicDataMap as decodePublicDataMapNative, decryptPublicFile as decryptPublicFileNative, } from "../pkg/ant_core.js"; -import { getChunkFromClosest, hexToBytes, verifyChunk } from "./protocol.js"; +import { + BrowserNodeClientPool, + getChunkFromClosest, + hexToBytes, + verifyChunk, +} from "./protocol.js"; const MAX_DOWNLOAD_CONCURRENCY = 6; @@ -40,42 +45,53 @@ export async function downloadPublicFile( } const boundedConcurrency = Math.min(concurrency, MAX_DOWNLOAD_CONCURRENCY); - onProgress(`Fetching public DataMap ${file.address}`); - const dataMap = await downloadChunk(seedEndpoints, file.address, { onProgress }); - if (dataMap.content.length !== file.data_map_size) { - throw new Error( - `Public DataMap has ${dataMap.content.length} bytes, expected ${file.data_map_size}`, + const clientPool = new BrowserNodeClientPool(); + try { + onProgress(`Fetching public DataMap ${file.address}`); + const dataMap = await downloadChunk(seedEndpoints, file.address, { + onProgress, + clientPool, + }); + if (dataMap.content.length !== file.data_map_size) { + throw new Error( + `Public DataMap has ${dataMap.content.length} bytes, expected ${file.data_map_size}`, + ); + } + onProgress(`Verified public DataMap (${dataMap.content.length} bytes)`); + const chunks = decodeDataMap(dataMap.content); + if (!Array.isArray(chunks) || chunks.length < 3) { + throw new Error("ant-core WASM returned an invalid public DataMap"); + } + + const encryptedChunks = await mapWithConcurrency( + chunks, + boundedConcurrency, + async (chunk) => { + onProgress( + `Fetching encrypted file chunk ${chunk.index + 1}/${chunks.length} (${chunk.dst_hash})`, + ); + const downloaded = await downloadChunk(seedEndpoints, chunk.dst_hash, { + onProgress, + clientPool, + }); + return downloaded.content; + }, ); - } - onProgress(`Verified public DataMap (${dataMap.content.length} bytes)`); - const chunks = decodeDataMap(dataMap.content); - if (!Array.isArray(chunks) || chunks.length < 3) { - throw new Error("ant-core WASM returned an invalid public DataMap"); - } - const encryptedChunks = await mapWithConcurrency( - chunks, - boundedConcurrency, - async (chunk) => { - onProgress( - `Fetching encrypted file chunk ${chunk.index + 1}/${chunks.length} (${chunk.dst_hash})`, + onProgress(`Reconstructing ${file.name} with native ant-core WASM`); + const content = decrypt(dataMap.content, encryptedChunks); + if (!(content instanceof Uint8Array)) { + throw new Error("ant-core WASM returned non-byte file content"); + } + if (content.length !== file.size) { + throw new Error( + `Reconstructed file has ${content.length} bytes, expected ${file.size}`, ); - const downloaded = await downloadChunk(seedEndpoints, chunk.dst_hash, { - onProgress, - }); - return downloaded.content; - }, - ); - - onProgress(`Reconstructing ${file.name} with native ant-core WASM`); - const content = decrypt(dataMap.content, encryptedChunks); - if (!(content instanceof Uint8Array)) { - throw new Error("ant-core WASM returned non-byte file content"); - } - if (content.length !== file.size) { - throw new Error(`Reconstructed file has ${content.length} bytes, expected ${file.size}`); + } + const hash = verifyChunk(file.blake3, content); + onProgress(`Verified complete ${file.name} as ${hash}`); + return { content, hash, dataMapNode: dataMap.node }; + } finally { + clientPool.close(); } - const hash = verifyChunk(file.blake3, content); - onProgress(`Verified complete ${file.name} as ${hash}`); - return { content, hash, dataMapNode: dataMap.node }; } diff --git a/web/src/main.js b/web/src/main.js index dc2c27f5..3ff53bc5 100644 --- a/web/src/main.js +++ b/web/src/main.js @@ -52,7 +52,8 @@ function timestamp() { } function log(message, value) { - const suffix = value === undefined ? "" : `\n${JSON.stringify(value, null, 2)}`; + const suffix = + value === undefined ? "" : `\n${JSON.stringify(value, null, 2)}`; elements.log.textContent += `[${timestamp()}] ${message}${suffix}\n`; elements.log.scrollTop = elements.log.scrollHeight; } @@ -70,7 +71,9 @@ function seedEndpoints() { async function loadManifest() { elements.manifestState.classList.remove("connected"); elements.manifestState.textContent = "Loading…"; - const manifest = await fetchBrowserManifest(elements.manifestUrl.value.trim()); + const manifest = await fetchBrowserManifest( + elements.manifestUrl.value.trim(), + ); browserManifest = manifest; const first = manifest.endpoints[0]; @@ -158,7 +161,7 @@ elements.findClosest.addEventListener("click", async () => { message: error.message, })), }); - for (const lookupClient of result.clients.values()) lookupClient.close(); + if (result.ownsClientPool) result.clientPool.close(); } catch (error) { reportError("Lookup", error); } @@ -172,7 +175,8 @@ elements.uploadFile.addEventListener("click", async () => { let walletSecret = elements.walletSecret.value.trim(); elements.walletSecret.value = ""; try { - if (!browserManifest) throw new Error("Load the browser testnet manifest first"); + if (!browserManifest) + throw new Error("Load the browser testnet manifest first"); const file = elements.uploadInput.files?.[0]; if (!file) throw new Error("Choose a file to upload"); if (!walletSecret) throw new Error("Enter the paying wallet secret key"); @@ -207,7 +211,10 @@ elements.uploadFile.addEventListener("click", async () => { elements.uploadResult.hidden = false; elements.uploadState.textContent = "Uploaded · ready to download"; elements.uploadState.classList.add("connected"); - log(`Uploaded and registered ${result.file.name} for immediate download`, result); + log( + `Uploaded and registered ${result.file.name} for immediate download`, + result, + ); } catch (error) { elements.uploadState.textContent = "Upload failed"; log(`File upload failed: ${error.message}`); @@ -225,9 +232,13 @@ elements.downloadFile.addEventListener("click", async () => { try { const address = elements.fileAddress.value.trim().toLowerCase(); hexToBytes(address, 32); - const published = browserManifest?.files.find((file) => file.address === address); + const published = browserManifest?.files.find( + (file) => file.address === address, + ); if (!published) { - throw new Error("That public file address is not described by the loaded testnet manifest"); + throw new Error( + "That public file address is not described by the loaded testnet manifest", + ); } const saveHandle = await chooseSaveHandle(published.name); elements.downloadState.textContent = "Downloading…"; @@ -253,7 +264,8 @@ elements.downloadFile.addEventListener("click", async () => { { data_map_node: dataMapNode.peer_id, chunks: published.chunks.length }, ); } catch (error) { - elements.downloadState.textContent = error.name === "AbortError" ? "Save cancelled" : "Failed"; + elements.downloadState.textContent = + error.name === "AbortError" ? "Save cancelled" : "Failed"; log(`File download failed: ${error.message}`); console.error(error); } finally { @@ -284,7 +296,9 @@ async function exposeSavedFile(file, content, saveHandle) { if (downloadObjectUrl) URL.revokeObjectURL(downloadObjectUrl); downloadObjectUrl = URL.createObjectURL( - new Blob([content], { type: file.content_type ?? "application/octet-stream" }), + new Blob([content], { + type: file.content_type ?? "application/octet-stream", + }), ); const anchor = document.createElement("a"); anchor.href = downloadObjectUrl; diff --git a/web/src/manifest.js b/web/src/manifest.js index b42edd43..63ba8c04 100644 --- a/web/src/manifest.js +++ b/web/src/manifest.js @@ -1,6 +1,6 @@ -import { hexToBytes, parseWebTransportMultiaddr } from "./protocol.js"; +import { hexToBytes, parseWebRtcDirectMultiaddr } from "./protocol.js"; -export const BROWSER_MANIFEST_VERSION = 4; +export const BROWSER_MANIFEST_VERSION = 5; const MAX_PUBLIC_FILE_BYTES = 64 * 1024 * 1024; const MAX_DATA_MAP_BYTES = 4 * 1024 * 1024; const MAX_FILE_CHUNKS = 1024; @@ -13,10 +13,10 @@ export function parseBrowserManifest(value) { throw new Error("Browser manifest has no network ID"); } if (!Array.isArray(value.endpoints) || value.endpoints.length === 0) { - throw new Error("Browser manifest contains no WebTransport endpoints"); + throw new Error("Browser manifest contains no WebRtcDirect endpoints"); } const endpoints = value.endpoints.map((endpoint) => { - const parsed = parseWebTransportMultiaddr(endpoint); + const parsed = parseWebRtcDirectMultiaddr(endpoint); return { multiaddr: parsed.multiaddr }; }); const payment = parsePaymentNetwork(value.payment); diff --git a/web/src/manifest.test.js b/web/src/manifest.test.js index 4322ca1e..62efca80 100644 --- a/web/src/manifest.test.js +++ b/web/src/manifest.test.js @@ -4,13 +4,13 @@ import { parseBrowserManifest } from "./manifest.js"; test("browser manifest validates and normalizes endpoints and files", () => { const manifest = parseBrowserManifest({ - version: 4, + version: 5, network_id: "local-test", created_at: "2026-08-03T00:00:00Z", payment: paymentNetwork(), endpoints: [ { - multiaddr: webtransportMultiaddr("AA".repeat(32), 0xbb), + multiaddr: webrtc_directMultiaddr("AA".repeat(32), 0xbb), }, ], files: [ @@ -33,7 +33,7 @@ test("browser manifest validates and normalizes endpoints and files", () => { assert.equal( manifest.endpoints[0].multiaddr, - webtransportMultiaddr("AA".repeat(32), 0xbb), + webrtc_directMultiaddr("AA".repeat(32), 0xbb), ); assert.equal(manifest.files[0].address, "cc".repeat(32)); assert.equal(manifest.files[0].blake3, "dd".repeat(32)); @@ -49,23 +49,23 @@ test("browser manifest rejects missing endpoints and malformed multiaddresses", assert.throws( () => parseBrowserManifest({ - version: 4, + version: 5, network_id: "test", payment: paymentNetwork(), endpoints: [], }), - /no WebTransport endpoints/, + /no WebRtcDirect endpoints/, ); assert.throws( () => parseBrowserManifest({ - version: 4, + version: 5, network_id: "test", payment: paymentNetwork(), endpoints: [ { multiaddr: - "/ip4/127.0.0.1/udp/22000/quic-v1/webtransport/certhash/uAA/p2p/wrong", + "/ip4/127.0.0.1/udp/22000/webrtc-direct/certhash/uAA/p2p/wrong", }, ], }), @@ -74,15 +74,15 @@ test("browser manifest rejects missing endpoints and malformed multiaddresses", }); test("browser manifest requires public payment contract configuration", () => { - const endpoint = { multiaddr: webtransportMultiaddr("aa".repeat(32), 0xbb) }; + const endpoint = { multiaddr: webrtc_directMultiaddr("aa".repeat(32), 0xbb) }; assert.throws( - () => parseBrowserManifest({ version: 4, network_id: "test", endpoints: [endpoint] }), + () => parseBrowserManifest({ version: 5, network_id: "test", endpoints: [endpoint] }), /no payment network/, ); assert.throws( () => parseBrowserManifest({ - version: 4, + version: 5, network_id: "test", endpoints: [endpoint], payment: { ...paymentNetwork(), rpc_url: "file:///tmp/anvil" }, @@ -99,12 +99,12 @@ function paymentNetwork() { }; } -function webtransportMultiaddr(peerId, certificateByte) { +function webrtc_directMultiaddr(peerId, certificateByte) { const multihash = Uint8Array.from([ 0x12, 0x20, ...Array(32).fill(certificateByte), ]); const certhash = `u${Buffer.from(multihash).toString("base64url")}`; - return `/ip4/127.0.0.1/udp/22000/quic-v1/webtransport/certhash/${certhash}/p2p/${peerId}`; + return `/ip4/127.0.0.1/udp/22000/webrtc-direct/certhash/${certhash}/p2p/${peerId}`; } diff --git a/web/src/payment.js b/web/src/payment.js index f5ce0e94..7f2cc62d 100644 --- a/web/src/payment.js +++ b/web/src/payment.js @@ -224,7 +224,7 @@ export function verifyStorageQuote(quote, expectedAddress, expectedPeerId) { throw new Error("Storage quote is for a different chunk"); } if (quote.peer_id?.toLowerCase() !== expectedPeerId.toLowerCase()) { - throw new Error("Storage quote belongs to a different WebTransport peer"); + throw new Error("Storage quote belongs to a different WebRtcDirect peer"); } const publicKey = hexToBytes(quote.public_key); const signature = hexToBytes(quote.signature); diff --git a/web/src/payment.test.js b/web/src/payment.test.js index bdb7d1d7..84a4f680 100644 --- a/web/src/payment.test.js +++ b/web/src/payment.test.js @@ -191,7 +191,7 @@ test("rejects quote field tampering before any payment", () => { ); assert.throws( () => verifyStorageQuote(fixture.quote, fixture.address, "ff".repeat(32)), - /different WebTransport peer/, + /different WebRtcDirect peer/, ); const signature = Uint8Array.from(Buffer.from(fixture.quote.signature, "hex")); signature[100] ^= 1; diff --git a/web/src/protocol.js b/web/src/protocol.js index af8b2205..bfab9f99 100644 --- a/web/src/protocol.js +++ b/web/src/protocol.js @@ -1,4 +1,6 @@ import { bytesToHex as nobleBytesToHex } from "@noble/hashes/utils.js"; +import { blake3 } from "@noble/hashes/blake3.js"; +import { ml_dsa65 } from "@noble/post-quantum/ml-dsa.js"; import { BrowserIterativeLookup as BrowserIterativeLookupNative, verifyRecord as verifyRecordNative, @@ -6,16 +8,23 @@ import { export const PROTOCOL_VERSION = 3; export const PROTOCOL_NAME = "autonomi.web.poc.v3"; -export const WEBTRANSPORT_PATH = "/autonomi/webtransport/v1"; +export const WEBRTC_DIRECT_DATA_CHANNEL = "autonomi.web.v3"; export const MAX_CHUNK_SIZE = 4 * 1024 * 1024; export const MAX_RESPONSE_HEADER_BYTES = 64 * 1024; const MAX_RESPONSE_BYTES = 4 + MAX_RESPONSE_HEADER_BYTES + MAX_CHUNK_SIZE; -const MAX_WEBTRANSPORT_MULTIADDR_LENGTH = 2048; -const MAX_CERTIFICATE_HASHES = 2; +const MAX_WEBRTC_DIRECT_MULTIADDR_LENGTH = 2048; +const WEBRTC_WRITE_CHUNK_BYTES = 16 * 1024; +const MAX_BUFFERED_AMOUNT = 2 * 1024 * 1024; +const REQUEST_TIMEOUT_MS = 10_000; +const ICE_CREDENTIAL_PREFIX = "saorsa+webrtc+v1/"; +const ICE_RANDOM_LENGTH = 32; +const ICE_ALPHABET = + "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789"; const SHA2_256_MULTIHASH_CODE = 0x12; const SHA2_256_MULTIHASH_LENGTH = 32; const encoder = new TextEncoder(); const decoder = new TextDecoder("utf-8", { fatal: true }); +const HELLO_DOMAIN = encoder.encode("autonomi-webrtc-direct-hello-v1\0"); let nextRequestId = 1; @@ -28,7 +37,9 @@ export function hexToBytes(value, expectedLength) { normalized.match(/.{2}/g)?.map((pair) => Number.parseInt(pair, 16)) ?? [], ); if (expectedLength !== undefined && bytes.length !== expectedLength) { - throw new Error(`Expected ${expectedLength} bytes, received ${bytes.length}`); + throw new Error( + `Expected ${expectedLength} bytes, received ${bytes.length}`, + ); } return bytes; } @@ -37,6 +48,18 @@ export function bytesToHex(bytes) { return nobleBytesToHex(bytes); } +function concatBytes(...parts) { + const result = new Uint8Array( + parts.reduce((total, part) => total + part.length, 0), + ); + let offset = 0; + for (const part of parts) { + result.set(part, offset); + offset += part.length; + } + return result; +} + export function verifyChunk(address, content) { const expected = address.trim().replace(/^0x/i, "").toLowerCase(); hexToBytes(expected, 32); @@ -44,6 +67,16 @@ export function verifyChunk(address, content) { } export function parseResponseFrame(frame) { + const { header, contentOffset, frameLength } = parseResponseHeader(frame); + if (frame.length !== frameLength) { + throw new Error( + `Response length mismatch: declared ${header.content_length} content bytes`, + ); + } + return { header, content: frame.slice(contentOffset) }; +} + +function parseResponseHeader(frame) { if (!(frame instanceof Uint8Array) || frame.length < 4) { throw new Error("Response ended before its four-byte header length"); } @@ -61,7 +94,9 @@ export function parseResponseFrame(frame) { try { header = JSON.parse(decoder.decode(frame.subarray(4, contentOffset))); } catch (error) { - throw new Error(`Invalid response JSON: ${error.message}`, { cause: error }); + throw new Error(`Invalid response JSON: ${error.message}`, { + cause: error, + }); } if (header.version !== PROTOCOL_VERSION) { throw new Error(`Unsupported response version ${header.version}`); @@ -73,43 +108,63 @@ export function parseResponseFrame(frame) { ) { throw new Error(`Invalid response content length ${header.content_length}`); } - if (frame.length !== contentOffset + header.content_length) { - throw new Error( - `Response length mismatch: declared ${header.content_length} content bytes`, - ); - } - return { header, content: frame.slice(contentOffset) }; + return { + header, + contentOffset, + frameLength: contentOffset + header.content_length, + }; } -async function readAll(readable, limit = MAX_RESPONSE_BYTES) { - const reader = readable.getReader(); - const chunks = []; +export async function readResponseFrame(stream, limit = MAX_RESPONSE_BYTES) { + if (!Number.isSafeInteger(limit) || limit < 4) { + throw new Error(`Invalid response limit ${limit}`); + } + let frame = new Uint8Array(Math.min(limit, 8 * 1024)); let total = 0; - try { - while (true) { - const { value, done } = await reader.read(); - if (done) break; - if (!(value instanceof Uint8Array)) { - throw new Error("WebTransport returned a non-byte stream chunk"); + let expectedLength; + for await (const value of stream) { + const chunk = value instanceof Uint8Array ? value : value.subarray(); + const nextTotal = total + chunk.length; + if (nextTotal > limit) { + throw new Error(`Response exceeded the ${limit}-byte client limit`); + } + if (nextTotal > frame.length) { + let capacity = frame.length; + while (capacity < nextTotal) capacity = Math.min(limit, capacity * 2); + const grown = new Uint8Array(capacity); + grown.set(frame.subarray(0, total)); + frame = grown; + } + frame.set(chunk, total); + total = nextTotal; + + if (expectedLength === undefined && total >= 4) { + const headerLength = new DataView( + frame.buffer, + frame.byteOffset, + total, + ).getUint32(0, false); + if (headerLength === 0 || headerLength > MAX_RESPONSE_HEADER_BYTES) { + throw new Error(`Invalid response header length ${headerLength}`); } - total += value.length; - if (total > limit) { - await reader.cancel("response exceeded client limit"); - throw new Error(`Response exceeded the ${limit}-byte client limit`); + if (total >= 4 + headerLength) { + expectedLength = parseResponseHeader( + frame.subarray(0, total), + ).frameLength; + if (expectedLength > limit) { + throw new Error(`Response exceeded the ${limit}-byte client limit`); + } } - chunks.push(value); } - } finally { - reader.releaseLock(); - } - const result = new Uint8Array(total); - let offset = 0; - for (const chunk of chunks) { - result.set(chunk, offset); - offset += chunk.length; + if (expectedLength !== undefined && total >= expectedLength) { + if (total !== expectedLength) { + throw new Error("Response contains bytes after its declared frame"); + } + return frame.slice(0, total); + } } - return result; + throw new Error("Response ended before its declared frame was complete"); } function decodeBase64Url(value) { @@ -138,7 +193,9 @@ function decodeCertificateMultihash(value) { decoded[0] !== SHA2_256_MULTIHASH_CODE || decoded[1] !== SHA2_256_MULTIHASH_LENGTH ) { - throw new Error("Certificate multihash must contain a 32-byte SHA-256 digest"); + throw new Error( + "Certificate multihash must contain a 32-byte SHA-256 digest", + ); } return decoded.slice(2); } @@ -149,7 +206,8 @@ function validateIpv4(value) { octets.length !== 4 || octets.some( (octet) => - !/^(0|[1-9][0-9]{0,2})$/.test(octet) || Number.parseInt(octet, 10) > 255, + !/^(0|[1-9][0-9]{0,2})$/.test(octet) || + Number.parseInt(octet, 10) > 255, ) ) { throw new Error(`Invalid IPv4 address ${value}`); @@ -159,165 +217,438 @@ function validateIpv4(value) { function endpointMultiaddr(endpoint) { if (typeof endpoint === "string") return endpoint; - if (endpoint && typeof endpoint.multiaddr === "string") return endpoint.multiaddr; - throw new Error("A WebTransport multiaddress is required"); + if (endpoint && typeof endpoint.multiaddr === "string") + return endpoint.multiaddr; + throw new Error("A WebRtcDirect multiaddress is required"); } -export function parseWebTransportMultiaddr(endpoint) { +export function parseWebRtcDirectMultiaddr(endpoint) { const multiaddr = endpointMultiaddr(endpoint).trim(); if ( multiaddr.length === 0 || - multiaddr.length > MAX_WEBTRANSPORT_MULTIADDR_LENGTH || + multiaddr.length > MAX_WEBRTC_DIRECT_MULTIADDR_LENGTH || !multiaddr.startsWith("/") ) { - throw new Error("Invalid WebTransport multiaddress length or prefix"); + throw new Error("Invalid WebRtcDirect multiaddress length or prefix"); } const segments = multiaddr.split("/"); - if (segments.length < 9) { - throw new Error("WebTransport multiaddress is incomplete"); + if (segments.length !== 10) { + throw new Error("WebRtcDirect multiaddress is incomplete"); } const hostProtocol = segments[1]; const hostValue = segments[2]; - if (!hostValue) throw new Error("WebTransport multiaddress host is empty"); - let urlHost; + if (!hostValue) throw new Error("WebRtcDirect multiaddress host is empty"); if (hostProtocol === "ip4") { - urlHost = validateIpv4(hostValue); + validateIpv4(hostValue); } else if (hostProtocol === "ip6") { - urlHost = `[${hostValue}]`; - } else if (["dns", "dns4", "dns6"].includes(hostProtocol)) { - urlHost = hostValue.toLowerCase(); + if (!hostValue.includes(":")) + throw new Error(`Invalid IPv6 address ${hostValue}`); } else { - throw new Error(`Unsupported WebTransport host protocol ${hostProtocol}`); + throw new Error( + "WebRTC Direct multiaddresses must use a literal IP address", + ); } if (segments[3] !== "udp") { - throw new Error("WebTransport multiaddress must use UDP"); + throw new Error("WebRtcDirect multiaddress must use UDP"); } if (!/^[0-9]{1,5}$/.test(segments[4])) { - throw new Error("WebTransport multiaddress has an invalid UDP port"); + throw new Error("WebRtcDirect multiaddress has an invalid UDP port"); } const port = Number.parseInt(segments[4], 10); if (port < 1 || port > 65535) { - throw new Error("WebTransport multiaddress has an invalid UDP port"); + throw new Error("WebRtcDirect multiaddress has an invalid UDP port"); + } + if (segments[5] !== "webrtc-direct") { + throw new Error("WebRTC Direct multiaddress must contain /webrtc-direct"); } - if (segments[5] !== "quic-v1" || segments[6] !== "webtransport") { + if (segments[6] !== "certhash" || !segments[7]) { throw new Error( - "WebTransport multiaddress must contain /quic-v1/webtransport", + "WebRTC Direct multiaddress must contain exactly one certhash", ); } + const certificateHash = decodeCertificateMultihash(segments[7]); + if (segments[8] !== "p2p") { + throw new Error("WebRtcDirect multiaddress must end with /p2p/"); + } + const peerId = segments[9]?.toLowerCase() ?? ""; + hexToBytes(peerId, 32); + return { + multiaddr, + hostProtocol, + host: hostValue, + port, + peerId, + certificateHash, + }; +} + +const normalizeEndpoint = parseWebRtcDirectMultiaddr; - let index = 7; - const certificateHashes = []; - const certificateHashMultihashes = []; - while (segments[index] === "certhash") { - const encoded = segments[index + 1]; - if (!encoded) throw new Error("WebTransport multiaddress has an empty certhash"); - certificateHashes.push(decodeCertificateMultihash(encoded)); - certificateHashMultihashes.push(encoded); - index += 2; +export function verifyHelloIdentity(header, expectedEndpoint, challengeBytes) { + const endpoint = normalizeEndpoint(expectedEndpoint); + if (!(challengeBytes instanceof Uint8Array) || challengeBytes.length !== 32) { + throw new Error("HELLO verification requires a 32-byte challenge"); + } + if (header.type !== "hello") throw new Error("Expected a HELLO response"); + if (header.protocol !== PROTOCOL_NAME) { + throw new Error(`Unsupported browser protocol ${header.protocol}`); } + hexToBytes(header.peer_id, 32); + if (header.challenge?.toLowerCase() !== bytesToHex(challengeBytes)) { + throw new Error("Node signed a different HELLO challenge"); + } + const advertisedEndpoint = normalizeEndpoint(header.endpoint); if ( - certificateHashes.length < 1 || - certificateHashes.length > MAX_CERTIFICATE_HASHES + advertisedEndpoint.multiaddr !== endpoint.multiaddr || + advertisedEndpoint.peerId !== header.peer_id.toLowerCase() ) { + throw new Error("Node advertised a different WebRTC Direct endpoint"); + } + if (header.peer_id.toLowerCase() !== endpoint.peerId.toLowerCase()) { throw new Error( - `WebTransport multiaddress must contain between 1 and ${MAX_CERTIFICATE_HASHES} certificate hashes`, + `Endpoint identity mismatch: expected ${endpoint.peerId}, received ${header.peer_id}`, ); } - if (new Set(certificateHashMultihashes).size !== certificateHashes.length) { - throw new Error("WebTransport multiaddress contains duplicate certificate hashes"); + const publicKey = hexToBytes(header.public_key); + const signature = hexToBytes(header.signature); + if (publicKey.length !== ml_dsa65.lengths.publicKey) { + throw new Error(`HELLO has a ${publicKey.length}-byte public key`); } - if (segments[index] !== "p2p" || index + 2 !== segments.length) { - throw new Error("WebTransport multiaddress must end with /p2p/"); + if (signature.length !== ml_dsa65.lengths.signature) { + throw new Error(`HELLO has a ${signature.length}-byte signature`); } - const peerId = segments[index + 1]?.toLowerCase() ?? ""; - hexToBytes(peerId, 32); + if (bytesToHex(blake3(publicKey)) !== header.peer_id.toLowerCase()) { + throw new Error("HELLO public key is not bound to the ANT peer ID"); + } + const transcript = concatBytes( + HELLO_DOMAIN, + challengeBytes, + encoder.encode(header.peer_id.toLowerCase()), + encoder.encode(advertisedEndpoint.multiaddr), + ); + if (!ml_dsa65.verify(signature, transcript, publicKey)) { + throw new Error("HELLO has an invalid ML-DSA-65 signature"); + } + return header.peer_id.toLowerCase(); +} - let url; - try { - url = new URL(`https://${urlHost}:${port}${WEBTRANSPORT_PATH}`).toString(); - } catch (error) { - throw new Error("WebTransport multiaddress contains an invalid host", { - cause: error, - }); +function randomIceCredential() { + const random = crypto.getRandomValues(new Uint8Array(ICE_RANDOM_LENGTH)); + let suffix = ""; + for (const byte of random) suffix += ICE_ALPHABET[byte % ICE_ALPHABET.length]; + return ICE_CREDENTIAL_PREFIX + suffix; +} + +function certificateFingerprint(certificateHash) { + return Array.from(certificateHash, (byte) => + byte.toString(16).padStart(2, "0").toUpperCase(), + ).join(":"); +} + +export function serverAnswerFromEndpoint(endpoint, iceCredential) { + const normalized = normalizeEndpoint(endpoint); + if ( + typeof iceCredential !== "string" || + !iceCredential.startsWith(ICE_CREDENTIAL_PREFIX) || + !/^[a-zA-Z0-9+/]{22,256}$/.test(iceCredential) + ) { + throw new Error("Invalid Saorsa WebRTC Direct ICE credential"); } + const ipVersion = normalized.hostProtocol === "ip4" ? "IP4" : "IP6"; return { - multiaddr, - url, - peerId, - certificateHashes, + type: "answer", + sdp: `v=0\r +o=- 0 0 IN ${ipVersion} ${normalized.host}\r +s=-\r +t=0 0\r +a=ice-lite\r +m=application ${normalized.port} UDP/DTLS/SCTP webrtc-datachannel\r +c=IN ${ipVersion} ${normalized.host}\r +a=mid:0\r +a=ice-options:ice2\r +a=ice-ufrag:${iceCredential}\r +a=ice-pwd:${iceCredential}\r +a=fingerprint:sha-256 ${certificateFingerprint(normalized.certificateHash)}\r +a=setup:passive\r +a=sctp-port:5000\r +a=max-message-size:${WEBRTC_WRITE_CHUNK_BYTES}\r +a=candidate:1467250027 1 UDP 1467250027 ${normalized.host} ${normalized.port} typ host\r +a=end-of-candidates\r +`, }; } -const normalizeEndpoint = parseWebTransportMultiaddr; +export function mungeOfferIceCredentials(offer, iceCredential) { + if (!offer?.sdp) throw new Error("Browser created an empty WebRTC offer"); + const sdp = offer.sdp + .replace(/a=ice-ufrag:[^\r\n]+/, `a=ice-ufrag:${iceCredential}`) + .replace(/a=ice-pwd:[^\r\n]+/, `a=ice-pwd:${iceCredential}`); + if ( + !sdp.includes(`a=ice-ufrag:${iceCredential}`) || + !sdp.includes(`a=ice-pwd:${iceCredential}`) + ) { + throw new Error("Browser offer did not contain ICE credentials"); + } + return { type: "offer", sdp }; +} + +class DataChannelInbox { + constructor(channel) { + this.queue = []; + this.waiters = []; + this.closed = false; + this.error = undefined; + channel.binaryType = "arraybuffer"; + channel.addEventListener("message", ({ data }) => { + try { + let message; + if (data instanceof ArrayBuffer) { + message = new Uint8Array(data); + } else if (ArrayBuffer.isView(data)) { + message = new Uint8Array( + data.buffer, + data.byteOffset, + data.byteLength, + ); + } else { + throw new Error("Node sent a non-binary DataChannel message"); + } + this.push(message); + } catch (error) { + this.fail(error); + } + }); + channel.addEventListener("error", (event) => { + this.fail(event.error ?? new Error("WebRTC DataChannel failed")); + }); + channel.addEventListener("close", () => this.finish()); + } + + push(message) { + const waiter = this.waiters.shift(); + if (waiter) waiter.resolve({ value: message, done: false }); + else this.queue.push(message); + } + + finish() { + this.closed = true; + for (const waiter of this.waiters.splice(0)) waiter.resolve({ done: true }); + } + + fail(error) { + this.error = error; + for (const waiter of this.waiters.splice(0)) waiter.reject(error); + } + + next() { + if (this.queue.length > 0) { + return Promise.resolve({ value: this.queue.shift(), done: false }); + } + if (this.error) return Promise.reject(this.error); + if (this.closed) return Promise.resolve({ done: true }); + return new Promise((resolve, reject) => + this.waiters.push({ resolve, reject }), + ); + } + + [Symbol.asyncIterator]() { + return this; + } +} + +function waitForDataChannelOpen(channel, timeoutMs = REQUEST_TIMEOUT_MS) { + if (channel.readyState === "open") return Promise.resolve(); + return new Promise((resolve, reject) => { + const timeout = setTimeout(() => { + cleanup(); + reject(new Error("WebRTC DataChannel opening timed out")); + }, timeoutMs); + const cleanup = () => { + clearTimeout(timeout); + channel.removeEventListener("open", opened); + channel.removeEventListener("close", closed); + channel.removeEventListener("error", failed); + }; + const opened = () => { + cleanup(); + resolve(); + }; + const closed = () => { + cleanup(); + reject(new Error("WebRTC DataChannel closed before opening")); + }; + const failed = (event) => { + cleanup(); + reject( + event.error ?? new Error("WebRTC DataChannel failed while opening"), + ); + }; + channel.addEventListener("open", opened, { once: true }); + channel.addEventListener("close", closed, { once: true }); + channel.addEventListener("error", failed, { once: true }); + }); +} + +async function waitForDataChannelCapacity(channel) { + if (channel.bufferedAmount <= MAX_BUFFERED_AMOUNT) return; + channel.bufferedAmountLowThreshold = MAX_BUFFERED_AMOUNT / 2; + await new Promise((resolve, reject) => { + const drained = () => { + cleanup(); + resolve(); + }; + const closed = () => { + cleanup(); + reject(new Error("WebRTC DataChannel closed while draining")); + }; + const cleanup = () => { + channel.removeEventListener("bufferedamountlow", drained); + channel.removeEventListener("close", closed); + }; + channel.addEventListener("bufferedamountlow", drained, { once: true }); + channel.addEventListener("close", closed, { once: true }); + }); +} export class BrowserNodeClient { constructor(endpoint) { this.endpoint = normalizeEndpoint(endpoint); - this.transport = undefined; + this.peerConnection = undefined; + this.dataChannel = undefined; + this.inbox = undefined; + this.connectPromise = undefined; + this.requestTail = Promise.resolve(); this.peerId = undefined; + this.helloResponse = undefined; + this.helloPromise = undefined; } async connect() { - if (this.transport) return; - if (typeof WebTransport === "undefined") { - throw new Error("This browser does not expose the WebTransport API"); - } - const transport = new WebTransport(this.endpoint.url, { - serverCertificateHashes: this.endpoint.certificateHashes.map((value) => ({ - algorithm: "sha-256", - value, - })), - }); + if (this.dataChannel?.readyState === "open") return; + if (this.connectPromise) return this.connectPromise; + if (typeof RTCPeerConnection === "undefined") { + throw new Error("This browser does not expose the RTCPeerConnection API"); + } + this.connectPromise = this.openDirectConnection(); try { - await transport.ready; + await this.connectPromise; } catch (error) { - transport.close(); + this.close(); throw error; + } finally { + this.connectPromise = undefined; } - this.transport = transport; } async request(type, fields = {}, content = new Uint8Array()) { + const operation = this.requestTail.then(() => + this.requestDirect(type, fields, content), + ); + this.requestTail = operation.then( + () => undefined, + () => undefined, + ); + return operation; + } + + async openDirectConnection() { + const configuration = { iceServers: [] }; + if (typeof RTCPeerConnection.generateCertificate === "function") { + configuration.certificates = [ + await RTCPeerConnection.generateCertificate({ + name: "ECDSA", + namedCurve: "P-256", + }), + ]; + } + const peerConnection = new RTCPeerConnection(configuration); + const dataChannel = peerConnection.createDataChannel( + WEBRTC_DIRECT_DATA_CHANNEL, + { + ordered: true, + }, + ); + const inbox = new DataChannelInbox(dataChannel); + this.peerConnection = peerConnection; + this.dataChannel = dataChannel; + this.inbox = inbox; + const iceCredential = randomIceCredential(); + const offer = mungeOfferIceCredentials( + await peerConnection.createOffer(), + iceCredential, + ); + await peerConnection.setLocalDescription(offer); + await peerConnection.setRemoteDescription( + serverAnswerFromEndpoint(this.endpoint, iceCredential), + ); + await waitForDataChannelOpen(dataChannel); + } + + async requestDirect(type, fields, content) { await this.connect(); if (!(content instanceof Uint8Array) || content.length > MAX_CHUNK_SIZE) { - throw new Error(`Request content must be at most ${MAX_CHUNK_SIZE} bytes`); + throw new Error( + `Request content must be at most ${MAX_CHUNK_SIZE} bytes`, + ); } const requestId = nextRequestId; nextRequestId += 1; - const stream = await this.transport.createBidirectionalStream(); - const writer = stream.writable.getWriter(); + const header = encoder.encode( + JSON.stringify({ + version: PROTOCOL_VERSION, + request_id: requestId, + content_length: content.length, + type, + ...fields, + }), + ); + if (header.length === 0 || header.length > MAX_RESPONSE_HEADER_BYTES) { + throw new Error(`Request header is ${header.length} bytes`); + } + const frame = new Uint8Array(4 + header.length + content.length); + new DataView(frame.buffer).setUint32(0, header.length, false); + frame.set(header, 4); + frame.set(content, 4 + header.length); + for ( + let offset = 0; + offset < frame.length; + offset += WEBRTC_WRITE_CHUNK_BYTES + ) { + await waitForDataChannelCapacity(this.dataChannel); + this.dataChannel.send( + frame.subarray(offset, offset + WEBRTC_WRITE_CHUNK_BYTES), + ); + } + let timeout; + let responseFrame; try { - const header = encoder.encode( - JSON.stringify({ - version: PROTOCOL_VERSION, - request_id: requestId, - content_length: content.length, - type, - ...fields, + responseFrame = await Promise.race([ + readResponseFrame(this.inbox), + new Promise((_, reject) => { + timeout = setTimeout( + () => reject(new Error("WebRTC request timed out")), + REQUEST_TIMEOUT_MS, + ); }), - ); - if (header.length === 0 || header.length > MAX_RESPONSE_HEADER_BYTES) { - throw new Error(`Request header is ${header.length} bytes`); - } - const prefix = new Uint8Array(4); - new DataView(prefix.buffer).setUint32(0, header.length, false); - await writer.write(prefix); - await writer.write(header); - if (content.length > 0) await writer.write(content); - await writer.close(); + ]); + } catch (error) { + this.close(); + throw error; } finally { - writer.releaseLock(); + clearTimeout(timeout); } - const response = parseResponseFrame(await readAll(stream.readable)); + const response = parseResponseFrame(responseFrame); if (response.header.request_id !== requestId) { throw new Error( `Response ID ${response.header.request_id} does not match request ${requestId}`, ); } if (response.header.status === "error") { - const error = new Error(response.header.message ?? "Node returned an error"); + const error = new Error( + response.header.message ?? "Node returned an error", + ); error.code = response.header.code; throw error; } @@ -325,29 +656,23 @@ export class BrowserNodeClient { } async hello() { - const { header } = await this.request("hello"); - if (header.type !== "hello") throw new Error("Expected a HELLO response"); - if (header.protocol !== PROTOCOL_NAME) { - throw new Error(`Unsupported browser protocol ${header.protocol}`); - } - hexToBytes(header.peer_id, 32); - const advertisedEndpoint = normalizeEndpoint(header.endpoint); - if ( - advertisedEndpoint.multiaddr !== this.endpoint.multiaddr || - advertisedEndpoint.peerId !== header.peer_id.toLowerCase() - ) { - throw new Error("Node advertised a different WebTransport endpoint"); + if (this.helloResponse && this.dataChannel?.readyState === "open") { + return this.helloResponse; } - if ( - this.endpoint.peerId && - header.peer_id.toLowerCase() !== this.endpoint.peerId.toLowerCase() - ) { - throw new Error( - `Endpoint identity mismatch: expected ${this.endpoint.peerId}, received ${header.peer_id}`, - ); + if (this.helloPromise) return this.helloPromise; + this.helloPromise = (async () => { + const challengeBytes = crypto.getRandomValues(new Uint8Array(32)); + const challenge = bytesToHex(challengeBytes); + const { header } = await this.request("hello", { challenge }); + this.peerId = verifyHelloIdentity(header, this.endpoint, challengeBytes); + this.helloResponse = header; + return header; + })(); + try { + return await this.helloPromise; + } finally { + this.helloPromise = undefined; } - this.peerId = header.peer_id; - return header; } async findNode(target, count = 20) { @@ -362,10 +687,12 @@ export class BrowserNodeClient { } for (const node of header.nodes) { hexToBytes(node.peer_id, 32); - if (node.webtransport) { - const parsed = normalizeEndpoint(node.webtransport); + if (node.webrtc_direct) { + const parsed = normalizeEndpoint(node.webrtc_direct); if (parsed.peerId !== node.peer_id.toLowerCase()) { - throw new Error(`Node ${node.peer_id} advertised another peer's endpoint`); + throw new Error( + `Node ${node.peer_id} advertised another peer's endpoint`, + ); } } } @@ -405,7 +732,10 @@ export class BrowserNodeClient { if (header.address.toLowerCase() !== address.toLowerCase()) { throw new Error("Node returned a quote for a different chunk address"); } - return { quote: header.quote, alreadyStored: Boolean(header.already_stored) }; + return { + quote: header.quote, + alreadyStored: Boolean(header.already_stored), + }; } async putChunk(address, content, quote, transactionHash) { @@ -423,12 +753,22 @@ export class BrowserNodeClient { if (header.address.toLowerCase() !== address.toLowerCase()) { throw new Error("Node stored a different chunk address"); } - return { address: header.address, alreadyStored: Boolean(header.already_stored) }; + return { + address: header.address, + alreadyStored: Boolean(header.already_stored), + }; } close() { - this.transport?.close({ closeCode: 0, reason: "client closed" }); - this.transport = undefined; + this.dataChannel?.close(); + this.peerConnection?.close(); + this.dataChannel = undefined; + this.peerConnection = undefined; + this.inbox = undefined; + this.connectPromise = undefined; + this.peerId = undefined; + this.helloResponse = undefined; + this.helloPromise = undefined; } } @@ -437,42 +777,145 @@ function endpointKey(endpoint) { return normalized.multiaddr; } +const DEFAULT_MAX_POOLED_CLIENTS = 10; + +/** + * A bounded set of reusable browser-to-node WebRTC associations. + * + * In Safari, the PoC exhausted WebRTC resources when it rapidly replaced every + * seed connection for each chunk, even though callers invoked `close()`. + * Keeping authenticated, persistent DataChannels avoids relying on prompt + * reclamation and avoids repeating ICE, DTLS, SCTP, and HELLO for each lookup + * and record. + */ +export class BrowserNodeClientPool { + constructor({ + maxClients = DEFAULT_MAX_POOLED_CLIENTS, + clientFactory = (endpoint) => new BrowserNodeClient(endpoint), + } = {}) { + if (!Number.isSafeInteger(maxClients) || maxClients < 1) { + throw new Error("WebRTC client pool size must be a positive integer"); + } + if (typeof clientFactory !== "function") { + throw new Error("WebRTC client pool factory must be a function"); + } + this.maxClients = maxClients; + this.clientFactory = clientFactory; + this.entries = new Map(); + this.waiters = []; + this.clock = 0; + this.closed = false; + } + + get size() { + return this.entries.size; + } + + async withClient(endpoint, operation) { + if (typeof operation !== "function") { + throw new Error("WebRTC client pool operation must be a function"); + } + const entry = await this.acquire(endpoint); + try { + return await operation(entry.client); + } finally { + this.release(entry); + } + } + + async acquire(endpoint) { + const key = endpointKey(endpoint); + for (;;) { + if (this.closed) throw new Error("WebRTC client pool is closed"); + + const existing = this.entries.get(key); + if (existing) { + existing.active += 1; + existing.lastUsed = ++this.clock; + return existing; + } + + if (this.entries.size < this.maxClients) { + const entry = { + key, + client: this.clientFactory(endpoint), + active: 1, + lastUsed: ++this.clock, + }; + this.entries.set(key, entry); + return entry; + } + + let oldestIdle; + for (const candidate of this.entries.values()) { + if ( + candidate.active === 0 && + (!oldestIdle || candidate.lastUsed < oldestIdle.lastUsed) + ) { + oldestIdle = candidate; + } + } + if (oldestIdle) { + this.entries.delete(oldestIdle.key); + oldestIdle.client.close(); + continue; + } + + await new Promise((resolve) => this.waiters.push(resolve)); + } + } + + release(entry) { + entry.active = Math.max(0, entry.active - 1); + entry.lastUsed = ++this.clock; + this.waiters.shift()?.(); + } + + close() { + if (this.closed) return; + this.closed = true; + for (const entry of this.entries.values()) entry.client.close(); + this.entries.clear(); + for (const wake of this.waiters.splice(0)) wake(); + } +} + export async function iterativeFindClosest( seedEndpoints, target, - { k = 20, alpha = 3, maxIterations = 20, onProgress = () => {} } = {}, + { + k = 20, + alpha = 3, + maxIterations = 20, + onProgress = () => {}, + clientPool, + } = {}, ) { hexToBytes(target, 32); if (!Array.isArray(seedEndpoints) || seedEndpoints.length === 0) { throw new Error("At least one seed endpoint is required"); } - const clients = new Map(); + const ownsClientPool = clientPool === undefined; + const pool = clientPool ?? new BrowserNodeClientPool(); const failures = []; const seedNodes = []; - const clientFor = (endpoint) => { - const key = endpointKey(endpoint); - let client = clients.get(key); - if (!client) { - client = new BrowserNodeClient(endpoint); - clients.set(key, client); - } - return client; - }; - await Promise.all( seedEndpoints.map(async (endpoint) => { const seedName = - typeof endpoint === "string" ? endpoint : endpoint?.multiaddr ?? "seed"; + typeof endpoint === "string" + ? endpoint + : (endpoint?.multiaddr ?? "seed"); try { - const client = clientFor(endpoint); - const hello = await client.hello(); + const hello = await pool.withClient(endpoint, (client) => + client.hello(), + ); seedNodes.push({ peer_id: hello.peer_id, native_addresses: [], reliability: 1, - webtransport: hello.endpoint, + webrtc_direct: hello.endpoint, }); onProgress(`Connected seed ${hello.peer_id}`); } catch (error) { @@ -482,38 +925,59 @@ export async function iterativeFindClosest( }), ); if (seedNodes.length === 0) { - for (const client of clients.values()) client.close(); + if (ownsClientPool) pool.close(); const detail = failures.map(({ error }) => error.message).join("; "); - throw new Error(`Could not connect to any WebTransport seed: ${detail}`); + throw new Error(`Could not connect to any WebRtcDirect seed: ${detail}`); } - const lookup = new BrowserIterativeLookupNative(target, k, alpha, maxIterations); - lookup.addCandidates(seedNodes); - await lookup.run(async ({ target: lookupTarget, count, iteration, candidates }) => - Promise.all( - candidates.map(async (candidate) => { - try { - const candidateClient = clientFor(candidate.webtransport); - if (!candidateClient.peerId) await candidateClient.hello(); - const nodes = await candidateClient.findNode(lookupTarget, count); - onProgress( - `Iteration ${iteration}: ${candidate.peer_id} returned ${nodes.length} nodes`, - ); - return { - status: "succeeded", - responder: candidate.peer_id, - candidates: nodes, - }; - } catch (error) { - failures.push({ peerId: candidate.peer_id, error }); - onProgress(`Query ${candidate.peer_id} failed: ${error.message}`); - return { status: "failed", responder: candidate.peer_id }; - } - }), - ), + const lookup = new BrowserIterativeLookupNative( + target, + k, + alpha, + maxIterations, ); + lookup.addCandidates(seedNodes); + try { + await lookup.run( + async ({ target: lookupTarget, count, iteration, candidates }) => + Promise.all( + candidates.map(async (candidate) => { + try { + const nodes = await pool.withClient( + candidate.webrtc_direct, + async (client) => { + if (!client.peerId) await client.hello(); + return client.findNode(lookupTarget, count); + }, + ); + onProgress( + `Iteration ${iteration}: ${candidate.peer_id} returned ${nodes.length} nodes`, + ); + return { + status: "succeeded", + responder: candidate.peer_id, + candidates: nodes, + }; + } catch (error) { + failures.push({ peerId: candidate.peer_id, error }); + onProgress(`Query ${candidate.peer_id} failed: ${error.message}`); + return { status: "failed", responder: candidate.peer_id }; + } + }), + ), + ); - return { nodes: lookup.results(), queried: lookup.queriedPeers(), failures, clients }; + return { + nodes: lookup.results(), + queried: lookup.queriedPeers(), + failures, + clientPool: pool, + ownsClientPool, + }; + } catch (error) { + if (ownsClientPool) pool.close(); + throw error; + } } export async function getChunkFromClosest( @@ -530,33 +994,32 @@ export async function getChunkFromClosest( try { for (const node of lookup.nodes) { - if (!node.webtransport) continue; - const endpoint = node.webtransport; - const key = endpointKey(endpoint); - let client = lookup.clients.get(key); - if (!client) { - client = new BrowserNodeClient(endpoint); - lookup.clients.set(key, client); - } - + if (!node.webrtc_direct) continue; try { - if (!client.peerId) await client.hello(); onProgress(`Requesting ${address} from ${node.peer_id}`); - const chunk = await client.getChunk(address); + const chunk = await lookup.clientPool.withClient( + node.webrtc_direct, + async (client) => { + if (!client.peerId) await client.hello(); + return client.getChunk(address); + }, + ); return { ...chunk, node, lookup }; } catch (error) { attempted.push({ peerId: node.peer_id, error }); - onProgress(`Node ${node.peer_id} did not return the file: ${error.message}`); + onProgress( + `Node ${node.peer_id} did not return the file: ${error.message}`, + ); } } } finally { - for (const client of lookup.clients.values()) client.close(); + if (lookup.ownsClientPool) lookup.clientPool.close(); } const detail = attempted .map(({ peerId, error }) => `${peerId}: ${error.message}`) .join("; "); throw new Error( - `No closest WebTransport node returned chunk ${address}${detail ? ` (${detail})` : ""}`, + `No closest WebRtcDirect node returned chunk ${address}${detail ? ` (${detail})` : ""}`, ); } diff --git a/web/src/protocol.test.js b/web/src/protocol.test.js index 5959317d..dbc7998a 100644 --- a/web/src/protocol.test.js +++ b/web/src/protocol.test.js @@ -1,14 +1,18 @@ import assert from "node:assert/strict"; import test from "node:test"; import { blake3 } from "@noble/hashes/blake3.js"; +import { ml_dsa65 } from "@noble/post-quantum/ml-dsa.js"; import { - BrowserNodeClient, + BrowserNodeClientPool, bytesToHex, - getChunkFromClosest, hexToBytes, + mungeOfferIceCredentials, parseResponseFrame, - parseWebTransportMultiaddr, + parseWebRtcDirectMultiaddr, + readResponseFrame, + serverAnswerFromEndpoint, verifyChunk, + verifyHelloIdentity, } from "./protocol.js"; test("hex conversion enforces fixed widths", () => { @@ -40,263 +44,262 @@ test("response framing preserves a raw binary body", () => { assert.deepEqual([...parsed.content], [1, 2, 3]); }); -test("WebTransport multiaddresses carry current and next certificate hashes", () => { +test("response framing completes without waiting for stream EOF", async () => { + const header = new TextEncoder().encode( + JSON.stringify({ + version: 3, + request_id: 10, + status: "ok", + content_length: 3, + type: "chunk", + }), + ); + const frame = new Uint8Array(4 + header.length + 3); + new DataView(frame.buffer).setUint32(0, header.length, false); + frame.set(header, 4); + frame.set([4, 5, 6], 4 + header.length); + let requestedAnotherChunk = false; + const stream = { + async *[Symbol.asyncIterator]() { + yield frame.subarray(0, 2); + yield frame.subarray(2, frame.length); + requestedAnotherChunk = true; + await new Promise(() => {}); + }, + }; + + const received = await readResponseFrame(stream); + assert.deepEqual(received, frame); + assert.equal(requestedAnotherChunk, false); +}); + +test("WebRTC Direct multiaddresses carry one stable certificate hash", () => { const peerId = "ab".repeat(32); - const multiaddr = webtransportMultiaddr("ip4", "127.0.0.1", 24000, peerId, [ + const multiaddr = webrtc_directMultiaddr( + "ip4", + "127.0.0.1", + 24000, + peerId, 0x11, - 0x22, - ]); - const parsed = parseWebTransportMultiaddr(multiaddr); + ); + const parsed = parseWebRtcDirectMultiaddr(multiaddr); - assert.equal(parsed.url, "https://127.0.0.1:24000/autonomi/webtransport/v1"); + assert.equal(parsed.hostProtocol, "ip4"); + assert.equal(parsed.host, "127.0.0.1"); + assert.equal(parsed.port, 24000); assert.equal(parsed.peerId, peerId); - assert.deepEqual( - parsed.certificateHashes.map((hash) => [...hash]), - [Array(32).fill(0x11), Array(32).fill(0x22)], + assert.deepEqual([...parsed.certificateHash], Array(32).fill(0x11)); + assert.throws( + () => + parseWebRtcDirectMultiaddr( + `/ip4/127.0.0.1/udp/24000/webrtc-direct/p2p/${peerId}`, + ), + /certhash|incomplete/, ); assert.throws( () => - parseWebTransportMultiaddr( - `/ip4/127.0.0.1/udp/24000/quic-v1/webtransport/p2p/${peerId}`, + parseWebRtcDirectMultiaddr( + `/dns/node.example/udp/24000/webrtc-direct/certhash/${certificateMultihash(0x11)}/p2p/${peerId}`, ), - /certificate hashes/, + /literal IP/, ); }); -test("BLAKE3 verification accepts the canonical empty hash", () => { - const emptyHash = - "af1349b9f5f9a1a6a0404dea36dcc9499bcb25c9adc112b7cc9a93cae41f3262"; - assert.equal(verifyChunk(emptyHash, new Uint8Array()), emptyHash); - assert.throws(() => verifyChunk("00".repeat(32), new Uint8Array()), /BLAKE3 mismatch/); +test("the browser synthesizes a certificate-pinned ICE-lite answer", () => { + const endpoint = webrtc_directMultiaddr( + "ip4", + "127.0.0.1", + 24000, + "ab".repeat(32), + 0x11, + ); + const credential = `saorsa+webrtc+v1/${"a".repeat(32)}`; + const answer = serverAnswerFromEndpoint(endpoint, credential); + + assert.equal(answer.type, "answer"); + assert.match(answer.sdp, /a=ice-lite/); + assert.match( + answer.sdp, + /m=application 24000 UDP\/DTLS\/SCTP webrtc-datachannel/, + ); + assert.match( + answer.sdp, + new RegExp(`a=ice-ufrag:${credential.replaceAll("+", "\\+")}`), + ); + assert.match(answer.sdp, /a=fingerprint:sha-256 11:11:11:11/); + assert.match(answer.sdp, /a=setup:passive/); }); -test("browser lookup discovers a direct node and downloads a verified chunk", async (t) => { - const content = new TextEncoder().encode("direct browser test file"); - const address = bytesToHex(blake3(content)); - const seedPeer = "ff".repeat(32); - const storagePeer = address; - const seed = endpoint("seed.test", seedPeer, 0x11); - const storage = endpoint("storage.test", storagePeer, 0x22); - const calls = []; - const routes = new Map([ - [ - seed.url, - (request) => { - calls.push(["seed", request.type]); - if (request.type === "hello") return helloResponse(request, seed); - if (request.type === "find_node") { - return response(request, { - type: "nodes", - target: request.target, - nodes: [browserNode(storage), browserNode(seed)], - }); - } - throw new Error(`Unexpected seed request ${request.type}`); - }, - ], - [ - storage.url, - (request) => { - calls.push(["storage", request.type]); - if (request.type === "hello") return helloResponse(request, storage); - if (request.type === "find_node") { - return response(request, { - type: "nodes", - target: request.target, - nodes: [browserNode(storage), browserNode(seed)], - }); - } - if (request.type === "get_chunk") { - return response( - request, - { type: "chunk", address: request.address, size: content.length }, - content, - ); - } - throw new Error(`Unexpected storage request ${request.type}`); - }, - ], - ]); +test("the browser offer uses the Saorsa credential as ufrag and password", () => { + const credential = `saorsa+webrtc+v1/${"b".repeat(32)}`; + const offer = mungeOfferIceCredentials( + { + type: "offer", + sdp: "v=0\r\na=ice-ufrag:browser-generated\r\na=ice-pwd:browser-secret\r\n", + }, + credential, + ); + assert.match( + offer.sdp, + new RegExp(`a=ice-ufrag:${credential.replaceAll("+", "\\+")}`), + ); + assert.match( + offer.sdp, + new RegExp(`a=ice-pwd:${credential.replaceAll("+", "\\+")}`), + ); +}); - const previousWebTransport = globalThis.WebTransport; - globalThis.WebTransport = mockWebTransport(routes); - t.after(() => { - globalThis.WebTransport = previousWebTransport; +test("the browser reuses a bounded pool of WebRTC node connections", async () => { + const endpoints = [0x11, 0x22, 0x33].map((hashByte, index) => + webrtc_directMultiaddr( + "ip4", + "127.0.0.1", + 24000 + index, + (index + 1).toString(16).padStart(2, "0").repeat(32), + hashByte, + ), + ); + const created = []; + const closed = []; + const pool = new BrowserNodeClientPool({ + maxClients: 2, + clientFactory: (endpoint) => { + const client = { endpoint, close: () => closed.push(endpoint) }; + created.push(client); + return client; + }, }); - const downloaded = await getChunkFromClosest([seed.multiaddr], address); - assert.deepEqual(downloaded.content, content); - assert.equal(downloaded.hash, address); - assert.equal(downloaded.node.peer_id, storagePeer); - assert.deepEqual(calls, [ - ["seed", "hello"], - ["seed", "find_node"], - ["storage", "hello"], - ["storage", "find_node"], - ["storage", "get_chunk"], - ]); -}); - -test("paid PUT frames the encrypted chunk as a binary request body", async (t) => { - const content = new TextEncoder().encode("encrypted record"); - const address = bytesToHex(blake3(content)); - const node = endpoint("storage.test", "33".repeat(32), 0x44); - const transactionHash = "55".repeat(32); - const quote = { quote_hash: "66".repeat(32) }; - let observed; - const routes = new Map([ - [ - node.url, - (request) => { - if (request.type === "put_chunk") { - observed = request; - return response(request, { - type: "chunk_stored", - address, - already_stored: false, - }); - } - throw new Error(`Unexpected request ${request.type}`); - }, - ], - ]); - const previousWebTransport = globalThis.WebTransport; - globalThis.WebTransport = mockWebTransport(routes); - t.after(() => { - globalThis.WebTransport = previousWebTransport; + let firstClient; + await pool.withClient(endpoints[0], async (client) => { + firstClient = client; }); + await pool.withClient(endpoints[0], async (client) => { + assert.equal(client, firstClient); + }); + await pool.withClient(endpoints[1], async () => {}); + await pool.withClient(endpoints[2], async () => {}); - const client = new BrowserNodeClient(node.multiaddr); - const stored = await client.putChunk(address, content, quote, transactionHash); - assert.deepEqual(stored, { address, alreadyStored: false }); - assert.equal(observed.content_length, content.length); - assert.deepEqual(observed.content, content); - assert.deepEqual(observed.quote, quote); - assert.equal(observed.transaction_hash, transactionHash); + assert.equal(created.length, 3); + assert.equal(pool.size, 2); + assert.deepEqual(closed, [endpoints[0]]); + pool.close(); + assert.equal(closed.length, 3); }); -function browserNode(endpoint) { - return { - peer_id: endpoint.peer_id, - native_addresses: [], - reliability: 1, - webtransport: { - multiaddr: endpoint.multiaddr, +test("the WebRTC client pool waits instead of exceeding its connection cap", async () => { + const firstEndpoint = webrtc_directMultiaddr( + "ip4", + "127.0.0.1", + 24000, + "11".repeat(32), + 0x11, + ); + const secondEndpoint = webrtc_directMultiaddr( + "ip4", + "127.0.0.1", + 24001, + "22".repeat(32), + 0x22, + ); + const created = []; + let releaseFirst; + let secondEntered = false; + const pool = new BrowserNodeClientPool({ + maxClients: 1, + clientFactory: (endpoint) => { + const client = { endpoint, close() {} }; + created.push(client); + return client; }, - }; -} + }); + const first = pool.withClient( + firstEndpoint, + () => new Promise((resolve) => (releaseFirst = resolve)), + ); + const second = pool.withClient(secondEndpoint, async () => { + secondEntered = true; + }); -function helloResponse(request, endpoint) { - return response(request, { + await Promise.resolve(); + await Promise.resolve(); + assert.equal(created.length, 1); + assert.equal(secondEntered, false); + releaseFirst(); + await Promise.all([first, second]); + assert.equal(created.length, 2); + assert.equal(secondEntered, true); + pool.close(); +}); + +test("HELLO binds the persistent ANT identity to the WebRTC endpoint", () => { + const challenge = new Uint8Array(32).fill(0x22); + const { publicKey, secretKey } = ml_dsa65.keygen( + new Uint8Array(32).fill(0x33), + ); + const peerId = bytesToHex(blake3(publicKey)); + const multiaddr = webrtc_directMultiaddr( + "ip4", + "127.0.0.1", + 24000, + peerId, + 0x44, + ); + const transcript = concatBytes( + new TextEncoder().encode("autonomi-webrtc-direct-hello-v1\0"), + challenge, + new TextEncoder().encode(peerId), + new TextEncoder().encode(multiaddr), + ); + const header = { type: "hello", protocol: "autonomi.web.poc.v3", - peer_id: endpoint.peer_id, - max_chunk_size: 4 * 1024 * 1024, - endpoint: { - multiaddr: endpoint.multiaddr, - }, - capabilities: ["find_node", "get_chunk"], - }); -} - -function response(request, fields, content = new Uint8Array()) { - return { - header: { - version: 3, - request_id: request.request_id, - status: "ok", - content_length: content.length, - ...fields, - }, - content, + challenge: bytesToHex(challenge), + peer_id: peerId, + public_key: bytesToHex(publicKey), + signature: bytesToHex(ml_dsa65.sign(transcript, secretKey)), + endpoint: { multiaddr }, }; -} + + assert.equal(verifyHelloIdentity(header, multiaddr, challenge), peerId); + assert.throws( + () => + verifyHelloIdentity( + { ...header, challenge: "00".repeat(32) }, + multiaddr, + challenge, + ), + /different HELLO challenge/, + ); +}); + +test("BLAKE3 verification accepts the canonical empty hash", () => { + const emptyHash = + "af1349b9f5f9a1a6a0404dea36dcc9499bcb25c9adc112b7cc9a93cae41f3262"; + assert.equal(verifyChunk(emptyHash, new Uint8Array()), emptyHash); + assert.throws( + () => verifyChunk("00".repeat(32), new Uint8Array()), + /BLAKE3 mismatch/, + ); +}); function certificateMultihash(byte) { const multihash = Uint8Array.from([0x12, 0x20, ...Array(32).fill(byte)]); return `u${Buffer.from(multihash).toString("base64url")}`; } -function webtransportMultiaddr(hostProtocol, host, port, peerId, hashBytes) { - const hashes = hashBytes - .map((byte) => `/certhash/${certificateMultihash(byte)}`) - .join(""); - return `/${hostProtocol}/${host}/udp/${port}/quic-v1/webtransport${hashes}/p2p/${peerId}`; -} - -function endpoint(host, peerId, certificateByte) { - return { - peer_id: peerId, - url: `https://${host}/autonomi/webtransport/v1`, - multiaddr: webtransportMultiaddr("dns", host, 443, peerId, [certificateByte]), - }; -} - -function encodeResponse({ header, content }) { - const headerBytes = new TextEncoder().encode(JSON.stringify(header)); - const frame = new Uint8Array(4 + headerBytes.length + content.length); - new DataView(frame.buffer).setUint32(0, headerBytes.length, false); - frame.set(headerBytes, 4); - frame.set(content, 4 + headerBytes.length); - return frame; +function concatBytes(...parts) { + const output = new Uint8Array( + parts.reduce((length, part) => length + part.length, 0), + ); + let offset = 0; + for (const part of parts) { + output.set(part, offset); + offset += part.length; + } + return output; } -function mockWebTransport(routes) { - return class MockWebTransport { - constructor(url, options) { - this.url = url; - this.options = options; - this.ready = routes.has(url) - ? Promise.resolve() - : Promise.reject(new Error(`No mock endpoint for ${url}`)); - } - - async createBidirectionalStream() { - const requestChunks = []; - let responseController; - const readable = new ReadableStream({ - start(controller) { - responseController = controller; - }, - }); - const writable = new WritableStream({ - write(chunk) { - requestChunks.push(chunk); - }, - close: () => { - try { - const length = requestChunks.reduce((total, chunk) => total + chunk.length, 0); - const encoded = new Uint8Array(length); - let offset = 0; - for (const chunk of requestChunks) { - encoded.set(chunk, offset); - offset += chunk.length; - } - if (encoded.length < 4) throw new Error("Request omitted its frame prefix"); - const headerLength = new DataView( - encoded.buffer, - encoded.byteOffset, - encoded.byteLength, - ).getUint32(0, false); - const contentOffset = 4 + headerLength; - const request = JSON.parse( - new TextDecoder().decode(encoded.subarray(4, contentOffset)), - ); - request.content = encoded.slice(contentOffset); - if (request.content.length !== request.content_length) { - throw new Error("Request content length mismatch"); - } - const handler = routes.get(this.url); - responseController.enqueue(encodeResponse(handler(request))); - responseController.close(); - } catch (error) { - responseController.error(error); - } - }, - }); - return { readable, writable }; - } - - close() {} - }; +function webrtc_directMultiaddr(hostProtocol, host, port, peerId, hashByte) { + return `/${hostProtocol}/${host}/udp/${port}/webrtc-direct/certhash/${certificateMultihash(hashByte)}/p2p/${peerId}`; } diff --git a/web/src/upload.js b/web/src/upload.js index f5bff9ce..45cdf7d1 100644 --- a/web/src/upload.js +++ b/web/src/upload.js @@ -1,5 +1,5 @@ import { encryptPublicFile as encryptPublicFileNative } from "../pkg/ant_core.js"; -import { BrowserNodeClient, iterativeFindClosest } from "./protocol.js"; +import { BrowserNodeClientPool, iterativeFindClosest } from "./protocol.js"; import { payForStorageQuotes, verifyStorageQuote } from "./payment.js"; export const MAX_BROWSER_UPLOAD_BYTES = 64 * 1024 * 1024; @@ -13,10 +13,14 @@ export async function encryptPublicFile( encrypt = encryptPublicFileNative, } = {}, ) { - if (!(content instanceof Uint8Array)) throw new Error("Upload content must be bytes"); - if (content.length < 3) throw new Error("Self-encryption requires a file of at least 3 bytes"); + if (!(content instanceof Uint8Array)) + throw new Error("Upload content must be bytes"); + if (content.length < 3) + throw new Error("Self-encryption requires a file of at least 3 bytes"); if (content.length > MAX_BROWSER_UPLOAD_BYTES) { - throw new Error(`Browser uploads are limited to ${MAX_BROWSER_UPLOAD_BYTES} bytes`); + throw new Error( + `Browser uploads are limited to ${MAX_BROWSER_UPLOAD_BYTES} bytes`, + ); } const encrypted = encrypt(content); return { @@ -34,10 +38,6 @@ export async function encryptPublicFile( }; } -function closeClients(clients) { - for (const client of clients.values()) client.close(); -} - function assertUploadNode(hello, paymentNetwork) { if ( !Array.isArray(hello.capabilities) || @@ -49,64 +49,92 @@ function assertUploadNode(hello, paymentNetwork) { const advertised = hello.payment; if ( !advertised || - new URL(advertised.rpc_url).toString() !== new URL(paymentNetwork.rpc_url).toString() || + new URL(advertised.rpc_url).toString() !== + new URL(paymentNetwork.rpc_url).toString() || advertised.payment_token_address?.toLowerCase() !== paymentNetwork.payment_token_address.toLowerCase() || advertised.payment_vault_address?.toLowerCase() !== paymentNetwork.payment_vault_address.toLowerCase() ) { - throw new Error("Node advertises a different payment network than the manifest"); + throw new Error( + "Node advertises a different payment network than the manifest", + ); } } -async function prepareRecord(seedEndpoints, paymentNetwork, record, onProgress) { +async function prepareRecord( + seedEndpoints, + paymentNetwork, + record, + clientPool, + onProgress, +) { onProgress(`Finding closest nodes for ${record.address}`); - const lookup = await iterativeFindClosest(seedEndpoints, record.address, { onProgress }); + const lookup = await iterativeFindClosest(seedEndpoints, record.address, { + onProgress, + clientPool, + }); const endpoints = lookup.nodes - .filter((node) => node.webtransport) + .filter((node) => node.webrtc_direct) .slice(0, MAX_STORE_TARGETS) - .map((node) => ({ peerId: node.peer_id, endpoint: node.webtransport })); - try { - const failures = []; - for (const target of endpoints) { - const client = new BrowserNodeClient(target.endpoint); - try { - assertUploadNode(await client.hello(), paymentNetwork); - const response = await client.quoteChunk(record.address, record.content.length); - const verified = verifyStorageQuote( - response.quote, - record.address, - target.peerId, - ); - if (response.alreadyStored) { - onProgress(`Chunk ${record.address} is already stored; skipping payment`); - return { record, alreadyStored: true, targets: endpoints }; - } - onProgress(`Verified storage quote ${verified.quoteHash} from ${target.peerId}`); - return { - record, - alreadyStored: false, - targets: [target, ...endpoints.filter((candidate) => candidate !== target)], - verified, - }; - } catch (error) { - failures.push(`${target.peerId}: ${error.message}`); - } finally { - client.close(); - } + .map((node) => ({ peerId: node.peer_id, endpoint: node.webrtc_direct })); + const failures = []; + for (const target of endpoints) { + try { + const prepared = await clientPool.withClient( + target.endpoint, + async (client) => { + assertUploadNode(await client.hello(), paymentNetwork); + const response = await client.quoteChunk( + record.address, + record.content.length, + ); + const verified = verifyStorageQuote( + response.quote, + record.address, + target.peerId, + ); + if (response.alreadyStored) { + onProgress( + `Chunk ${record.address} is already stored; skipping payment`, + ); + return { record, alreadyStored: true, targets: endpoints }; + } + onProgress( + `Verified storage quote ${verified.quoteHash} from ${target.peerId}`, + ); + return { + record, + alreadyStored: false, + targets: [ + target, + ...endpoints.filter((candidate) => candidate !== target), + ], + verified, + }; + }, + ); + return prepared; + } catch (error) { + failures.push(`${target.peerId}: ${error.message}`); } - throw new Error(`No closest node supplied a valid quote (${failures.join("; ")})`); - } finally { - closeClients(lookup.clients); } + throw new Error( + `No closest node supplied a valid quote (${failures.join("; ")})`, + ); } -async function storePrepared(prepared, paymentNetwork, transactionHash, onProgress) { +async function storePrepared( + prepared, + paymentNetwork, + transactionHash, + clientPool, + onProgress, +) { if (prepared.alreadyStored) return 1; const attempts = await Promise.allSettled( prepared.targets.map(async (target) => { - const client = new BrowserNodeClient(target.endpoint); - try { + return clientPool.withClient(target.endpoint, async (client) => { assertUploadNode(await client.hello(), paymentNetwork); const result = await client.putChunk( prepared.record.address, @@ -118,17 +146,19 @@ async function storePrepared(prepared, paymentNetwork, transactionHash, onProgre `${result.alreadyStored ? "Confirmed" : "Stored"} ${prepared.record.address} on ${target.peerId}`, ); return result; - } finally { - client.close(); - } + }); }), ); - const stored = attempts.filter((attempt) => attempt.status === "fulfilled").length; + const stored = attempts.filter( + (attempt) => attempt.status === "fulfilled", + ).length; if (stored === 0) { const failures = attempts .filter((attempt) => attempt.status === "rejected") .map((attempt) => attempt.reason?.message ?? String(attempt.reason)); - throw new Error(`Paid chunk was rejected by every closest node: ${failures.join("; ")}`); + throw new Error( + `Paid chunk was rejected by every closest node: ${failures.join("; ")}`, + ); } return stored; } @@ -141,48 +171,62 @@ export async function uploadPublicFile( { onProgress = () => {}, encrypt = encryptPublicFileNative } = {}, ) { const content = new Uint8Array(await file.arrayBuffer()); - onProgress(`Self-encrypting ${file.name} with native ant-core WASM (${content.length.toLocaleString()} bytes)`); + onProgress( + `Self-encrypting ${file.name} with native ant-core WASM (${content.length.toLocaleString()} bytes)`, + ); const encrypted = await encryptPublicFile(content, { name: file.name, contentType: file.type, encrypt, }); - const prepared = []; - for (let index = 0; index < encrypted.records.length; index += 1) { - onProgress(`Preparing record ${index + 1}/${encrypted.records.length}`); - prepared.push( - await prepareRecord(seedEndpoints, paymentNetwork, encrypted.records[index], onProgress), - ); - } - const payable = prepared.filter((record) => !record.alreadyStored); - let transactionHash; - let totalAmount = 0n; - if (payable.length > 0) { - const payment = await payForStorageQuotes( - paymentNetwork, - payable.map((record) => record.verified), - walletSecret, - { onProgress }, - ); - transactionHash = payment.transactionHash; - totalAmount = payment.totalAmount; - } - let replicas = Number.POSITIVE_INFINITY; - for (let index = 0; index < prepared.length; index += 1) { - onProgress(`Storing record ${index + 1}/${prepared.length}`); - const stored = await storePrepared( - prepared[index], - paymentNetwork, + const clientPool = new BrowserNodeClientPool(); + try { + const prepared = []; + for (let index = 0; index < encrypted.records.length; index += 1) { + onProgress(`Preparing record ${index + 1}/${encrypted.records.length}`); + prepared.push( + await prepareRecord( + seedEndpoints, + paymentNetwork, + encrypted.records[index], + clientPool, + onProgress, + ), + ); + } + const payable = prepared.filter((record) => !record.alreadyStored); + let transactionHash; + let totalAmount = 0n; + if (payable.length > 0) { + const payment = await payForStorageQuotes( + paymentNetwork, + payable.map((record) => record.verified), + walletSecret, + { onProgress }, + ); + transactionHash = payment.transactionHash; + totalAmount = payment.totalAmount; + } + let replicas = Number.POSITIVE_INFINITY; + for (let index = 0; index < prepared.length; index += 1) { + onProgress(`Storing record ${index + 1}/${prepared.length}`); + const stored = await storePrepared( + prepared[index], + paymentNetwork, + transactionHash, + clientPool, + onProgress, + ); + replicas = Math.min(replicas, stored); + } + encrypted.descriptor.replicas = Number.isFinite(replicas) ? replicas : 0; + return { + file: encrypted.descriptor, transactionHash, - onProgress, - ); - replicas = Math.min(replicas, stored); + storageCostAtto: totalAmount.toString(), + records: encrypted.records.length, + }; + } finally { + clientPool.close(); } - encrypted.descriptor.replicas = Number.isFinite(replicas) ? replicas : 0; - return { - file: encrypted.descriptor, - transactionHash, - storageCostAtto: totalAmount.toString(), - records: encrypted.records.length, - }; } diff --git a/web/src/wasm.test.js b/web/src/wasm.test.js index 8f376039..8aa261f2 100644 --- a/web/src/wasm.test.js +++ b/web/src/wasm.test.js @@ -19,7 +19,7 @@ function lookupNode(lastByte, stringEndpoint = false) { peer_id: `${"00".repeat(31)}${lastByte.toString(16).padStart(2, "0")}`, native_addresses: [], reliability: 1, - webtransport: stringEndpoint ? `/test/${lastByte}` : { multiaddr: `/test/${lastByte}` }, + webrtc_direct: stringEndpoint ? `/test/${lastByte}` : { multiaddr: `/test/${lastByte}` }, }; } @@ -30,7 +30,7 @@ test("generated WASM drives Saorsa's complete shared iterative lookup", async () const termination = await lookup.run(async ({ iteration, candidates }) => { batches.push(candidates.map((node) => node.peer_id)); if (iteration === 1) { - assert.equal(candidates[1].webtransport, "/test/2"); + assert.equal(candidates[1].webrtc_direct, "/test/2"); return [ { status: "succeeded", From d10567d57378ead34cf9a0027eb421485d398adc Mon Sep 17 00:00:00 2001 From: Mick van Dijke <12992260+mickvandijke@users.noreply.github.com> Date: Wed, 26 Aug 2026 13:46:09 +0200 Subject: [PATCH 07/31] refactor browser client into Rust WASM --- Cargo.lock | 19 + README.md | 8 +- ant-core/Cargo.toml | 32 +- ant-core/src/browser.rs | 162 ++- ant-core/src/browser/crypto.rs | 71 ++ ant-core/src/browser/manifest.rs | 244 ++++ ant-core/src/browser/payment.rs | 476 +++++++ ant-core/src/browser/protocol.rs | 569 +++++++++ ant-core/src/browser/wasm_transport.rs | 1622 ++++++++++++++++++++++++ ant-core/src/lib.rs | 2 +- web/README.md | 63 +- web/package-lock.json | 74 +- web/package.json | 3 - web/src/file.js | 97 -- web/src/file.test.js | 60 - web/src/main.js | 87 +- web/src/manifest.js | 126 +- web/src/manifest.test.js | 2 +- web/src/payment.js | 263 +--- web/src/payment.test.js | 235 ---- web/src/protocol.js | 1025 --------------- web/src/protocol.test.js | 284 +---- web/src/upload.js | 232 ---- web/src/upload.test.js | 38 - 24 files changed, 3334 insertions(+), 2460 deletions(-) create mode 100644 ant-core/src/browser/crypto.rs create mode 100644 ant-core/src/browser/manifest.rs create mode 100644 ant-core/src/browser/payment.rs create mode 100644 ant-core/src/browser/protocol.rs create mode 100644 ant-core/src/browser/wasm_transport.rs delete mode 100644 web/src/file.js delete mode 100644 web/src/file.test.js delete mode 100644 web/src/payment.test.js delete mode 100644 web/src/protocol.js delete mode 100644 web/src/upload.js delete mode 100644 web/src/upload.test.js diff --git a/Cargo.lock b/Cargo.lock index 2a43938a..6c34de84 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -843,15 +843,19 @@ dependencies = [ "anyhow", "async-stream", "axum", + "base64", "blake3", "bytes", "console_error_panic_hook", + "fips204", "flate2", "fs2", "futures", + "futures-channel", "futures-core", "futures-util", "getrandom 0.2.17", + "gloo-timers", "hex", "js-sys", "libc", @@ -874,6 +878,7 @@ dependencies = [ "tar", "tempfile", "thiserror 2.0.18", + "tiny-keccak", "tokio", "tokio-test", "tokio-util", @@ -881,9 +886,11 @@ dependencies = [ "tower-http", "tracing", "tracing-subscriber", + "url", "utoipa", "wasm-bindgen", "wasm-bindgen-futures", + "web-sys", "windows-sys 0.61.2", "xor_name", "zip", @@ -2929,6 +2936,18 @@ version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0cc23270f6e1808e30a928bdc84dea0b9b4136a8bc82338574f23baf47bbd280" +[[package]] +name = "gloo-timers" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbb143cf96099802033e0d4f4963b19fd2e0b728bcf076cd9cf7f6634f092994" +dependencies = [ + "futures-channel", + "futures-core", + "js-sys", + "wasm-bindgen", +] + [[package]] name = "group" version = "0.13.0" diff --git a/README.md b/README.md index 75bac8fc..f96ecc9d 100644 --- a/README.md +++ b/README.md @@ -6,11 +6,11 @@ A unified CLI and Rust library for storing data on the Autonomi decentralized ne This project provides two Rust crates and a browser client: -- **ant-core** — A headless Rust library containing all business logic: data storage/retrieval with self-encryption and EVM payments, node lifecycle management, and local devnet tooling. Its portable immutable-data core also builds for browsers with the `browser-wasm` feature. +- **ant-core** — A headless Rust library containing the Autonomi business logic: data storage/retrieval with self-encryption and EVM payments, node lifecycle management, and local devnet tooling. Its `browser` module shares manifest validation, protocol framing, authenticated WebRTC Direct access, lookup, quote verification, and complete public-file workflows with browser applications through the `browser-wasm` feature. - **ant-cli** — A thin CLI binary (`ant`) built on `ant-core`. -- **web** — A direct WebRTC Direct client and test site. It drives Saorsa's - shared iterative lookup engine and uses `ant-core` through WASM to - self-encrypt and reconstruct complete public files without a data gateway. +- **web** — A thin browser UI around `ant-core`'s Rust/WASM client. JavaScript + is limited to DOM/file-save integration and submitting the already-verified + payment plan with Ethers; no gateway proxies Autonomi data. Data on Autonomi is **content-addressed**. Files are split into encrypted chunks (via [self-encryption](https://en.wikipedia.org/wiki/Convergent_encryption)), each stored at an XOR address derived from its content. A `DataMap` tracks which chunks belong to a file. Payments for storage are made on an EVM-compatible blockchain (Arbitrum). diff --git a/ant-core/Cargo.toml b/ant-core/Cargo.toml index 7ebfd32a..48721471 100644 --- a/ant-core/Cargo.toml +++ b/ant-core/Cargo.toml @@ -15,14 +15,19 @@ crate-type = ["rlib", "cdylib"] # the browser-WASM target; native networking and node management stay behind # the `native` feature below. blake3 = "1" +base64 = "0.22" bytes = "1" hex = "0.4" +postcard = { version = "1.1.3", features = ["use-std"] } rmp-serde = "1" serde = { version = "1", features = ["derive"] } serde_bytes = "0.11" +serde_json = "1" self_encryption = "0.36" saorsa-dht-lookup = { version = "0.1.0", path = "../../saorsa-core-web-support/crates/saorsa-dht-lookup" } thiserror = "2" +tiny-keccak = { version = "2", features = ["keccak"] } +url = "2" # Node management async-stream = { version = "0.3", optional = true } @@ -30,10 +35,10 @@ axum = { version = "0.8", optional = true } flate2 = { version = "1", optional = true } fs2 = { version = "0.4", optional = true } futures-core = { version = "0.3", optional = true } +futures-channel = { version = "0.3", optional = true } futures-util = { version = "0.3", optional = true } self-replace = { version = "1", optional = true } semver = { version = "1", optional = true } -serde_json = { version = "1", optional = true } reqwest = { version = "0.12", features = ["json", "stream"], optional = true } tar = { version = "0.4", optional = true } tempfile = { version = "3", optional = true } @@ -53,7 +58,6 @@ tower-http = { version = "0.6.8", features = ["cors"], optional = true } ant-protocol = { version = "2.3.1", optional = true } xor_name = { version = "5", optional = true } futures = { version = "0.3", optional = true } -postcard = { version = "1.1.3", features = ["use-std"], optional = true } tracing = { version = "0.1", optional = true } lru = { version = "0.16", optional = true } rand = { version = "0.8", optional = true } @@ -83,6 +87,23 @@ js-sys = { version = "0.3", optional = true } serde-wasm-bindgen = { version = "0.6", optional = true } wasm-bindgen = { version = "0.2", optional = true } wasm-bindgen-futures = { version = "0.4", optional = true } +web-sys = { version = "0.3", optional = true, features = [ + "Event", + "MessageEvent", + "RtcConfiguration", + "RtcDataChannel", + "RtcDataChannelInit", + "RtcDataChannelState", + "RtcDataChannelType", + "RtcPeerConnection", + "RtcSdpType", + "RtcSessionDescriptionInit", +] } +gloo-timers = { version = "0.3", features = ["futures"], optional = true } +# `ant-protocol` currently pulls native Tokio networking on wasm32. Use the +# same FIPS-204 primitive as `saorsa-pqc` for verification-only browser builds +# until ant-protocol's pure wire layer can be built independently of transport. +fips204 = { version = "0.4.6", default-features = false, features = ["ml-dsa-65"], optional = true } [target.'cfg(target_arch = "wasm32")'.dependencies] # self_encryption/rand use getrandom 0.2. Browser entropy is supplied by the @@ -114,12 +135,10 @@ native = [ "dep:futures-core", "dep:futures-util", "dep:lru", - "dep:postcard", "dep:rand", "dep:reqwest", "dep:self-replace", "dep:semver", - "dep:serde_json", "dep:sysinfo", "dep:tar", "dep:tempfile", @@ -135,10 +154,15 @@ native = [ ] browser-wasm = [ "dep:console_error_panic_hook", + "dep:futures-channel", + "dep:futures-util", + "dep:gloo-timers", "dep:js-sys", + "dep:fips204", "dep:serde-wasm-bindgen", "dep:wasm-bindgen", "dep:wasm-bindgen-futures", + "dep:web-sys", ] # Enable `LocalDevnet` (ant-core/src/node/devnet.rs) which wraps # `ant_node::devnet::Devnet` and an Anvil EVM testnet. diff --git a/ant-core/src/browser.rs b/ant-core/src/browser.rs index 899b7b53..8cf4f6e1 100644 --- a/ant-core/src/browser.rs +++ b/ant-core/src/browser.rs @@ -1,9 +1,30 @@ -//! Browser-safe immutable-data primitives. +//! Cross-platform Autonomi client logic and browser bindings. //! -//! This module deliberately contains no transport, filesystem, Tokio runtime, -//! or EVM provider. It is the shared compatibility-sensitive core used by the -//! native client and by the browser WASM package: native self-encryption, -//! public DataMap encoding, content addressing, and reconstruction. +//! The manifest, protocol, payment, and immutable-data modules are portable +//! Rust shared by native clients and the browser WASM package. The +//! `browser-wasm` feature additionally provides the `web-sys` WebRTC Direct +//! host adapter; only DOM, file-save, and wallet transaction submission remain +//! in JavaScript. + +mod crypto; +pub mod manifest; +pub mod payment; +pub mod protocol; + +pub use manifest::{ + parse_browser_manifest, validate_browser_payment_network, BrowserManifest, + BrowserManifestEndpoint, BrowserPaymentNetwork, PublicFileDescriptor, BROWSER_MANIFEST_VERSION, +}; +pub use payment::{ + storage_payment_total, verify_storage_quote, BrowserQuoteArtifact, VerifiedStorageQuote, +}; +pub use protocol::{ + parse_webrtc_direct_multiaddr, WebRtcDirectEndpoint, BROWSER_PROTOCOL_NAME, + BROWSER_PROTOCOL_VERSION, WEBRTC_DIRECT_DATA_CHANNEL, +}; + +#[cfg(all(target_arch = "wasm32", feature = "browser-wasm"))] +mod wasm_transport; use bytes::Bytes; use self_encryption::{DataMap, EncryptedChunk}; @@ -212,6 +233,12 @@ fn chunk_infos(data_map: &DataMap) -> Vec { #[cfg(all(target_arch = "wasm32", feature = "browser-wasm"))] mod wasm { + use super::manifest::parse_browser_manifest; + use super::payment::{payment_quote_hash, verify_storage_quote, BrowserQuoteArtifact}; + use super::protocol::{ + munge_offer_ice_credentials, parse_response_frame, parse_webrtc_direct_multiaddr, + server_answer_sdp, verify_hello_identity, BrowserEndpointInput, BrowserHello, + }; use super::{content_address, decrypt_public_file, encrypt_public_file, verify_record}; use js_sys::{Array, Function, Promise, Uint8Array}; use saorsa_dht_lookup::{ @@ -223,6 +250,18 @@ mod wasm { use wasm_bindgen::prelude::*; use wasm_bindgen_futures::JsFuture; + #[derive(Debug, Deserialize)] + struct BrowserOffer { + sdp: String, + } + + #[derive(Debug, Serialize)] + struct BrowserSessionDescription { + #[serde(rename = "type")] + description_type: &'static str, + sdp: String, + } + #[derive(Debug, Clone, Serialize, Deserialize)] #[serde(untagged)] enum BrowserLookupEndpoint { @@ -494,6 +533,119 @@ mod wasm { console_error_panic_hook::set_once(); } + /// Validate and normalize a WebRTC Direct multiaddress in shared Rust. + #[wasm_bindgen(js_name = parseWebRtcDirectMultiaddr)] + pub fn parse_webrtc_direct_multiaddr_wasm(endpoint: JsValue) -> Result { + let input: BrowserEndpointInput = serde_wasm_bindgen::from_value(endpoint) + .map_err(|error| JsValue::from_str(&error.to_string()))?; + let parsed = parse_webrtc_direct_multiaddr(input.multiaddr()) + .map_err(|error| JsValue::from_str(&error.to_string()))?; + serde_wasm_bindgen::to_value(&parsed).map_err(|error| JsValue::from_str(&error.to_string())) + } + + /// Decode and bound-check a complete WebRTC browser response frame. + #[wasm_bindgen(js_name = parseResponseFrame)] + pub fn parse_response_frame_wasm(frame: &[u8]) -> Result { + let parsed = + parse_response_frame(frame).map_err(|error| JsValue::from_str(&error.to_string()))?; + parsed + .serialize(&serde_wasm_bindgen::Serializer::new().serialize_maps_as_objects(true)) + .map_err(|error| JsValue::from_str(&error.to_string())) + } + + /// Build the ICE-lite answer pinned by a WebRTC Direct endpoint. + #[wasm_bindgen(js_name = serverAnswerFromEndpoint)] + pub fn server_answer_from_endpoint_wasm( + endpoint: JsValue, + ice_credential: &str, + ) -> Result { + let input: BrowserEndpointInput = serde_wasm_bindgen::from_value(endpoint) + .map_err(|error| JsValue::from_str(&error.to_string()))?; + let endpoint = parse_webrtc_direct_multiaddr(input.multiaddr()) + .map_err(|error| JsValue::from_str(&error.to_string()))?; + let sdp = server_answer_sdp(&endpoint, ice_credential) + .map_err(|error| JsValue::from_str(&error.to_string()))?; + serde_wasm_bindgen::to_value(&BrowserSessionDescription { + description_type: "answer", + sdp, + }) + .map_err(|error| JsValue::from_str(&error.to_string())) + } + + /// Replace a browser offer's generated ICE credentials with Saorsa's profile. + #[wasm_bindgen(js_name = mungeOfferIceCredentials)] + pub fn munge_offer_ice_credentials_wasm( + offer: JsValue, + ice_credential: &str, + ) -> Result { + let offer: BrowserOffer = serde_wasm_bindgen::from_value(offer) + .map_err(|error| JsValue::from_str(&error.to_string()))?; + let sdp = munge_offer_ice_credentials(&offer.sdp, ice_credential) + .map_err(|error| JsValue::from_str(&error.to_string()))?; + serde_wasm_bindgen::to_value(&BrowserSessionDescription { + description_type: "offer", + sdp, + }) + .map_err(|error| JsValue::from_str(&error.to_string())) + } + + /// Authenticate a node HELLO against its expected endpoint and challenge. + #[wasm_bindgen(js_name = verifyHelloIdentity)] + pub fn verify_hello_identity_wasm( + hello: JsValue, + endpoint: JsValue, + challenge: &[u8], + ) -> Result { + let hello: BrowserHello = serde_wasm_bindgen::from_value(hello) + .map_err(|error| JsValue::from_str(&error.to_string()))?; + let input: BrowserEndpointInput = serde_wasm_bindgen::from_value(endpoint) + .map_err(|error| JsValue::from_str(&error.to_string()))?; + let endpoint = parse_webrtc_direct_multiaddr(input.multiaddr()) + .map_err(|error| JsValue::from_str(&error.to_string()))?; + let challenge: [u8; 32] = challenge + .try_into() + .map_err(|_| JsValue::from_str("HELLO verification requires a 32-byte challenge"))?; + verify_hello_identity(&hello, &endpoint, &challenge) + .map_err(|error| JsValue::from_str(&error.to_string())) + } + + /// Validate and normalize browser bootstrap and public-file metadata. + #[wasm_bindgen(js_name = parseBrowserManifest)] + pub fn parse_browser_manifest_wasm(value: JsValue) -> Result { + let value: serde_json::Value = serde_wasm_bindgen::from_value(value) + .map_err(|error| JsValue::from_str(&error.to_string()))?; + let manifest = + parse_browser_manifest(value).map_err(|error| JsValue::from_str(&error.to_string()))?; + serde_wasm_bindgen::to_value(&manifest) + .map_err(|error| JsValue::from_str(&error.to_string())) + } + + /// Compute the native EVM `PaymentQuote` hash. + #[wasm_bindgen(js_name = paymentQuoteHash)] + #[must_use] + pub fn payment_quote_hash_wasm( + signed_bytes: &[u8], + public_key: &[u8], + signature: &[u8], + ) -> String { + hex::encode(payment_quote_hash(signed_bytes, public_key, signature)) + } + + /// Fully verify a storage quote before exposing it to a wallet signer. + #[wasm_bindgen(js_name = verifyStorageQuote)] + pub fn verify_storage_quote_wasm( + quote: JsValue, + expected_address: &str, + expected_peer_id: &str, + ) -> Result { + let quote: BrowserQuoteArtifact = serde_wasm_bindgen::from_value(quote) + .map_err(|error| JsValue::from_str(&error.to_string()))?; + let verified = verify_storage_quote(quote, expected_address, expected_peer_id) + .map_err(|error| JsValue::from_str(&error.to_string()))?; + serde_wasm_bindgen::to_value(&verified) + .map_err(|error| JsValue::from_str(&error.to_string())) + } + /// Native `self_encryption` plus public DataMap generation. #[wasm_bindgen(js_name = encryptPublicFile)] pub fn encrypt_public_file_wasm(content: &[u8]) -> Result { diff --git a/ant-core/src/browser/crypto.rs b/ant-core/src/browser/crypto.rs new file mode 100644 index 00000000..acdb442b --- /dev/null +++ b/ant-core/src/browser/crypto.rs @@ -0,0 +1,71 @@ +#[cfg(feature = "native")] +use ant_protocol::pqc::api::{ml_dsa_65, MlDsaPublicKey, MlDsaSignature, MlDsaVariant}; +#[cfg(all(not(feature = "native"), feature = "browser-wasm"))] +use fips204::{ + ml_dsa_65, + traits::{SerDes as _, Verifier as _}, +}; +use tiny_keccak::{Hasher as _, Keccak}; + +pub(crate) fn verify_ml_dsa_65( + public_key: &[u8], + signature: &[u8], + message: &[u8], + context: &[u8], +) -> bool { + verify_ml_dsa_65_inner(public_key, signature, message, context).unwrap_or(false) +} + +#[cfg(feature = "native")] +fn verify_ml_dsa_65_inner( + public_key: &[u8], + signature: &[u8], + message: &[u8], + context: &[u8], +) -> Result { + let public_key = MlDsaPublicKey::from_bytes(MlDsaVariant::MlDsa65, public_key) + .map_err(|error| error.to_string())?; + let signature = MlDsaSignature::from_bytes(MlDsaVariant::MlDsa65, signature) + .map_err(|error| error.to_string())?; + ml_dsa_65() + .verify_with_context(&public_key, message, &signature, context) + .map_err(|error| error.to_string()) +} + +#[cfg(all(not(feature = "native"), feature = "browser-wasm"))] +fn verify_ml_dsa_65_inner( + public_key: &[u8], + signature: &[u8], + message: &[u8], + context: &[u8], +) -> Result { + let public_key: [u8; ml_dsa_65::PK_LEN] = public_key + .try_into() + .map_err(|_| "invalid ML-DSA-65 public key length".to_string())?; + let signature: [u8; ml_dsa_65::SIG_LEN] = signature + .try_into() + .map_err(|_| "invalid ML-DSA-65 signature length".to_string())?; + let public_key = + ml_dsa_65::PublicKey::try_from_bytes(public_key).map_err(ToString::to_string)?; + Ok(public_key.verify(message, &signature, context)) +} + +#[cfg(not(any(feature = "native", feature = "browser-wasm")))] +fn verify_ml_dsa_65_inner( + _public_key: &[u8], + _signature: &[u8], + _message: &[u8], + _context: &[u8], +) -> Result { + Err("ML-DSA verification requires the native or browser-wasm feature".to_string()) +} + +pub(crate) fn keccak256(parts: &[&[u8]]) -> [u8; 32] { + let mut hasher = Keccak::v256(); + for part in parts { + hasher.update(part); + } + let mut output = [0u8; 32]; + hasher.finalize(&mut output); + output +} diff --git a/ant-core/src/browser/manifest.rs b/ant-core/src/browser/manifest.rs new file mode 100644 index 00000000..47a3c93b --- /dev/null +++ b/ant-core/src/browser/manifest.rs @@ -0,0 +1,244 @@ +//! Cross-platform validation for browser bootstrap and public-file metadata. + +use super::protocol::{normalize_hex, parse_webrtc_direct_multiaddr, BrowserEndpoint}; +use super::BrowserChunkInfo; +use serde::{Deserialize, Serialize}; + +/// Current browser testnet manifest version. +pub const BROWSER_MANIFEST_VERSION: u16 = 5; +const MAX_DATA_MAP_BYTES: usize = 4 * 1024 * 1024; +const MAX_FILE_CHUNKS: usize = 1024; + +/// A validated WebRTC Direct bootstrap endpoint. +pub type BrowserManifestEndpoint = BrowserEndpoint; + +/// Public EVM configuration required to pay for storage. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct BrowserPaymentNetwork { + /// HTTP(S) JSON-RPC endpoint. + pub rpc_url: String, + /// ERC-20 payment token contract. + pub payment_token_address: String, + /// Autonomi payment vault contract. + pub payment_vault_address: String, +} + +/// Complete public-file metadata shared by native tooling and the web client. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct PublicFileDescriptor { + /// Display and save-as filename. + pub name: String, + /// Public DataMap content address. + pub address: String, + /// Plaintext file size. + pub size: usize, + /// Browser MIME type. + pub content_type: String, + /// Whole-file plaintext BLAKE3 hash. + pub blake3: String, + /// Encoded public DataMap size. + pub data_map_size: usize, + /// Self-encryption chunk descriptors. + pub chunks: Vec, + /// Minimum confirmed record replica count. + pub replicas: usize, +} + +/// Validated bootstrap, payment, and public-file description. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct BrowserManifest { + /// Manifest schema version. + pub version: u16, + /// Network instance identifier. + pub network_id: String, + /// Optional manifest creation timestamp. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub created_at: Option, + /// Stable WebRTC Direct bootstrap addresses. + pub endpoints: Vec, + /// Storage payment configuration. + pub payment: BrowserPaymentNetwork, + /// Known public files offered by the manifest. + #[serde(default)] + pub files: Vec, +} + +/// Manifest validation error. +#[derive(Debug, thiserror::Error)] +#[error("invalid browser manifest: {0}")] +pub struct BrowserManifestError(pub String); + +/// Decode, validate, and normalize an untrusted browser manifest. +pub fn parse_browser_manifest( + value: serde_json::Value, +) -> Result { + let mut manifest: BrowserManifest = + serde_json::from_value(value).map_err(|error| BrowserManifestError(error.to_string()))?; + if manifest.version != BROWSER_MANIFEST_VERSION { + return Err(BrowserManifestError(format!( + "unsupported browser manifest version {}", + manifest.version + ))); + } + if manifest.network_id.is_empty() { + return Err(BrowserManifestError( + "browser manifest has no network ID".to_string(), + )); + } + if manifest.endpoints.is_empty() { + return Err(BrowserManifestError( + "browser manifest contains no WebRtcDirect endpoints".to_string(), + )); + } + for endpoint in &mut manifest.endpoints { + let parsed = parse_webrtc_direct_multiaddr(&endpoint.multiaddr) + .map_err(|error| BrowserManifestError(error.to_string()))?; + endpoint.multiaddr = parsed.multiaddr; + } + manifest.payment = validate_browser_payment_network(manifest.payment)?; + for file in &mut manifest.files { + normalize_file(file)?; + } + Ok(manifest) +} + +/// Validate and normalize payment configuration supplied independently of a +/// manifest, such as to the browser network client WASM binding. +pub fn validate_browser_payment_network( + mut payment: BrowserPaymentNetwork, +) -> Result { + let mut rpc_url = url::Url::parse(&payment.rpc_url) + .map_err(|error| BrowserManifestError(format!("payment RPC URL is invalid: {error}")))?; + if !matches!(rpc_url.scheme(), "http" | "https") { + return Err(BrowserManifestError( + "payment RPC URL must use HTTP or HTTPS".to_string(), + )); + } + if !rpc_url.username().is_empty() || rpc_url.password().is_some() { + return Err(BrowserManifestError( + "payment RPC URL must not contain credentials".to_string(), + )); + } + if rpc_url.path().is_empty() { + rpc_url.set_path("/"); + } + payment.rpc_url = rpc_url.to_string(); + payment.payment_token_address = format!( + "0x{}", + normalize_hex(&payment.payment_token_address, 20).map_err(BrowserManifestError)? + ); + payment.payment_vault_address = format!( + "0x{}", + normalize_hex(&payment.payment_vault_address, 20).map_err(BrowserManifestError)? + ); + Ok(payment) +} + +fn normalize_file(file: &mut PublicFileDescriptor) -> Result<(), BrowserManifestError> { + if file.name.is_empty() { + return Err(BrowserManifestError( + "browser manifest file has no name".to_string(), + )); + } + file.address = normalize_hex(&file.address, 32).map_err(BrowserManifestError)?; + if !(self_encryption::MIN_ENCRYPTABLE_BYTES..=super::MAX_BROWSER_FILE_BYTES) + .contains(&file.size) + { + return Err(BrowserManifestError(format!( + "invalid public file size {}", + file.size + ))); + } + file.blake3 = normalize_hex(&file.blake3, 32).map_err(BrowserManifestError)?; + if !(1..=MAX_DATA_MAP_BYTES).contains(&file.data_map_size) { + return Err(BrowserManifestError(format!( + "invalid DataMap size {}", + file.data_map_size + ))); + } + if !(3..=MAX_FILE_CHUNKS).contains(&file.chunks.len()) { + return Err(BrowserManifestError( + "public file has an invalid self-encryption chunk list".to_string(), + )); + } + file.chunks.sort_by_key(|chunk| chunk.index); + let mut reconstructed_size = 0usize; + for (expected_index, chunk) in file.chunks.iter_mut().enumerate() { + if chunk.index != expected_index { + return Err(BrowserManifestError( + "file chunk indices must be contiguous from zero".to_string(), + )); + } + chunk.dst_hash = normalize_hex(&chunk.dst_hash, 32).map_err(BrowserManifestError)?; + chunk.src_hash = normalize_hex(&chunk.src_hash, 32).map_err(BrowserManifestError)?; + if chunk.src_size == 0 { + return Err(BrowserManifestError(format!( + "invalid plaintext chunk size {}", + chunk.src_size + ))); + } + reconstructed_size = reconstructed_size + .checked_add(chunk.src_size) + .ok_or_else(|| BrowserManifestError("file size overflow".to_string()))?; + } + if reconstructed_size != file.size { + return Err(BrowserManifestError(format!( + "file chunk sizes total {reconstructed_size}, expected {}", + file.size + ))); + } + if file.content_type.is_empty() { + file.content_type = "application/octet-stream".to_string(); + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + use base64::engine::general_purpose::URL_SAFE_NO_PAD; + use base64::Engine as _; + + fn endpoint() -> String { + let mut multihash = vec![0x12, 0x20]; + multihash.extend([0xbb; 32]); + format!( + "/ip4/127.0.0.1/udp/22000/webrtc-direct/certhash/u{}/p2p/{}", + URL_SAFE_NO_PAD.encode(multihash), + "AA".repeat(32) + ) + } + + #[test] + fn validates_and_normalizes_manifest() { + let value = serde_json::json!({ + "version": 5, + "network_id": "local-test", + "created_at": "2026-08-03T00:00:00Z", + "payment": { + "rpc_url": "http://127.0.0.1:8545", + "payment_token_address": format!("0x{}", "11".repeat(20)), + "payment_vault_address": format!("0x{}", "22".repeat(20)), + }, + "endpoints": [{ "multiaddr": endpoint() }], + "files": [{ + "name": "hello.txt", + "address": "CC".repeat(32), + "size": 12, + "content_type": "text/plain", + "blake3": "DD".repeat(32), + "data_map_size": 128, + "chunks": [ + { "index": 2, "dst_hash": "13".repeat(32), "src_hash": "23".repeat(32), "src_size": 4 }, + { "index": 0, "dst_hash": "11".repeat(32), "src_hash": "21".repeat(32), "src_size": 4 }, + { "index": 1, "dst_hash": "12".repeat(32), "src_hash": "22".repeat(32), "src_size": 4 } + ], + "replicas": 5 + }] + }); + let manifest = parse_browser_manifest(value).expect("valid manifest"); + assert_eq!(manifest.files[0].address, "cc".repeat(32)); + assert_eq!(manifest.files[0].chunks[0].index, 0); + assert_eq!(manifest.payment.rpc_url, "http://127.0.0.1:8545/"); + } +} diff --git a/ant-core/src/browser/payment.rs b/ant-core/src/browser/payment.rs new file mode 100644 index 00000000..0cede796 --- /dev/null +++ b/ant-core/src/browser/payment.rs @@ -0,0 +1,476 @@ +//! Verification and payment planning shared by native and browser clients. + +use super::crypto::{keccak256, verify_ml_dsa_65}; +use super::protocol::normalize_hex; +use serde::{Deserialize, Serialize}; + +const PAYMENT_MULTIPLIER: u128 = 3; +const PRICE_BASELINE_WEI: u128 = 3_906_250_000_000_000; +const PRICE_COEFFICIENT_WEI: u128 = 35_156_250_000_000_000; +const PRICE_DIVISOR_SQUARED: u128 = 6_000 * 6_000; +const MAX_COMMITMENT_KEY_COUNT: u32 = 1_000_000; +const MAX_COMMITMENT_SIDECAR_BYTES: usize = 8 * 1024; +const DOMAIN_COMMITMENT: &[u8] = b"autonomi.ant.replication.storage_commitment.v1"; +const DOMAIN_COMMITMENT_HASH: &[u8] = b"autonomi.ant.replication.commitment_hash.v1"; + +/// JSON-safe form of a node's signed storage commitment. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct BrowserCommitmentArtifact { + /// MessagePack-encoded native commitment. + pub encoded: String, + /// Merkle root. + pub root: String, + /// Number of committed keys. + pub key_count: u32, + /// Signing peer ID. + pub sender_peer_id: String, + /// ML-DSA-65 public key. + pub sender_public_key: String, + /// ML-DSA-65 signature. + pub signature: String, +} + +/// JSON-safe form of the native EVM payment quote returned over WebRTC. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct BrowserQuoteArtifact { + /// Quoting node peer ID. + pub peer_id: String, + /// Content address being quoted. + pub content: String, + /// Quote timestamp in seconds since the Unix epoch. + pub timestamp_secs: u64, + /// Decimal token price. + pub price: String, + /// Twenty-byte node rewards address. + pub rewards_address: String, + /// ML-DSA-65 public key. + pub public_key: String, + /// ML-DSA-65 signature. + pub signature: String, + /// Storage commitment key count used by the pricing curve. + pub committed_key_count: u32, + /// Optional pinned storage commitment. + pub commitment_pin: Option, + /// Keccak-256 EVM payment quote hash. + pub quote_hash: String, + /// Optional resolved native storage commitment. + pub commitment: Option, +} + +/// A quote that is safe to hand to a transaction signer. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct VerifiedStorageQuote { + /// Original verified quote sent back to the selected storage nodes. + pub quote: BrowserQuoteArtifact, + /// Lowercase EVM quote hash without `0x`. + #[serde(rename = "quoteHash")] + pub quote_hash: String, + /// Checksummed-independent lowercase rewards address with `0x`. + #[serde(rename = "rewardsAddress")] + pub rewards_address: String, + /// Decimal amount paid after applying Autonomi's replication multiplier. + pub amount: String, +} + +/// Storage quote validation error. +#[derive(Debug, thiserror::Error)] +#[error("invalid storage quote: {0}")] +pub struct StorageQuoteError(pub String); + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +struct NativeStorageCommitment { + root: [u8; 32], + key_count: u32, + sender_peer_id: [u8; 32], + sender_public_key: Vec, + signature: Vec, +} + +/// Compute the EVM-facing `PaymentQuote` Keccak-256 hash. +#[must_use] +pub fn payment_quote_hash(signed_bytes: &[u8], public_key: &[u8], signature: &[u8]) -> [u8; 32] { + keccak256(&[signed_bytes, public_key, signature]) +} + +/// Sum verified decimal quote amounts without exposing integer arithmetic to +/// JavaScript or a wallet adapter. +pub fn storage_payment_total(quotes: &[VerifiedStorageQuote]) -> Result { + quotes + .iter() + .try_fold(0u128, |total, quote| { + let amount = parse_decimal_u128("e.amount, "storage payment amount")?; + total + .checked_add(amount) + .ok_or_else(|| StorageQuoteError("storage payment total overflow".to_string())) + }) + .map(|total| total.to_string()) +} + +/// Fully verify a quote, its commitment, peer binding, price, and EVM hash. +pub fn verify_storage_quote( + mut quote: BrowserQuoteArtifact, + expected_address: &str, + expected_peer_id: &str, +) -> Result { + let expected_address = normalize_hex(expected_address, 32).map_err(StorageQuoteError)?; + let expected_peer_id = normalize_hex(expected_peer_id, 32).map_err(StorageQuoteError)?; + quote.content = normalize_hex("e.content, 32).map_err(StorageQuoteError)?; + quote.peer_id = normalize_hex("e.peer_id, 32).map_err(StorageQuoteError)?; + if quote.content != expected_address { + return Err(StorageQuoteError( + "storage quote is for a different chunk".to_string(), + )); + } + if quote.peer_id != expected_peer_id { + return Err(StorageQuoteError( + "storage quote belongs to a different WebRtcDirect peer".to_string(), + )); + } + if quote.committed_key_count > MAX_COMMITMENT_KEY_COUNT { + return Err(StorageQuoteError(format!( + "invalid committed key count {}", + quote.committed_key_count + ))); + } + let public_key = decode_unbounded_hex("e.public_key, "quote public key")?; + let signature = decode_unbounded_hex("e.signature, "quote signature")?; + if blake3::hash(&public_key).to_hex().as_str() != quote.peer_id { + return Err(StorageQuoteError( + "storage quote public key is not bound to its peer ID".to_string(), + )); + } + let price = parse_decimal_u128("e.price, "quote price")?; + let expected_price = calculate_price(quote.committed_key_count); + if price != expected_price { + return Err(StorageQuoteError( + "storage quote price is not bound to its committed key count".to_string(), + )); + } + let rewards = normalize_hex("e.rewards_address, 20).map_err(StorageQuoteError)?; + quote.rewards_address.clone_from(&rewards); + let commitment_pin = quote + .commitment_pin + .as_deref() + .map(|pin| normalize_hex(pin, 32).map_err(StorageQuoteError)) + .transpose()?; + quote.commitment_pin.clone_from(&commitment_pin); + let signed_bytes = canonical_quote_bytes("e, price, &rewards, commitment_pin.as_deref())?; + if !verify_ml_dsa_65(&public_key, &signature, &signed_bytes, b"") { + return Err(StorageQuoteError( + "storage quote has an invalid ML-DSA-65 signature".to_string(), + )); + } + let quote_hash = hex::encode(payment_quote_hash(&signed_bytes, &public_key, &signature)); + if normalize_hex("e.quote_hash, 32).map_err(StorageQuoteError)? != quote_hash { + return Err(StorageQuoteError( + "storage quote hash does not match its signed fields".to_string(), + )); + } + quote.quote_hash.clone_from("e_hash); + + if quote.committed_key_count == 0 { + if quote.commitment_pin.is_some() || quote.commitment.is_some() { + return Err(StorageQuoteError( + "baseline storage quote has an incoherent commitment".to_string(), + )); + } + } else { + let pin = commitment_pin + .ok_or_else(|| StorageQuoteError("bound storage quote omitted its pin".to_string()))?; + verify_commitment( + quote.commitment.as_mut().ok_or_else(|| { + StorageQuoteError("bound quote omitted its storage commitment".to_string()) + })?, + "e.peer_id, + quote.committed_key_count, + &pin, + )?; + } + + let amount = price + .checked_mul(PAYMENT_MULTIPLIER) + .ok_or_else(|| StorageQuoteError("storage payment amount overflow".to_string()))?; + Ok(VerifiedStorageQuote { + quote, + quote_hash, + rewards_address: format!("0x{rewards}"), + amount: amount.to_string(), + }) +} + +fn canonical_quote_bytes( + quote: &BrowserQuoteArtifact, + price: u128, + rewards: &str, + commitment_pin: Option<&str>, +) -> Result, StorageQuoteError> { + let content = + hex::decode("e.content).map_err(|error| StorageQuoteError(error.to_string()))?; + let rewards = hex::decode(rewards).map_err(|error| StorageQuoteError(error.to_string()))?; + let mut bytes = Vec::with_capacity(32 + 8 + 32 + 20 + 4 + 33); + bytes.extend_from_slice(&content); + bytes.extend_from_slice("e.timestamp_secs.to_le_bytes()); + bytes.extend_from_slice(&price.to_le_bytes()); + bytes.extend_from_slice(&[0u8; 16]); + bytes.extend_from_slice(&rewards); + bytes.extend_from_slice("e.committed_key_count.to_le_bytes()); + if let Some(pin) = commitment_pin { + bytes.push(1); + bytes.extend(hex::decode(pin).map_err(|error| StorageQuoteError(error.to_string()))?); + } else { + bytes.push(0); + } + Ok(bytes) +} + +fn verify_commitment( + artifact: &mut BrowserCommitmentArtifact, + expected_peer_id: &str, + expected_key_count: u32, + expected_pin: &str, +) -> Result<(), StorageQuoteError> { + let encoded = decode_unbounded_hex(&artifact.encoded, "storage commitment sidecar")?; + if encoded.len() > MAX_COMMITMENT_SIDECAR_BYTES { + return Err(StorageQuoteError( + "storage commitment sidecar exceeds the protocol limit".to_string(), + )); + } + let commitment: NativeStorageCommitment = rmp_serde::from_slice(&encoded).map_err(|error| { + StorageQuoteError(format!( + "storage commitment sidecar is not valid MessagePack: {error}" + )) + })?; + let root = normalize_hex(&artifact.root, 32).map_err(StorageQuoteError)?; + let peer_id = normalize_hex(&artifact.sender_peer_id, 32).map_err(StorageQuoteError)?; + let public_key = + decode_unbounded_hex(&artifact.sender_public_key, "storage commitment public key")?; + let signature = decode_unbounded_hex(&artifact.signature, "storage commitment signature")?; + if commitment.root != decode_array_32(&root)? + || commitment.key_count != artifact.key_count + || commitment.sender_peer_id != decode_array_32(&peer_id)? + || commitment.sender_public_key != public_key + || commitment.signature != signature + { + return Err(StorageQuoteError( + "storage commitment sidecar differs from the verified commitment".to_string(), + )); + } + artifact.root = root; + artifact.sender_peer_id = peer_id.clone(); + artifact.sender_public_key = hex::encode(&public_key); + artifact.signature = hex::encode(&signature); + artifact.encoded = hex::encode(&encoded); + if commitment.key_count != expected_key_count { + return Err(StorageQuoteError( + "storage commitment key count does not match quote".to_string(), + )); + } + if peer_id != expected_peer_id + || blake3::hash(&public_key).to_hex().as_str() != expected_peer_id + { + return Err(StorageQuoteError( + "storage commitment belongs to a different peer".to_string(), + )); + } + let signed_payload = commitment_signed_payload(&commitment)?; + if !verify_ml_dsa_65(&public_key, &signature, &signed_payload, DOMAIN_COMMITMENT) { + return Err(StorageQuoteError( + "storage commitment has an invalid ML-DSA-65 signature".to_string(), + )); + } + let postcard = + postcard::to_allocvec(&commitment).map_err(|error| StorageQuoteError(error.to_string()))?; + let mut hasher = blake3::Hasher::new(); + hasher.update(DOMAIN_COMMITMENT_HASH); + hasher.update(&postcard); + if hasher.finalize().to_hex().as_str() != expected_pin { + return Err(StorageQuoteError( + "storage commitment does not resolve the quote pin".to_string(), + )); + } + Ok(()) +} + +fn commitment_signed_payload( + commitment: &NativeStorageCommitment, +) -> Result, StorageQuoteError> { + let key_length = u32::try_from(commitment.sender_public_key.len()) + .map_err(|_| StorageQuoteError("storage commitment public key is too large".to_string()))?; + let mut payload = Vec::with_capacity(32 + 4 + 32 + 4 + commitment.sender_public_key.len()); + payload.extend_from_slice(&commitment.root); + payload.extend_from_slice(&commitment.key_count.to_le_bytes()); + payload.extend_from_slice(&commitment.sender_peer_id); + payload.extend_from_slice(&key_length.to_le_bytes()); + payload.extend_from_slice(&commitment.sender_public_key); + Ok(payload) +} + +fn calculate_price(key_count: u32) -> u128 { + let count = u128::from(key_count); + PRICE_BASELINE_WEI + + count + .saturating_mul(count) + .saturating_mul(PRICE_COEFFICIENT_WEI) + / PRICE_DIVISOR_SQUARED +} + +fn parse_decimal_u128(value: &str, label: &str) -> Result { + if value.is_empty() + || (value.len() > 1 && value.starts_with('0')) + || !value.bytes().all(|byte| byte.is_ascii_digit()) + { + return Err(StorageQuoteError(format!("invalid {label}"))); + } + value + .parse::() + .map_err(|_| StorageQuoteError(format!("{label} exceeds the supported protocol range"))) +} + +fn decode_unbounded_hex(value: &str, label: &str) -> Result, StorageQuoteError> { + let value = value.strip_prefix("0x").unwrap_or(value); + if (value.len() & 1) != 0 || !value.bytes().all(|byte| byte.is_ascii_hexdigit()) { + return Err(StorageQuoteError(format!("invalid {label}"))); + } + hex::decode(value).map_err(|error| StorageQuoteError(format!("invalid {label}: {error}"))) +} + +fn decode_array_32(value: &str) -> Result<[u8; 32], StorageQuoteError> { + let decoded = hex::decode(value).map_err(|error| StorageQuoteError(error.to_string()))?; + decoded.try_into().map_err(|bytes: Vec| { + StorageQuoteError(format!("expected 32 bytes, received {}", bytes.len())) + }) +} + +#[cfg(test)] +mod tests { + use super::*; + use ant_protocol::pqc::api::ml_dsa_65; + + fn baseline_quote() -> (BrowserQuoteArtifact, String, String) { + let content = [0x31; 32]; + let rewards = [0x44; 20]; + let timestamp = 1_775_000_000; + let (public_key, secret_key) = ml_dsa_65().generate_keypair().expect("keypair"); + let public_key = public_key.to_bytes(); + let peer_id = blake3::hash(&public_key).to_hex().to_string(); + let mut quote = BrowserQuoteArtifact { + peer_id: peer_id.clone(), + content: hex::encode(content), + timestamp_secs: timestamp, + price: PRICE_BASELINE_WEI.to_string(), + rewards_address: hex::encode(rewards), + public_key: hex::encode(&public_key), + signature: String::new(), + committed_key_count: 0, + commitment_pin: None, + quote_hash: String::new(), + commitment: None, + }; + let payload = + canonical_quote_bytes("e, PRICE_BASELINE_WEI, &hex::encode(rewards), None) + .expect("payload"); + let signature = ml_dsa_65() + .sign(&secret_key, &payload) + .expect("signature") + .to_bytes(); + quote.signature = hex::encode(&signature); + quote.quote_hash = hex::encode(payment_quote_hash(&payload, &public_key, &signature)); + (quote, hex::encode(content), peer_id) + } + + fn bound_quote() -> (BrowserQuoteArtifact, String, String) { + let content = [0x31; 32]; + let rewards = [0x42; 20]; + let root = [0x53; 32]; + let key_count = 23; + let timestamp = 1_775_000_001; + let (public_key, secret_key) = ml_dsa_65().generate_keypair().expect("keypair"); + let public_key = public_key.to_bytes(); + let peer_id = blake3::hash(&public_key).into(); + let mut commitment = NativeStorageCommitment { + root, + key_count, + sender_peer_id: peer_id, + sender_public_key: public_key.clone(), + signature: Vec::new(), + }; + let commitment_payload = commitment_signed_payload(&commitment).expect("payload"); + commitment.signature = ml_dsa_65() + .sign_with_context(&secret_key, &commitment_payload, DOMAIN_COMMITMENT) + .expect("commitment signature") + .to_bytes(); + let encoded = rmp_serde::to_vec(&commitment).expect("MessagePack commitment"); + let postcard = postcard::to_allocvec(&commitment).expect("postcard commitment"); + let mut pin_hasher = blake3::Hasher::new(); + pin_hasher.update(DOMAIN_COMMITMENT_HASH); + pin_hasher.update(&postcard); + let pin = pin_hasher.finalize().to_hex().to_string(); + let peer_id = hex::encode(peer_id); + let price = calculate_price(key_count); + let mut quote = BrowserQuoteArtifact { + peer_id: peer_id.clone(), + content: hex::encode(content), + timestamp_secs: timestamp, + price: price.to_string(), + rewards_address: hex::encode(rewards), + public_key: hex::encode(&public_key), + signature: String::new(), + committed_key_count: key_count, + commitment_pin: Some(pin.clone()), + quote_hash: String::new(), + commitment: Some(BrowserCommitmentArtifact { + encoded: hex::encode(encoded), + root: hex::encode(commitment.root), + key_count, + sender_peer_id: peer_id.clone(), + sender_public_key: hex::encode(&public_key), + signature: hex::encode(&commitment.signature), + }), + }; + let payload = canonical_quote_bytes("e, price, &hex::encode(rewards), Some(&pin)) + .expect("quote payload"); + let signature = ml_dsa_65() + .sign(&secret_key, &payload) + .expect("quote signature") + .to_bytes(); + quote.signature = hex::encode(&signature); + quote.quote_hash = hex::encode(payment_quote_hash(&payload, &public_key, &signature)); + (quote, hex::encode(content), peer_id) + } + + #[test] + fn payment_hash_matches_evmlib_vector() { + assert_eq!( + hex::encode(payment_quote_hash(&[0, 1], &[2], &[3])), + "d98f2e8134922f73748703c8e7084d42f13d2fa1439936ef5a3abcf5646fe83f" + ); + } + + #[test] + fn verifies_baseline_quote_and_rejects_tampering() { + let (quote, content, peer_id) = baseline_quote(); + let verified = + verify_storage_quote(quote.clone(), &content, &peer_id).expect("valid quote"); + assert_eq!(verified.amount, (PRICE_BASELINE_WEI * 3).to_string()); + let mut tampered = quote; + tampered.price = (PRICE_BASELINE_WEI + 1).to_string(); + assert!(verify_storage_quote(tampered, &content, &peer_id).is_err()); + } + + #[test] + fn verifies_bound_commitment_and_exact_native_sidecar() { + let (quote, content, peer_id) = bound_quote(); + let verified = + verify_storage_quote(quote.clone(), &content, &peer_id).expect("valid bound quote"); + assert_eq!( + storage_payment_total(&[verified]).expect("payment total"), + (calculate_price(23) * PAYMENT_MULTIPLIER).to_string() + ); + + let mut tampered = quote; + tampered.commitment.as_mut().expect("commitment").root = hex::encode([0x99; 32]); + let error = verify_storage_quote(tampered, &content, &peer_id) + .expect_err("sidecar mismatch must fail"); + assert!(error.to_string().contains("sidecar differs")); + } +} diff --git a/ant-core/src/browser/protocol.rs b/ant-core/src/browser/protocol.rs new file mode 100644 index 00000000..5daf8acc --- /dev/null +++ b/ant-core/src/browser/protocol.rs @@ -0,0 +1,569 @@ +//! Browser-facing WebRTC Direct wire profile. + +use super::crypto::verify_ml_dsa_65; +use base64::engine::general_purpose::URL_SAFE_NO_PAD; +use base64::Engine as _; +use serde::{Deserialize, Serialize}; +use serde_json::{Map, Value}; +use std::net::{Ipv4Addr, Ipv6Addr}; +use std::str::FromStr as _; + +/// Current browser request/response protocol version. +pub const BROWSER_PROTOCOL_VERSION: u16 = 3; +/// Protocol name authenticated by the node HELLO response. +pub const BROWSER_PROTOCOL_NAME: &str = "autonomi.web.poc.v3"; +/// Ordered WebRTC DataChannel label used by Autonomi nodes. +pub const WEBRTC_DIRECT_DATA_CHANNEL: &str = "autonomi.web.v3"; +/// Maximum content carried by one browser protocol frame. +pub const MAX_BROWSER_RECORD_BYTES: usize = 4 * 1024 * 1024; +/// Maximum JSON header carried by one browser protocol frame. +pub const MAX_BROWSER_HEADER_BYTES: usize = 64 * 1024; +/// Maximum complete browser response frame. +pub const MAX_BROWSER_RESPONSE_BYTES: usize = + 4 + MAX_BROWSER_HEADER_BYTES + MAX_BROWSER_RECORD_BYTES; +/// Maximum accepted WebRTC Direct multiaddress length. +pub const MAX_WEBRTC_DIRECT_MULTIADDR_LENGTH: usize = 2048; +/// DataChannel message size shared with the native WebRTC Direct listener. +pub const WEBRTC_WRITE_CHUNK_BYTES: usize = 16 * 1024; + +const HELLO_DOMAIN: &[u8] = b"autonomi-webrtc-direct-hello-v1\0"; +const SHA2_256_MULTIHASH_CODE: u8 = 0x12; +const SHA2_256_MULTIHASH_LENGTH: u8 = 32; + +/// Errors produced by browser address, framing, and identity validation. +#[derive(Debug, thiserror::Error)] +pub enum BrowserProtocolError { + /// An address or hexadecimal identifier is malformed. + #[error("invalid browser endpoint: {0}")] + Endpoint(String), + /// A request or response frame is malformed or exceeds a bound. + #[error("invalid browser frame: {0}")] + Frame(String), + /// A node identity response failed authentication. + #[error("invalid node HELLO: {0}")] + Identity(String), +} + +/// Manifest-compatible wrapper around a WebRTC Direct multiaddress. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct BrowserEndpoint { + /// `/ip4|ip6/.../udp/.../webrtc-direct/certhash/.../p2p/...` address. + pub multiaddr: String, +} + +/// Parsed, certificate-pinned direct endpoint. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct WebRtcDirectEndpoint { + /// Canonical input multiaddress. + pub multiaddr: String, + /// `ip4` or `ip6`. + #[serde(rename = "hostProtocol")] + pub host_protocol: String, + /// Literal IP address. + pub host: String, + /// UDP listener port. + pub port: u16, + /// Lowercase 32-byte ANT peer ID. + #[serde(rename = "peerId")] + pub peer_id: String, + /// SHA-256 DTLS certificate digest. + #[serde(rename = "certificateHash")] + pub certificate_hash: [u8; 32], +} + +/// Endpoint accepted from either a raw multiaddress or manifest object. +#[derive(Debug, Clone, Deserialize)] +#[serde(untagged)] +pub enum BrowserEndpointInput { + /// Raw multiaddress string. + Multiaddr(String), + /// Manifest endpoint object. + Structured(BrowserEndpoint), +} + +impl BrowserEndpointInput { + /// Return the contained multiaddress. + #[must_use] + pub fn multiaddr(&self) -> &str { + match self { + Self::Multiaddr(value) => value, + Self::Structured(value) => &value.multiaddr, + } + } +} + +/// Parse and validate a signaling-free WebRTC Direct multiaddress. +pub fn parse_webrtc_direct_multiaddr( + multiaddr: &str, +) -> Result { + let multiaddr = multiaddr.trim(); + if multiaddr.is_empty() + || multiaddr.len() > MAX_WEBRTC_DIRECT_MULTIADDR_LENGTH + || !multiaddr.starts_with('/') + { + return Err(BrowserProtocolError::Endpoint( + "invalid WebRtcDirect multiaddress length or prefix".to_string(), + )); + } + let segments = multiaddr.split('/').collect::>(); + if segments.len() != 10 { + return Err(BrowserProtocolError::Endpoint( + "WebRtcDirect multiaddress is incomplete".to_string(), + )); + } + + let host_protocol = segments[1]; + let host = segments[2]; + match host_protocol { + "ip4" => { + Ipv4Addr::from_str(host).map_err(|error| { + BrowserProtocolError::Endpoint(format!("invalid IPv4 address {host}: {error}")) + })?; + } + "ip6" => { + Ipv6Addr::from_str(host).map_err(|error| { + BrowserProtocolError::Endpoint(format!("invalid IPv6 address {host}: {error}")) + })?; + } + _ => { + return Err(BrowserProtocolError::Endpoint( + "WebRTC Direct multiaddresses must use a literal IP address".to_string(), + )); + } + } + if segments[3] != "udp" { + return Err(BrowserProtocolError::Endpoint( + "WebRtcDirect multiaddress must use UDP".to_string(), + )); + } + let port = segments[4].parse::().map_err(|error| { + BrowserProtocolError::Endpoint(format!( + "WebRtcDirect multiaddress has an invalid UDP port: {error}" + )) + })?; + if port == 0 { + return Err(BrowserProtocolError::Endpoint( + "WebRtcDirect multiaddress has an invalid UDP port".to_string(), + )); + } + if segments[5] != "webrtc-direct" { + return Err(BrowserProtocolError::Endpoint( + "WebRTC Direct multiaddress must contain /webrtc-direct".to_string(), + )); + } + if segments[6] != "certhash" || segments[7].is_empty() { + return Err(BrowserProtocolError::Endpoint( + "WebRTC Direct multiaddress must contain exactly one certhash".to_string(), + )); + } + let certificate_hash = decode_certificate_multihash(segments[7])?; + if segments[8] != "p2p" { + return Err(BrowserProtocolError::Endpoint( + "WebRtcDirect multiaddress must end with /p2p/".to_string(), + )); + } + let peer_id = normalize_hex(segments[9], 32).map_err(BrowserProtocolError::Endpoint)?; + + Ok(WebRtcDirectEndpoint { + multiaddr: multiaddr.to_string(), + host_protocol: host_protocol.to_string(), + host: host.to_string(), + port, + peer_id, + certificate_hash, + }) +} + +fn decode_certificate_multihash(value: &str) -> Result<[u8; 32], BrowserProtocolError> { + let encoded = value.strip_prefix('u').ok_or_else(|| { + BrowserProtocolError::Endpoint( + "certificate multihash must use base64url multibase (`u`)".to_string(), + ) + })?; + let decoded = URL_SAFE_NO_PAD.decode(encoded).map_err(|error| { + BrowserProtocolError::Endpoint(format!( + "certificate multihash is not valid unpadded base64url: {error}" + )) + })?; + if decoded.len() != 34 + || decoded[0] != SHA2_256_MULTIHASH_CODE + || decoded[1] != SHA2_256_MULTIHASH_LENGTH + { + return Err(BrowserProtocolError::Endpoint( + "certificate multihash must contain a 32-byte SHA-256 digest".to_string(), + )); + } + decoded[2..].try_into().map_err(|_| { + BrowserProtocolError::Endpoint( + "certificate multihash must contain a 32-byte SHA-256 digest".to_string(), + ) + }) +} + +/// Normalize a fixed-width hexadecimal wire field. +pub fn normalize_hex(value: &str, bytes: usize) -> Result { + let normalized = value + .trim() + .strip_prefix("0x") + .or_else(|| value.trim().strip_prefix("0X")) + .unwrap_or(value.trim()) + .replace(':', ""); + if normalized.len() != bytes.saturating_mul(2) + || !normalized.bytes().all(|byte| byte.is_ascii_hexdigit()) + { + return Err(format!("expected {bytes} hexadecimal bytes")); + } + Ok(normalized.to_ascii_lowercase()) +} + +/// Decode a fixed-width hexadecimal wire field. +pub fn decode_hex(value: &str, bytes: usize) -> Result, String> { + let normalized = normalize_hex(value, bytes)?; + hex::decode(normalized).map_err(|error| error.to_string()) +} + +/// A decoded response frame retaining its JSON header and binary content. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct BrowserResponseFrame { + /// Untrusted JSON response header after structural validation. + pub header: Value, + /// Binary response body. + #[serde(with = "serde_bytes")] + pub content: Vec, +} + +/// Parse one complete length-prefixed browser response. +pub fn parse_response_frame(frame: &[u8]) -> Result { + let (header, content_offset, frame_length) = parse_response_header(frame)?; + if frame.len() != frame_length { + return Err(BrowserProtocolError::Frame(format!( + "response length mismatch: declared {} content bytes", + header["content_length"] + ))); + } + Ok(BrowserResponseFrame { + header, + content: frame[content_offset..].to_vec(), + }) +} + +/// Determine a frame's complete length once its JSON header is available. +pub fn response_frame_length(frame: &[u8]) -> Result, BrowserProtocolError> { + if frame.len() < 4 { + return Ok(None); + } + let header_length = u32::from_be_bytes( + frame[..4] + .try_into() + .map_err(|_| BrowserProtocolError::Frame("missing header length".to_string()))?, + ) as usize; + validate_header_length(header_length)?; + if frame.len() < 4 + header_length { + return Ok(None); + } + parse_response_header(frame).map(|(_, _, length)| Some(length)) +} + +fn parse_response_header(frame: &[u8]) -> Result<(Value, usize, usize), BrowserProtocolError> { + if frame.len() < 4 { + return Err(BrowserProtocolError::Frame( + "response ended before its four-byte header length".to_string(), + )); + } + let header_length = u32::from_be_bytes( + frame[..4] + .try_into() + .map_err(|_| BrowserProtocolError::Frame("missing header length".to_string()))?, + ) as usize; + validate_header_length(header_length)?; + let content_offset = 4 + header_length; + if content_offset > frame.len() { + return Err(BrowserProtocolError::Frame( + "response ended inside its JSON header".to_string(), + )); + } + let header: Value = serde_json::from_slice(&frame[4..content_offset]) + .map_err(|error| BrowserProtocolError::Frame(format!("invalid response JSON: {error}")))?; + if header.get("version").and_then(Value::as_u64) != Some(u64::from(BROWSER_PROTOCOL_VERSION)) { + return Err(BrowserProtocolError::Frame(format!( + "unsupported response version {}", + header.get("version").unwrap_or(&Value::Null) + ))); + } + let content_length = header + .get("content_length") + .and_then(Value::as_u64) + .and_then(|value| usize::try_from(value).ok()) + .filter(|value| *value <= MAX_BROWSER_RECORD_BYTES) + .ok_or_else(|| { + BrowserProtocolError::Frame("invalid response content length".to_string()) + })?; + let frame_length = content_offset + .checked_add(content_length) + .ok_or_else(|| BrowserProtocolError::Frame("response length overflow".to_string()))?; + Ok((header, content_offset, frame_length)) +} + +fn validate_header_length(header_length: usize) -> Result<(), BrowserProtocolError> { + if header_length == 0 || header_length > MAX_BROWSER_HEADER_BYTES { + return Err(BrowserProtocolError::Frame(format!( + "invalid response header length {header_length}" + ))); + } + Ok(()) +} + +/// Encode one JSON-header-plus-binary request frame. +pub fn encode_request_frame( + request_id: u64, + request_type: &str, + mut fields: Map, + content: &[u8], +) -> Result, BrowserProtocolError> { + if content.len() > MAX_BROWSER_RECORD_BYTES { + return Err(BrowserProtocolError::Frame(format!( + "request content must be at most {MAX_BROWSER_RECORD_BYTES} bytes" + ))); + } + fields.insert("version".to_string(), Value::from(BROWSER_PROTOCOL_VERSION)); + fields.insert("request_id".to_string(), Value::from(request_id)); + fields.insert("content_length".to_string(), Value::from(content.len())); + fields.insert("type".to_string(), Value::from(request_type)); + let header = serde_json::to_vec(&fields) + .map_err(|error| BrowserProtocolError::Frame(error.to_string()))?; + validate_header_length(header.len())?; + let capacity = 4usize + .checked_add(header.len()) + .and_then(|size| size.checked_add(content.len())) + .ok_or_else(|| BrowserProtocolError::Frame("request length overflow".to_string()))?; + let header_length = u32::try_from(header.len()) + .map_err(|_| BrowserProtocolError::Frame("request header length overflow".to_string()))?; + let mut frame = Vec::with_capacity(capacity); + frame.extend_from_slice(&header_length.to_be_bytes()); + frame.extend_from_slice(&header); + frame.extend_from_slice(content); + Ok(frame) +} + +/// Authenticated fields returned by a node's HELLO response. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct BrowserHello { + /// Response discriminator. + #[serde(rename = "type")] + pub response_type: String, + /// Browser protocol name. + pub protocol: String, + /// Lowercase peer ID. + pub peer_id: String, + /// Echoed client challenge. + pub challenge: String, + /// ML-DSA-65 public key. + pub public_key: String, + /// ML-DSA-65 signature. + pub signature: String, + /// Signed direct endpoint. + pub endpoint: BrowserEndpoint, + /// Maximum node record size. + #[serde(default)] + pub max_chunk_size: usize, + /// Advertised browser operations. + #[serde(default)] + pub capabilities: Vec, + /// Advertised payment network, retained as protocol JSON. + #[serde(default)] + pub payment: Value, +} + +/// Verify that a HELLO binds an ANT identity to the expected direct endpoint. +pub fn verify_hello_identity( + hello: &BrowserHello, + expected_endpoint: &WebRtcDirectEndpoint, + challenge: &[u8; 32], +) -> Result { + if hello.response_type != "hello" { + return Err(BrowserProtocolError::Identity( + "expected a HELLO response".to_string(), + )); + } + if hello.protocol != BROWSER_PROTOCOL_NAME { + return Err(BrowserProtocolError::Identity(format!( + "unsupported browser protocol {}", + hello.protocol + ))); + } + let peer_id = normalize_hex(&hello.peer_id, 32).map_err(BrowserProtocolError::Identity)?; + if normalize_hex(&hello.challenge, 32).map_err(BrowserProtocolError::Identity)? + != hex::encode(challenge) + { + return Err(BrowserProtocolError::Identity( + "node signed a different HELLO challenge".to_string(), + )); + } + let advertised = parse_webrtc_direct_multiaddr(&hello.endpoint.multiaddr) + .map_err(|error| BrowserProtocolError::Identity(error.to_string()))?; + if advertised.multiaddr != expected_endpoint.multiaddr || advertised.peer_id != peer_id { + return Err(BrowserProtocolError::Identity( + "node advertised a different WebRTC Direct endpoint".to_string(), + )); + } + if peer_id != expected_endpoint.peer_id { + return Err(BrowserProtocolError::Identity(format!( + "endpoint identity mismatch: expected {}, received {peer_id}", + expected_endpoint.peer_id + ))); + } + let public_key = hex::decode(&hello.public_key) + .map_err(|error| BrowserProtocolError::Identity(error.to_string()))?; + let signature = hex::decode(&hello.signature) + .map_err(|error| BrowserProtocolError::Identity(error.to_string()))?; + if blake3::hash(&public_key).to_hex().as_str() != peer_id { + return Err(BrowserProtocolError::Identity( + "HELLO public key is not bound to the ANT peer ID".to_string(), + )); + } + let mut transcript = Vec::with_capacity( + HELLO_DOMAIN.len() + challenge.len() + peer_id.len() + advertised.multiaddr.len(), + ); + transcript.extend_from_slice(HELLO_DOMAIN); + transcript.extend_from_slice(challenge); + transcript.extend_from_slice(peer_id.as_bytes()); + transcript.extend_from_slice(advertised.multiaddr.as_bytes()); + if !verify_ml_dsa_65(&public_key, &signature, &transcript, b"") { + return Err(BrowserProtocolError::Identity( + "HELLO has an invalid ML-DSA-65 signature".to_string(), + )); + } + Ok(peer_id) +} + +/// Build the certificate-pinned ICE-lite SDP answer for a direct endpoint. +pub fn server_answer_sdp( + endpoint: &WebRtcDirectEndpoint, + ice_credential: &str, +) -> Result { + validate_ice_credential(ice_credential)?; + let ip_version = if endpoint.host_protocol == "ip4" { + "IP4" + } else { + "IP6" + }; + let fingerprint = endpoint + .certificate_hash + .iter() + .map(|byte| format!("{byte:02X}")) + .collect::>() + .join(":"); + Ok(format!( + "v=0\r\no=- 0 0 IN {ip_version} {host}\r\ns=-\r\nt=0 0\r\na=ice-lite\r\nm=application {port} UDP/DTLS/SCTP webrtc-datachannel\r\nc=IN {ip_version} {host}\r\na=mid:0\r\na=ice-options:ice2\r\na=ice-ufrag:{credential}\r\na=ice-pwd:{credential}\r\na=fingerprint:sha-256 {fingerprint}\r\na=setup:passive\r\na=sctp-port:5000\r\na=max-message-size:{WEBRTC_WRITE_CHUNK_BYTES}\r\na=candidate:1467250027 1 UDP 1467250027 {host} {port} typ host\r\na=end-of-candidates\r\n", + host = endpoint.host, + port = endpoint.port, + credential = ice_credential, + )) +} + +/// Replace the browser-generated offer ICE credentials with Saorsa's shared credential. +pub fn munge_offer_ice_credentials( + sdp: &str, + ice_credential: &str, +) -> Result { + validate_ice_credential(ice_credential)?; + if sdp.is_empty() { + return Err(BrowserProtocolError::Frame( + "browser created an empty WebRTC offer".to_string(), + )); + } + let sdp = replace_sdp_attribute(sdp, "a=ice-ufrag:", ice_credential)?; + replace_sdp_attribute(&sdp, "a=ice-pwd:", ice_credential) +} + +fn replace_sdp_attribute( + sdp: &str, + prefix: &str, + value: &str, +) -> Result { + let start = sdp.find(prefix).ok_or_else(|| { + BrowserProtocolError::Frame("browser offer did not contain ICE credentials".to_string()) + })?; + let value_start = start + prefix.len(); + let value_end = sdp[value_start..] + .find(['\r', '\n']) + .map_or(sdp.len(), |offset| value_start + offset); + let mut output = String::with_capacity(sdp.len() + value.len()); + output.push_str(&sdp[..value_start]); + output.push_str(value); + output.push_str(&sdp[value_end..]); + Ok(output) +} + +fn validate_ice_credential(value: &str) -> Result<(), BrowserProtocolError> { + let suffix = value.strip_prefix("saorsa+webrtc+v1/").ok_or_else(|| { + BrowserProtocolError::Endpoint("invalid Saorsa WebRTC Direct ICE credential".to_string()) + })?; + if !(22..=256).contains(&suffix.len()) + || !suffix + .bytes() + .all(|byte| byte.is_ascii_alphanumeric() || matches!(byte, b'+' | b'/')) + { + return Err(BrowserProtocolError::Endpoint( + "invalid Saorsa WebRTC Direct ICE credential".to_string(), + )); + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn endpoint() -> String { + let mut multihash = vec![0x12, 0x20]; + multihash.extend([0x11; 32]); + format!( + "/ip4/127.0.0.1/udp/24000/webrtc-direct/certhash/u{}/p2p/{}", + URL_SAFE_NO_PAD.encode(multihash), + "ab".repeat(32) + ) + } + + #[test] + fn parses_literal_certificate_pinned_endpoint() { + let parsed = parse_webrtc_direct_multiaddr(&endpoint()).expect("parse endpoint"); + assert_eq!(parsed.host_protocol, "ip4"); + assert_eq!(parsed.host, "127.0.0.1"); + assert_eq!(parsed.port, 24000); + assert_eq!(parsed.certificate_hash, [0x11; 32]); + assert!(parse_webrtc_direct_multiaddr(&endpoint().replacen( + "/ip4/127.0.0.1", + "/dns/node.example", + 1 + )) + .is_err()); + } + + #[test] + fn response_frame_round_trip() { + let mut fields = Map::new(); + fields.insert("address".to_string(), Value::from("11".repeat(32))); + let frame = + encode_request_frame(9, "get_chunk", fields, &[1, 2, 3]).expect("encode request"); + let parsed = parse_response_frame(&frame).expect("parse response-shaped frame"); + assert_eq!(parsed.header["request_id"], 9); + assert_eq!(parsed.content, vec![1, 2, 3]); + } + + #[test] + fn synthesizes_pinned_answer_and_munges_offer() { + let endpoint = parse_webrtc_direct_multiaddr(&endpoint()).expect("parse endpoint"); + let credential = format!("saorsa+webrtc+v1/{}", "a".repeat(32)); + let answer = server_answer_sdp(&endpoint, &credential).expect("answer"); + assert!(answer.contains("a=ice-lite")); + assert!(answer.contains("a=fingerprint:sha-256 11:11:11:11")); + let offer = munge_offer_ice_credentials( + "v=0\r\na=ice-ufrag:old\r\na=ice-pwd:secret\r\n", + &credential, + ) + .expect("offer"); + assert!(offer.contains(&format!("a=ice-ufrag:{credential}"))); + assert!(offer.contains(&format!("a=ice-pwd:{credential}"))); + } +} diff --git a/ant-core/src/browser/wasm_transport.rs b/ant-core/src/browser/wasm_transport.rs new file mode 100644 index 00000000..98f52d55 --- /dev/null +++ b/ant-core/src/browser/wasm_transport.rs @@ -0,0 +1,1622 @@ +//! `web-sys` WebRTC Direct transport and typed node operations. + +use super::manifest::{ + validate_browser_payment_network, BrowserPaymentNetwork, PublicFileDescriptor, +}; +use super::payment::{ + storage_payment_total, verify_storage_quote, BrowserQuoteArtifact, VerifiedStorageQuote, +}; +use super::protocol::{ + encode_request_frame, munge_offer_ice_credentials, parse_response_frame, + parse_webrtc_direct_multiaddr, response_frame_length, server_answer_sdp, verify_hello_identity, + BrowserEndpoint, BrowserEndpointInput, BrowserHello, BrowserProtocolError, + BrowserResponseFrame, WebRtcDirectEndpoint, MAX_BROWSER_RESPONSE_BYTES, + WEBRTC_DIRECT_DATA_CHANNEL, WEBRTC_WRITE_CHUNK_BYTES, +}; +use futures_channel::{mpsc, oneshot}; +use futures_util::{ + future::{join_all, select, Either}, + lock::Mutex, + stream::{self, StreamExt as _}, +}; +use gloo_timers::future::TimeoutFuture; +use js_sys::{Array, ArrayBuffer, Promise, Uint8Array}; +use saorsa_dht_lookup::{ + run_iterative_lookup, IterativeLookup, LookupConfig, LookupKey, LookupNode, LookupQuery, + LookupQueryOutcome, +}; +use serde::{Deserialize, Serialize}; +use serde_json::{Map, Value}; +use std::cell::{Cell, RefCell}; +use std::collections::HashMap; +use std::future::Future; +use std::ops::Deref; +use std::rc::Rc; +use wasm_bindgen::prelude::*; +use wasm_bindgen_futures::JsFuture; +use web_sys::{ + Event, MessageEvent, RtcConfiguration, RtcDataChannel, RtcDataChannelInit, RtcDataChannelState, + RtcDataChannelType, RtcPeerConnection, RtcSdpType, RtcSessionDescriptionInit, +}; + +const REQUEST_TIMEOUT_MS: u32 = 10_000; +const MAX_BUFFERED_AMOUNT: u32 = 2 * 1024 * 1024; +const ICE_CREDENTIAL_PREFIX: &str = "saorsa+webrtc+v1/"; +const ICE_ALPHABET: &[u8] = b"abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789"; +const DEFAULT_MAX_POOLED_CLIENTS: usize = 10; +const DEFAULT_LOOKUP_K: usize = 20; +const DEFAULT_LOOKUP_ALPHA: usize = 3; +const DEFAULT_MAX_LOOKUP_ITERATIONS: usize = 20; +const MAX_STORE_TARGETS: usize = 7; +const MAX_DOWNLOAD_CONCURRENCY: usize = 6; + +type ResponseInbox = Rc, String>>>>; + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub(super) struct BrowserNode { + pub(super) peer_id: String, + #[serde(default)] + pub(super) native_addresses: Vec, + #[serde(default)] + pub(super) reliability: f64, + #[serde(default)] + pub(super) webrtc_direct: Option, +} + +#[derive(Debug, Serialize)] +struct BrowserLookupResult { + nodes: Vec, + queried: Vec, + failures: Vec, +} + +#[derive(Debug, Clone, Serialize)] +struct BrowserLookupFailure { + #[serde(rename = "peerId")] + peer_id: String, + message: String, +} + +#[derive(Debug, Clone)] +struct BrowserLookupCandidate { + peer_id: LookupKey, + wire: BrowserNode, +} + +impl LookupNode for BrowserLookupCandidate { + fn lookup_peer_id(&self) -> LookupKey { + self.peer_id + } +} + +impl BrowserLookupCandidate { + fn parse(mut wire: BrowserNode) -> Result { + let peer_id = parse_lookup_key(&wire.peer_id, "peer ID")?; + wire.peer_id = hex::encode(peer_id); + Ok(Self { peer_id, wire }) + } +} + +struct PoolEntry { + client: Rc, + last_used: u64, +} + +struct BrowserClientPool { + max_clients: usize, + clients: RefCell>, + clock: Cell, + closed: Cell, + available_tx: mpsc::UnboundedSender<()>, + available_rx: Mutex>, +} + +struct BrowserClientLease { + client: Rc, + available_tx: mpsc::UnboundedSender<()>, +} + +impl Deref for BrowserClientLease { + type Target = BrowserNodeClientCore; + + fn deref(&self) -> &Self::Target { + &self.client + } +} + +impl Drop for BrowserClientLease { + fn drop(&mut self) { + let _ = self.available_tx.unbounded_send(()); + } +} + +impl BrowserClientPool { + fn new(max_clients: usize) -> Result { + if max_clients == 0 { + return Err("WebRTC client pool size must be a positive integer".to_string()); + } + let (available_tx, available_rx) = mpsc::unbounded(); + Ok(Self { + max_clients, + clients: RefCell::new(HashMap::new()), + clock: Cell::new(0), + closed: Cell::new(false), + available_tx, + available_rx: Mutex::new(available_rx), + }) + } + + async fn client(&self, endpoint: &BrowserEndpoint) -> Result { + let endpoint = parse_webrtc_direct_multiaddr(&endpoint.multiaddr) + .map_err(|error| error.to_string())?; + let key = endpoint.multiaddr.clone(); + loop { + if self.closed.get() { + return Err("WebRTC client pool is closed".to_string()); + } + let now = self.clock.get().wrapping_add(1); + self.clock.set(now); + let client = { + let mut clients = self.clients.borrow_mut(); + if let Some(entry) = clients.get_mut(&key) { + entry.last_used = now; + Some(Rc::clone(&entry.client)) + } else { + if clients.len() >= self.max_clients { + let evict = clients + .iter() + .filter(|(_, entry)| Rc::strong_count(&entry.client) == 1) + .min_by_key(|(_, entry)| entry.last_used) + .map(|(key, _)| key.clone()); + if let Some(evict) = evict { + if let Some(entry) = clients.remove(&evict) { + entry.client.close(); + } + } + } + if clients.len() < self.max_clients { + let client = Rc::new(BrowserNodeClientCore::new(endpoint.clone())); + clients.insert( + key.clone(), + PoolEntry { + client: Rc::clone(&client), + last_used: now, + }, + ); + Some(client) + } else { + None + } + } + }; + if let Some(client) = client { + return Ok(BrowserClientLease { + client, + available_tx: self.available_tx.clone(), + }); + } + if self.available_rx.lock().await.next().await.is_none() { + return Err("WebRTC client pool closed while waiting for capacity".to_string()); + } + } + } + + fn close(&self) { + self.closed.set(true); + self.available_tx.close_channel(); + for (_, entry) in self.clients.borrow_mut().drain() { + entry.client.close(); + } + } +} + +#[derive(Debug, Serialize)] +struct BrowserChunk { + #[serde(with = "serde_bytes")] + content: Vec, + hash: String, +} + +#[derive(Debug, Serialize)] +struct BrowserQuoteResponse { + quote: BrowserQuoteArtifact, + #[serde(rename = "alreadyStored")] + already_stored: bool, +} + +#[derive(Debug, Serialize)] +struct BrowserPutResponse { + address: String, + #[serde(rename = "alreadyStored")] + already_stored: bool, +} + +#[derive(Debug, Deserialize)] +struct NodesResponse { + #[serde(rename = "type")] + response_type: String, + target: String, + nodes: Vec, +} + +#[derive(Debug, Deserialize)] +struct ChunkResponse { + #[serde(rename = "type")] + response_type: String, + address: String, + size: usize, +} + +#[derive(Debug, Deserialize)] +struct QuoteResponse { + #[serde(rename = "type")] + response_type: String, + address: String, + already_stored: bool, + quote: BrowserQuoteArtifact, +} + +#[derive(Debug, Deserialize)] +struct PutResponse { + #[serde(rename = "type")] + response_type: String, + address: String, + already_stored: bool, +} + +struct Connection { + peer_connection: RtcPeerConnection, + data_channel: RtcDataChannel, + inbox: ResponseInbox, + _on_message: Closure, + _on_error: Closure, + _on_close: Closure, + _on_open: Closure, +} + +impl Connection { + async fn open(endpoint: &WebRtcDirectEndpoint) -> Result { + let configuration = RtcConfiguration::new(); + configuration.set_ice_servers(&Array::new()); + let peer_connection = + RtcPeerConnection::new_with_configuration(&configuration).map_err(js_error_message)?; + let channel_configuration = RtcDataChannelInit::new(); + channel_configuration.set_ordered(true); + let data_channel = peer_connection.create_data_channel_with_data_channel_dict( + WEBRTC_DIRECT_DATA_CHANNEL, + &channel_configuration, + ); + data_channel.set_binary_type(RtcDataChannelType::Arraybuffer); + + let (inbox_tx, inbox_rx) = mpsc::unbounded::, String>>(); + let message_tx = inbox_tx.clone(); + let on_message = Closure::::new(move |event: MessageEvent| { + let data = event.data(); + let result = if data.is_instance_of::() || ArrayBuffer::is_view(&data) { + Ok(Uint8Array::new(&data).to_vec()) + } else { + Err("node sent a non-binary DataChannel message".to_string()) + }; + let _ = message_tx.unbounded_send(result); + }); + data_channel.set_onmessage(Some(on_message.as_ref().unchecked_ref())); + + let error_tx = inbox_tx.clone(); + let on_error = Closure::::new(move |_event: Event| { + let _ = error_tx.unbounded_send(Err("WebRTC DataChannel failed".to_string())); + }); + data_channel.set_onerror(Some(on_error.as_ref().unchecked_ref())); + let close_tx = inbox_tx; + let on_close = Closure::::new(move |_event: Event| { + let _ = close_tx.unbounded_send(Err("WebRTC DataChannel closed".to_string())); + }); + data_channel.set_onclose(Some(on_close.as_ref().unchecked_ref())); + + let (open_tx, open_rx) = oneshot::channel::<()>(); + let open_tx = Rc::new(RefCell::new(Some(open_tx))); + let open_sender = Rc::clone(&open_tx); + let on_open = Closure::::new(move |_event: Event| { + if let Some(sender) = open_sender.borrow_mut().take() { + let _ = sender.send(()); + } + }); + data_channel.set_onopen(Some(on_open.as_ref().unchecked_ref())); + + // Own the browser objects and every installed callback before the + // first await. Any setup error now runs `Drop`, detaches the callbacks, + // and closes the half-open peer connection deterministically. + let connection = Self { + peer_connection, + data_channel, + inbox: Rc::new(Mutex::new(inbox_rx)), + _on_message: on_message, + _on_error: on_error, + _on_close: on_close, + _on_open: on_open, + }; + + let credential = random_ice_credential()?; + let offer = JsFuture::from(connection.peer_connection.create_offer()) + .await + .map_err(js_error_message)?; + let offer: RtcSessionDescriptionInit = offer.dyn_into().map_err(js_error_message)?; + let offer_sdp = offer + .get_sdp() + .ok_or_else(|| "browser created an empty WebRTC offer".to_string())?; + let munged_sdp = munge_offer_ice_credentials(&offer_sdp, &credential) + .map_err(|error| error.to_string())?; + let local = RtcSessionDescriptionInit::new(RtcSdpType::Offer); + local.set_sdp(&munged_sdp); + JsFuture::from(connection.peer_connection.set_local_description(&local)) + .await + .map_err(js_error_message)?; + let answer_sdp = + server_answer_sdp(endpoint, &credential).map_err(|error| error.to_string())?; + let remote = RtcSessionDescriptionInit::new(RtcSdpType::Answer); + remote.set_sdp(&answer_sdp); + JsFuture::from(connection.peer_connection.set_remote_description(&remote)) + .await + .map_err(js_error_message)?; + + timeout( + async move { + open_rx + .await + .map_err(|_| "WebRTC DataChannel closed before opening".to_string()) + }, + "WebRTC DataChannel opening timed out", + ) + .await?; + connection.data_channel.set_onopen(None); + + Ok(connection) + } + + fn close(self) { + drop(self); + } +} + +impl Drop for Connection { + fn drop(&mut self) { + self.data_channel.set_onmessage(None); + self.data_channel.set_onerror(None); + self.data_channel.set_onclose(None); + self.data_channel.set_onopen(None); + self.data_channel.set_onbufferedamountlow(None); + self.data_channel.close(); + self.peer_connection.close(); + } +} + +pub(super) struct BrowserNodeClientCore { + endpoint: WebRtcDirectEndpoint, + connection: RefCell>, + request_lock: Mutex<()>, + next_request_id: Cell, + hello: RefCell>, + peer_id: RefCell>, +} + +impl BrowserNodeClientCore { + pub(super) fn new(endpoint: WebRtcDirectEndpoint) -> Self { + Self { + endpoint, + connection: RefCell::new(None), + request_lock: Mutex::new(()), + next_request_id: Cell::new(1), + hello: RefCell::new(None), + peer_id: RefCell::new(None), + } + } + + pub(super) fn peer_id(&self) -> Option { + self.peer_id.borrow().clone() + } + + async fn ensure_connected(&self) -> Result<(), String> { + let open = self.connection.borrow().as_ref().is_some_and(|connection| { + connection.data_channel.ready_state() == RtcDataChannelState::Open + }); + if open { + return Ok(()); + } + self.close(); + let connection = Connection::open(&self.endpoint).await?; + self.connection.replace(Some(connection)); + Ok(()) + } + + async fn request( + &self, + request_type: &str, + fields: Map, + content: &[u8], + ) -> Result { + let _guard = self.request_lock.lock().await; + self.ensure_connected().await?; + let request_id = self.next_request_id.get(); + self.next_request_id.set(request_id.wrapping_add(1).max(1)); + let frame = encode_request_frame(request_id, request_type, fields, content) + .map_err(|error| error.to_string())?; + let channel = self + .connection + .borrow() + .as_ref() + .map(|connection| connection.data_channel.clone()) + .ok_or_else(|| "WebRTC DataChannel is not connected".to_string())?; + for message in frame.chunks(WEBRTC_WRITE_CHUNK_BYTES) { + wait_for_capacity(&channel).await?; + channel + .send_with_u8_array(message) + .map_err(js_error_message)?; + } + let receiver = self + .connection + .borrow() + .as_ref() + .map(|connection| Rc::clone(&connection.inbox)) + .ok_or_else(|| "WebRTC response inbox is unavailable".to_string())?; + let response = timeout(read_response(receiver), "WebRTC request timed out").await; + let response = match response { + Ok(response) => response, + Err(error) => { + self.close(); + return Err(error); + } + }; + if response.header.get("request_id").and_then(Value::as_u64) != Some(request_id) { + return Err(format!( + "response ID {} does not match request {request_id}", + response.header.get("request_id").unwrap_or(&Value::Null) + )); + } + if response.header.get("status").and_then(Value::as_str) == Some("error") { + return Err(response + .header + .get("message") + .and_then(Value::as_str) + .unwrap_or("node returned an error") + .to_string()); + } + Ok(response) + } + + pub(super) async fn hello(&self) -> Result { + if let Some(hello) = self.hello.borrow().clone() { + if self.connection.borrow().as_ref().is_some_and(|connection| { + connection.data_channel.ready_state() == RtcDataChannelState::Open + }) { + return Ok(hello); + } + } + let mut challenge = [0u8; 32]; + getrandom::getrandom(&mut challenge) + .map_err(|error| format!("browser entropy failed: {error}"))?; + let mut fields = Map::new(); + fields.insert("challenge".to_string(), Value::from(hex::encode(challenge))); + let response = self.request("hello", fields, &[]).await?; + let hello: BrowserHello = serde_json::from_value(response.header) + .map_err(|error| format!("invalid HELLO response: {error}"))?; + let peer_id = verify_hello_identity(&hello, &self.endpoint, &challenge) + .map_err(|error| error.to_string())?; + self.peer_id.replace(Some(peer_id)); + self.hello.replace(Some(hello.clone())); + Ok(hello) + } + + pub(super) async fn find_node( + &self, + target: &str, + count: usize, + ) -> Result, String> { + let target = super::protocol::normalize_hex(target, 32)?; + let mut fields = Map::new(); + fields.insert("target".to_string(), Value::from(target.clone())); + fields.insert("count".to_string(), Value::from(count)); + let response = self.request("find_node", fields, &[]).await?; + let response: NodesResponse = serde_json::from_value(response.header) + .map_err(|error| format!("invalid NODES response: {error}"))?; + if response.response_type != "nodes" { + return Err("expected a NODES response".to_string()); + } + if response.target.to_ascii_lowercase() != target { + return Err("node returned results for a different lookup target".to_string()); + } + for node in &response.nodes { + let peer_id = super::protocol::normalize_hex(&node.peer_id, 32)?; + if let Some(endpoint) = &node.webrtc_direct { + let endpoint = parse_webrtc_direct_multiaddr(&endpoint.multiaddr) + .map_err(|error| error.to_string())?; + if endpoint.peer_id != peer_id { + return Err(format!("node {peer_id} advertised another peer's endpoint")); + } + } + } + Ok(response.nodes) + } + + pub(super) async fn get_chunk(&self, address: &str) -> Result<(Vec, String), String> { + let address = super::protocol::normalize_hex(address, 32)?; + let mut fields = Map::new(); + fields.insert("address".to_string(), Value::from(address.clone())); + let response = self.request("get_chunk", fields, &[]).await?; + if response.header.get("status").and_then(Value::as_str) == Some("not_found") { + return Err(format!("chunk {address} was not found on this node")); + } + let header: ChunkResponse = serde_json::from_value(response.header) + .map_err(|error| format!("invalid CHUNK response: {error}"))?; + if header.response_type != "chunk" { + return Err("expected a CHUNK response".to_string()); + } + if header.address.to_ascii_lowercase() != address { + return Err("node returned a different chunk address".to_string()); + } + if header.size != response.content.len() { + return Err("chunk metadata size does not match its content".to_string()); + } + super::verify_record(&address, &response.content).map_err(|error| error.to_string())?; + Ok((response.content, address)) + } + + pub(super) async fn quote_chunk( + &self, + address: &str, + size: usize, + ) -> Result<(BrowserQuoteArtifact, bool), String> { + let address = super::protocol::normalize_hex(address, 32)?; + if size > super::protocol::MAX_BROWSER_RECORD_BYTES { + return Err(format!("invalid chunk size {size}")); + } + let mut fields = Map::new(); + fields.insert("address".to_string(), Value::from(address.clone())); + fields.insert("size".to_string(), Value::from(size)); + let response = self.request("quote_chunk", fields, &[]).await?; + let header: QuoteResponse = serde_json::from_value(response.header) + .map_err(|error| format!("invalid STORAGE_QUOTE response: {error}"))?; + if header.response_type != "storage_quote" { + return Err("expected a STORAGE_QUOTE response".to_string()); + } + if header.address.to_ascii_lowercase() != address { + return Err("node returned a quote for a different chunk address".to_string()); + } + Ok((header.quote, header.already_stored)) + } + + pub(super) async fn put_chunk( + &self, + address: &str, + content: &[u8], + quote: BrowserQuoteArtifact, + transaction_hash: &str, + ) -> Result<(String, bool), String> { + let address = super::protocol::normalize_hex(address, 32)?; + let transaction_hash = super::protocol::normalize_hex(transaction_hash, 32)?; + super::verify_record(&address, content).map_err(|error| error.to_string())?; + let mut fields = Map::new(); + fields.insert("address".to_string(), Value::from(address.clone())); + fields.insert( + "quote".to_string(), + serde_json::to_value(quote).map_err(|error| error.to_string())?, + ); + fields.insert( + "transaction_hash".to_string(), + Value::from(transaction_hash), + ); + let response = self.request("put_chunk", fields, content).await?; + let header: PutResponse = serde_json::from_value(response.header) + .map_err(|error| format!("invalid CHUNK_STORED response: {error}"))?; + if header.response_type != "chunk_stored" { + return Err("expected a CHUNK_STORED response".to_string()); + } + if header.address.to_ascii_lowercase() != address { + return Err("node stored a different chunk address".to_string()); + } + Ok((address, header.already_stored)) + } + + pub(super) fn close(&self) { + if let Some(connection) = self.connection.borrow_mut().take() { + connection.close(); + } + self.hello.borrow_mut().take(); + self.peer_id.borrow_mut().take(); + } +} + +struct BrowserNetworkCore { + seeds: Vec, + pool: Rc, +} + +impl BrowserNetworkCore { + fn new(seeds: Vec) -> Result { + if seeds.is_empty() { + return Err("at least one seed endpoint is required".to_string()); + } + let seeds = seeds + .into_iter() + .map(|seed| { + parse_webrtc_direct_multiaddr(&seed.multiaddr) + .map(|endpoint| BrowserEndpoint { + multiaddr: endpoint.multiaddr, + }) + .map_err(|error| error.to_string()) + }) + .collect::, _>>()?; + Ok(Self { + seeds, + pool: Rc::new(BrowserClientPool::new(DEFAULT_MAX_POOLED_CLIENTS)?), + }) + } + + async fn find_closest( + &self, + target: &str, + progress: &ProgressReporter, + ) -> Result { + let target_key = parse_lookup_key(target, "lookup target")?; + let failures = Rc::new(RefCell::new(Vec::new())); + let seed_futures = self.seeds.iter().cloned().map(|endpoint| { + let pool = Rc::clone(&self.pool); + let failures = Rc::clone(&failures); + let progress = progress.clone(); + async move { + let seed_name = endpoint.multiaddr.clone(); + let result = async { + let client = pool.client(&endpoint).await?; + let hello = client.hello().await?; + progress.report(&format!("Connected seed {}", hello.peer_id)); + BrowserLookupCandidate::parse(BrowserNode { + peer_id: hello.peer_id, + native_addresses: Vec::new(), + reliability: 1.0, + webrtc_direct: Some(hello.endpoint), + }) + } + .await; + match result { + Ok(candidate) => Some(candidate), + Err(error) => { + progress.report(&format!("Seed {seed_name} failed: {error}")); + failures.borrow_mut().push(BrowserLookupFailure { + peer_id: seed_name, + message: error, + }); + None + } + } + } + }); + let seed_candidates = join_all(seed_futures) + .await + .into_iter() + .flatten() + .collect::>(); + if seed_candidates.is_empty() { + let detail = failures + .borrow() + .iter() + .map(|failure| failure.message.as_str()) + .collect::>() + .join("; "); + return Err(format!( + "could not connect to any WebRtcDirect seed: {detail}" + )); + } + + let config = LookupConfig { + count: DEFAULT_LOOKUP_K, + alpha: DEFAULT_LOOKUP_ALPHA, + max_iterations: DEFAULT_MAX_LOOKUP_ITERATIONS, + ..LookupConfig::saorsa(DEFAULT_LOOKUP_K) + }; + let mut lookup = + IterativeLookup::new(target_key, config).map_err(|error| error.to_string())?; + let mut known_endpoints = HashMap::new(); + for candidate in seed_candidates { + if let Some(endpoint) = candidate.wire.webrtc_direct.clone() { + known_endpoints.insert(candidate.peer_id, endpoint); + let _ = lookup.add_candidate(candidate); + } + } + let mut query = BrowserNetworkLookupQuery { + pool: Rc::clone(&self.pool), + progress: progress.clone(), + failures: Rc::clone(&failures), + known_endpoints, + }; + run_iterative_lookup(&mut lookup, &mut query) + .await + .map_err(|error| error.to_string())?; + let nodes = lookup + .results() + .into_iter() + .map(|candidate| candidate.wire) + .collect(); + let queried = lookup.queried_peers().iter().map(hex::encode).collect(); + let failures = failures.borrow().clone(); + Ok(BrowserLookupResult { + nodes, + queried, + failures, + }) + } + + async fn get_chunk_from_closest( + &self, + address: &str, + progress: &ProgressReporter, + ) -> Result<(Vec, BrowserNode), String> { + let address = super::protocol::normalize_hex(address, 32)?; + let lookup = self.find_closest(&address, progress).await?; + let mut failures = Vec::new(); + for node in lookup.nodes { + let Some(endpoint) = node.webrtc_direct.as_ref() else { + continue; + }; + progress.report(&format!("Requesting {address} from {}", node.peer_id)); + let result = async { + let client = self.pool.client(endpoint).await?; + if client.peer_id().is_none() { + client.hello().await?; + } + client.get_chunk(&address).await + } + .await; + match result { + Ok((content, _)) => return Ok((content, node)), + Err(error) => { + progress.report(&format!( + "Node {} did not return the file: {error}", + node.peer_id + )); + failures.push(format!("{}: {error}", node.peer_id)); + } + } + } + Err(format!( + "no closest WebRtcDirect node returned chunk {address}{}", + if failures.is_empty() { + String::new() + } else { + format!(" ({})", failures.join("; ")) + } + )) + } +} + +struct BrowserNetworkLookupQuery { + pool: Rc, + progress: ProgressReporter, + failures: Rc>>, + known_endpoints: HashMap, +} + +impl LookupQuery for BrowserNetworkLookupQuery { + type Error = String; + + async fn query_batch( + &mut self, + target: LookupKey, + count: usize, + iteration: usize, + batch: Vec, + ) -> Result>, Self::Error> { + let target = hex::encode(target); + let futures = batch.into_iter().map(|candidate| { + let pool = Rc::clone(&self.pool); + let progress = self.progress.clone(); + let failures = Rc::clone(&self.failures); + let target = target.clone(); + async move { + let responder = candidate.peer_id; + let peer_id = candidate.wire.peer_id.clone(); + let result = async { + let endpoint = candidate.wire.webrtc_direct.as_ref().ok_or_else(|| { + "lookup candidate has no WebRTC Direct endpoint".to_string() + })?; + let client = pool.client(endpoint).await?; + if client.peer_id().is_none() { + client.hello().await?; + } + client.find_node(&target, count).await + } + .await; + match result { + Ok(nodes) => { + progress.report(&format!( + "Iteration {iteration}: {peer_id} returned {} nodes", + nodes.len() + )); + let candidates = nodes + .into_iter() + .filter_map(|wire| match BrowserLookupCandidate::parse(wire) { + Ok(candidate) => Some(candidate), + Err(error) => { + progress.report(&format!( + "Ignoring invalid candidate from {peer_id}: {error}" + )); + None + } + }) + .collect(); + LookupQueryOutcome::Succeeded { + responder, + candidates, + } + } + Err(error) => { + progress.report(&format!("Query {peer_id} failed: {error}")); + failures.borrow_mut().push(BrowserLookupFailure { + peer_id, + message: error, + }); + LookupQueryOutcome::Failed { responder } + } + } + } + }); + let mut outcomes = join_all(futures).await; + for outcome in &mut outcomes { + if let LookupQueryOutcome::Succeeded { candidates, .. } = outcome { + candidates.retain_mut(|candidate| { + if let Some(endpoint) = candidate.wire.webrtc_direct.clone() { + self.known_endpoints.insert(candidate.peer_id, endpoint); + } else if let Some(endpoint) = self.known_endpoints.get(&candidate.peer_id) { + candidate.wire.webrtc_direct = Some(endpoint.clone()); + } + candidate.wire.webrtc_direct.is_some() + }); + } + } + Ok(outcomes) + } +} + +#[derive(Clone, Default)] +struct ProgressReporter(Option); + +impl ProgressReporter { + fn from_js(value: Option) -> Self { + Self(value) + } + + fn report(&self, message: &str) { + if let Some(callback) = &self.0 { + let _ = callback.call1(&JsValue::NULL, &JsValue::from_str(message)); + } + } +} + +fn parse_lookup_key(value: &str, label: &str) -> Result { + let value = value.strip_prefix("0x").unwrap_or(value); + let bytes = hex::decode(value).map_err(|error| format!("invalid {label}: {error}"))?; + bytes.try_into().map_err(|bytes: Vec| { + format!( + "invalid {label}: expected 32 bytes, received {}", + bytes.len() + ) + }) +} + +#[derive(Debug, Serialize)] +struct BrowserDownloadResult { + #[serde(with = "serde_bytes")] + content: Vec, + hash: String, + #[serde(rename = "dataMapNode")] + data_map_node: BrowserNode, +} + +#[derive(Clone)] +struct StoreTarget { + peer_id: String, + endpoint: BrowserEndpoint, +} + +struct PreparedRecord { + record: super::BrowserRecord, + already_stored: bool, + targets: Vec, + verified: Option, +} + +#[derive(Debug, Deserialize)] +struct BrowserPaymentSubmission { + #[serde(rename = "transactionHash")] + transaction_hash: Option, + #[serde(rename = "totalAmount")] + total_amount: String, +} + +#[derive(Debug, Serialize)] +struct BrowserUploadResult { + file: PublicFileDescriptor, + #[serde(rename = "transactionHash", skip_serializing_if = "Option::is_none")] + transaction_hash: Option, + #[serde(rename = "storageCostAtto")] + storage_cost_atto: String, + records: usize, +} + +/// Stateful Autonomi browser client sharing Rust lookup and data workflows. +#[wasm_bindgen(js_name = BrowserNetworkClient)] +pub struct BrowserNetworkClient { + inner: Rc, +} + +#[wasm_bindgen(js_class = BrowserNetworkClient)] +impl BrowserNetworkClient { + /// Construct a reusable client around stable WebRTC Direct seed addresses. + #[wasm_bindgen(constructor)] + pub fn new(endpoints: JsValue) -> Result { + let endpoints: Vec = serde_wasm_bindgen::from_value(endpoints) + .map_err(|error| JsValue::from_str(&error.to_string()))?; + let endpoints = endpoints + .into_iter() + .map(|endpoint| BrowserEndpoint { + multiaddr: endpoint.multiaddr().to_string(), + }) + .collect(); + let inner = + BrowserNetworkCore::new(endpoints).map_err(|error| JsValue::from_str(&error))?; + Ok(Self { + inner: Rc::new(inner), + }) + } + + /// Run Saorsa's iterative closest-node lookup over Rust-owned DataChannels. + #[wasm_bindgen(js_name = findClosest)] + pub async fn find_closest( + &self, + target: &str, + on_progress: Option, + ) -> Result { + let progress = ProgressReporter::from_js(on_progress); + let result = self + .inner + .find_closest(target, &progress) + .await + .map_err(|error| JsValue::from_str(&error))?; + serde_wasm_bindgen::to_value(&result).map_err(|error| JsValue::from_str(&error.to_string())) + } + + /// Download and reconstruct a complete public Autonomi file. + #[wasm_bindgen(js_name = downloadPublicFile)] + pub async fn download_public_file( + &self, + file: JsValue, + concurrency: usize, + on_progress: Option, + ) -> Result { + let file: PublicFileDescriptor = serde_wasm_bindgen::from_value(file) + .map_err(|error| JsValue::from_str(&error.to_string()))?; + let progress = ProgressReporter::from_js(on_progress); + let result = self + .download_public_file_inner(file, concurrency, &progress) + .await + .map_err(|error| JsValue::from_str(&error))?; + serde_wasm_bindgen::to_value(&result).map_err(|error| JsValue::from_str(&error.to_string())) + } + + /// Self-encrypt, quote, pay through a wallet callback, and store a public file. + #[wasm_bindgen(js_name = uploadPublicFile)] + pub async fn upload_public_file( + &self, + content: &[u8], + name: &str, + content_type: &str, + payment_network: JsValue, + pay_for_quotes: js_sys::Function, + on_progress: Option, + ) -> Result { + let payment_network: BrowserPaymentNetwork = + serde_wasm_bindgen::from_value(payment_network) + .map_err(|error| JsValue::from_str(&error.to_string()))?; + let payment_network = validate_browser_payment_network(payment_network) + .map_err(|error| JsValue::from_str(&error.to_string()))?; + let progress = ProgressReporter::from_js(on_progress); + let result = self + .upload_public_file_inner( + content, + name, + content_type, + payment_network, + &pay_for_quotes, + &progress, + ) + .await + .map_err(|error| JsValue::from_str(&error))?; + serde_wasm_bindgen::to_value(&result).map_err(|error| JsValue::from_str(&error.to_string())) + } + + /// Close all pooled WebRTC associations. + pub fn close(&self) { + self.inner.pool.close(); + } +} + +impl BrowserNetworkClient { + async fn download_public_file_inner( + &self, + file: PublicFileDescriptor, + concurrency: usize, + progress: &ProgressReporter, + ) -> Result { + let address = super::protocol::normalize_hex(&file.address, 32)?; + let expected_hash = super::protocol::normalize_hex(&file.blake3, 32)?; + if concurrency == 0 { + return Err("download concurrency must be a positive integer".to_string()); + } + let concurrency = concurrency.min(MAX_DOWNLOAD_CONCURRENCY); + progress.report(&format!("Fetching public DataMap {address}")); + let (data_map, data_map_node) = self + .inner + .get_chunk_from_closest(&address, progress) + .await?; + if data_map.len() != file.data_map_size { + return Err(format!( + "public DataMap has {} bytes, expected {}", + data_map.len(), + file.data_map_size + )); + } + progress.report(&format!( + "Verified public DataMap ({} bytes)", + data_map.len() + )); + let chunks = super::decode_public_data_map(&data_map).map_err(|error| error.to_string())?; + if chunks.len() < 3 { + return Err("ant-core returned an invalid public DataMap".to_string()); + } + let total = chunks.len(); + let downloads = stream::iter(chunks.into_iter().enumerate()) + .map(|(position, chunk)| { + let inner = Rc::clone(&self.inner); + let progress = progress.clone(); + async move { + progress.report(&format!( + "Fetching encrypted file chunk {}/{} ({})", + chunk.index + 1, + total, + chunk.dst_hash + )); + inner + .get_chunk_from_closest(&chunk.dst_hash, &progress) + .await + .map(|(content, _)| (position, content)) + } + }) + .buffer_unordered(concurrency) + .collect::>() + .await; + let mut encrypted_chunks = Vec::with_capacity(total); + for download in downloads { + encrypted_chunks.push(download?); + } + encrypted_chunks.sort_by_key(|(position, _)| *position); + let encrypted_chunks = encrypted_chunks + .into_iter() + .map(|(_, content)| content) + .collect::>(); + progress.report(&format!( + "Reconstructing {} with native ant-core WASM", + file.name + )); + let content = super::decrypt_public_file(&data_map, &encrypted_chunks) + .map_err(|error| error.to_string())?; + if content.len() != file.size { + return Err(format!( + "reconstructed file has {} bytes, expected {}", + content.len(), + file.size + )); + } + super::verify_record(&expected_hash, &content).map_err(|error| error.to_string())?; + progress.report(&format!( + "Verified complete {} as {expected_hash}", + file.name + )); + Ok(BrowserDownloadResult { + content, + hash: expected_hash, + data_map_node, + }) + } + + async fn upload_public_file_inner( + &self, + content: &[u8], + name: &str, + content_type: &str, + payment_network: BrowserPaymentNetwork, + pay_for_quotes: &js_sys::Function, + progress: &ProgressReporter, + ) -> Result { + if name.is_empty() { + return Err("upload file has no name".to_string()); + } + progress.report(&format!( + "Self-encrypting {name} with native ant-core WASM ({} bytes)", + content.len() + )); + let encrypted = super::encrypt_public_file(content).map_err(|error| error.to_string())?; + let mut prepared = Vec::with_capacity(encrypted.records.len()); + for (index, record) in encrypted.records.iter().cloned().enumerate() { + progress.report(&format!( + "Preparing record {}/{}", + index + 1, + encrypted.records.len() + )); + prepared.push( + self.prepare_record(record, &payment_network, progress) + .await?, + ); + } + let verified_quotes = prepared + .iter() + .filter_map(|record| record.verified.clone()) + .collect::>(); + let expected_total = + storage_payment_total(&verified_quotes).map_err(|error| error.to_string())?; + let mut payment = if verified_quotes.is_empty() { + BrowserPaymentSubmission { + transaction_hash: None, + total_amount: "0".to_string(), + } + } else { + invoke_payment(pay_for_quotes, &payment_network, &verified_quotes).await? + }; + if !verified_quotes.is_empty() && payment.transaction_hash.is_none() { + return Err("wallet callback returned no storage payment transaction".to_string()); + } + if payment.total_amount != expected_total { + return Err(format!( + "wallet callback reported payment total {}, expected {expected_total}", + payment.total_amount + )); + } + if let Some(transaction_hash) = payment.transaction_hash.as_mut() { + *transaction_hash = super::protocol::normalize_hex(transaction_hash, 32)?; + } + + let mut replicas = usize::MAX; + for (index, record) in prepared.iter().enumerate() { + progress.report(&format!("Storing record {}/{}", index + 1, prepared.len())); + let stored = self + .store_prepared( + record, + &payment_network, + payment.transaction_hash.as_deref(), + progress, + ) + .await?; + replicas = replicas.min(stored); + } + let descriptor = PublicFileDescriptor { + name: name.to_string(), + address: encrypted.address, + size: content.len(), + content_type: if content_type.is_empty() { + "application/octet-stream".to_string() + } else { + content_type.to_string() + }, + blake3: encrypted.blake3, + data_map_size: encrypted.data_map_size, + chunks: encrypted.chunks, + replicas: if replicas == usize::MAX { 0 } else { replicas }, + }; + Ok(BrowserUploadResult { + file: descriptor, + transaction_hash: payment.transaction_hash, + storage_cost_atto: payment.total_amount, + records: prepared.len(), + }) + } + + async fn prepare_record( + &self, + record: super::BrowserRecord, + payment_network: &BrowserPaymentNetwork, + progress: &ProgressReporter, + ) -> Result { + progress.report(&format!("Finding closest nodes for {}", record.address)); + let lookup = self.inner.find_closest(&record.address, progress).await?; + let targets = lookup + .nodes + .into_iter() + .filter_map(|node| { + node.webrtc_direct.map(|endpoint| StoreTarget { + peer_id: node.peer_id, + endpoint, + }) + }) + .take(MAX_STORE_TARGETS) + .collect::>(); + if targets.is_empty() { + return Err( + "closest-node lookup returned no WebRTC Direct storage targets".to_string(), + ); + } + let mut failures = Vec::new(); + for target in &targets { + let result = async { + let client = self.inner.pool.client(&target.endpoint).await?; + let hello = client.hello().await?; + assert_upload_node(&hello, payment_network)?; + let (quote, already_stored) = client + .quote_chunk(&record.address, record.content.len()) + .await?; + let verified = verify_storage_quote(quote, &record.address, &target.peer_id) + .map_err(|error| error.to_string())?; + Ok::<_, String>((already_stored, verified)) + } + .await; + match result { + Ok((true, _)) => { + progress.report(&format!( + "Chunk {} is already stored; skipping payment", + record.address + )); + return Ok(PreparedRecord { + record, + already_stored: true, + targets, + verified: None, + }); + } + Ok((false, verified)) => { + progress.report(&format!( + "Verified storage quote {} from {}", + verified.quote_hash, target.peer_id + )); + let mut ordered_targets = Vec::with_capacity(targets.len()); + ordered_targets.push(target.clone()); + ordered_targets.extend( + targets + .iter() + .filter(|candidate| candidate.peer_id != target.peer_id) + .cloned(), + ); + return Ok(PreparedRecord { + record, + already_stored: false, + targets: ordered_targets, + verified: Some(verified), + }); + } + Err(error) => failures.push(format!("{}: {error}", target.peer_id)), + } + } + Err(format!( + "no closest node supplied a valid quote ({})", + failures.join("; ") + )) + } + + async fn store_prepared( + &self, + prepared: &PreparedRecord, + payment_network: &BrowserPaymentNetwork, + transaction_hash: Option<&str>, + progress: &ProgressReporter, + ) -> Result { + if prepared.already_stored { + return Ok(1); + } + let transaction_hash = transaction_hash + .ok_or_else(|| "paid record has no transaction hash".to_string())? + .to_string(); + let verified = prepared + .verified + .as_ref() + .ok_or_else(|| "paid record has no verified quote".to_string())?; + let attempts = prepared.targets.iter().cloned().map(|target| { + let pool = Rc::clone(&self.inner.pool); + let record = prepared.record.clone(); + let quote = verified.quote.clone(); + let payment_network = payment_network.clone(); + let transaction_hash = transaction_hash.clone(); + let progress = progress.clone(); + async move { + let client = pool.client(&target.endpoint).await?; + let hello = client.hello().await?; + assert_upload_node(&hello, &payment_network)?; + let (_, already_stored) = client + .put_chunk(&record.address, &record.content, quote, &transaction_hash) + .await?; + progress.report(&format!( + "{} {} on {}", + if already_stored { + "Confirmed" + } else { + "Stored" + }, + record.address, + target.peer_id + )); + Ok::<(), String>(()) + } + }); + let attempts = join_all(attempts).await; + let stored = attempts.iter().filter(|attempt| attempt.is_ok()).count(); + if stored == 0 { + let failures = attempts + .into_iter() + .filter_map(Result::err) + .collect::>() + .join("; "); + return Err(format!( + "paid chunk was rejected by every closest node: {failures}" + )); + } + Ok(stored) + } +} + +async fn invoke_payment( + callback: &js_sys::Function, + payment_network: &BrowserPaymentNetwork, + quotes: &[VerifiedStorageQuote], +) -> Result { + let payment_network = + serde_wasm_bindgen::to_value(payment_network).map_err(|error| error.to_string())?; + let quotes = serde_wasm_bindgen::to_value(quotes).map_err(|error| error.to_string())?; + let returned = callback + .call2(&JsValue::NULL, &payment_network, "es) + .map_err(js_error_message)?; + let returned = JsFuture::from(Promise::resolve(&returned)) + .await + .map_err(js_error_message)?; + serde_wasm_bindgen::from_value(returned) + .map_err(|error| format!("wallet callback returned an invalid payment result: {error}")) +} + +fn assert_upload_node( + hello: &BrowserHello, + expected: &BrowserPaymentNetwork, +) -> Result<(), String> { + if !hello + .capabilities + .iter() + .any(|value| value == "quote_chunk") + || !hello.capabilities.iter().any(|value| value == "put_chunk") + { + return Err("node does not advertise paid browser uploads".to_string()); + } + let advertised: BrowserPaymentNetwork = serde_json::from_value(hello.payment.clone()) + .map_err(|error| format!("node advertises invalid payment configuration: {error}"))?; + let advertised_rpc = url::Url::parse(&advertised.rpc_url) + .map_err(|error| format!("node advertises invalid payment RPC URL: {error}"))?; + let expected_rpc = url::Url::parse(&expected.rpc_url) + .map_err(|error| format!("manifest has invalid payment RPC URL: {error}"))?; + if advertised_rpc != expected_rpc + || !advertised + .payment_token_address + .eq_ignore_ascii_case(&expected.payment_token_address) + || !advertised + .payment_vault_address + .eq_ignore_ascii_case(&expected.payment_vault_address) + { + return Err("node advertises a different payment network than the manifest".to_string()); + } + Ok(()) +} + +/// One authenticated browser-to-node WebRTC Direct client implemented in Rust. +#[wasm_bindgen(js_name = BrowserNodeClient)] +pub struct BrowserNodeClient { + inner: Rc, +} + +#[wasm_bindgen(js_class = BrowserNodeClient)] +impl BrowserNodeClient { + /// Construct a client from a raw or structured WebRTC Direct endpoint. + #[wasm_bindgen(constructor)] + pub fn new(endpoint: JsValue) -> Result { + let endpoint: BrowserEndpointInput = serde_wasm_bindgen::from_value(endpoint) + .map_err(|error| JsValue::from_str(&error.to_string()))?; + let endpoint = parse_webrtc_direct_multiaddr(endpoint.multiaddr()) + .map_err(|error| JsValue::from_str(&error.to_string()))?; + Ok(Self { + inner: Rc::new(BrowserNodeClientCore::new(endpoint)), + }) + } + + /// Authenticated peer ID, when HELLO has completed. + #[wasm_bindgen(getter, js_name = peerId)] + pub fn peer_id(&self) -> Option { + self.inner.peer_id() + } + + /// Open the direct DataChannel without issuing an application request. + pub async fn connect(&self) -> Result<(), JsValue> { + let _guard = self.inner.request_lock.lock().await; + self.inner + .ensure_connected() + .await + .map_err(|error| JsValue::from_str(&error)) + } + + /// Authenticate the connected node. + pub async fn hello(&self) -> Result { + let hello = self + .inner + .hello() + .await + .map_err(|error| JsValue::from_str(&error))?; + hello + .serialize(&serde_wasm_bindgen::Serializer::new().serialize_maps_as_objects(true)) + .map_err(|error| JsValue::from_str(&error.to_string())) + } + + /// Request nodes closest to a 32-byte target. + #[wasm_bindgen(js_name = findNode)] + pub async fn find_node(&self, target: &str, count: usize) -> Result { + let nodes = self + .inner + .find_node(target, count) + .await + .map_err(|error| JsValue::from_str(&error))?; + serde_wasm_bindgen::to_value(&nodes).map_err(|error| JsValue::from_str(&error.to_string())) + } + + /// Retrieve and BLAKE3-verify one content-addressed record. + #[wasm_bindgen(js_name = getChunk)] + pub async fn get_chunk(&self, address: &str) -> Result { + let (content, hash) = self + .inner + .get_chunk(address) + .await + .map_err(|error| JsValue::from_str(&error))?; + serde_wasm_bindgen::to_value(&BrowserChunk { content, hash }) + .map_err(|error| JsValue::from_str(&error.to_string())) + } + + /// Request a signed storage quote. + #[wasm_bindgen(js_name = quoteChunk)] + pub async fn quote_chunk(&self, address: &str, size: usize) -> Result { + let (quote, already_stored) = self + .inner + .quote_chunk(address, size) + .await + .map_err(|error| JsValue::from_str(&error))?; + serde_wasm_bindgen::to_value(&BrowserQuoteResponse { + quote, + already_stored, + }) + .map_err(|error| JsValue::from_str(&error.to_string())) + } + + /// Store a paid content-addressed record. + #[wasm_bindgen(js_name = putChunk)] + pub async fn put_chunk( + &self, + address: &str, + content: &[u8], + quote: JsValue, + transaction_hash: &str, + ) -> Result { + let quote: BrowserQuoteArtifact = serde_wasm_bindgen::from_value(quote) + .map_err(|error| JsValue::from_str(&error.to_string()))?; + let (address, already_stored) = self + .inner + .put_chunk(address, content, quote, transaction_hash) + .await + .map_err(|error| JsValue::from_str(&error))?; + serde_wasm_bindgen::to_value(&BrowserPutResponse { + address, + already_stored, + }) + .map_err(|error| JsValue::from_str(&error.to_string())) + } + + /// Close the DataChannel and peer connection. + pub fn close(&self) { + self.inner.close(); + } +} + +async fn read_response(receiver: ResponseInbox) -> Result { + let mut frame = Vec::with_capacity(8 * 1024); + let mut expected_length = None; + loop { + let next = receiver.lock().await.next().await; + let message = next + .ok_or_else(|| "response ended before its declared frame was complete".to_string())??; + let next_length = frame + .len() + .checked_add(message.len()) + .ok_or_else(|| "response length overflow".to_string())?; + if next_length > MAX_BROWSER_RESPONSE_BYTES { + return Err(format!( + "response exceeded the {MAX_BROWSER_RESPONSE_BYTES}-byte client limit" + )); + } + frame.extend_from_slice(&message); + if expected_length.is_none() { + expected_length = response_frame_length(&frame).map_err(|error| error.to_string())?; + } + if let Some(expected) = expected_length { + if frame.len() > expected { + return Err("response contains bytes after its declared frame".to_string()); + } + if frame.len() == expected { + return parse_response_frame(&frame).map_err(|error| error.to_string()); + } + } + } +} + +async fn wait_for_capacity(channel: &RtcDataChannel) -> Result<(), String> { + if channel.buffered_amount() <= MAX_BUFFERED_AMOUNT { + return Ok(()); + } + channel.set_buffered_amount_low_threshold(MAX_BUFFERED_AMOUNT / 2); + let (sender, receiver) = oneshot::channel::<()>(); + let sender = Rc::new(RefCell::new(Some(sender))); + let ready_sender = Rc::clone(&sender); + let on_ready = Closure::::new(move |_event: Event| { + if let Some(sender) = ready_sender.borrow_mut().take() { + let _ = sender.send(()); + } + }); + channel.set_onbufferedamountlow(Some(on_ready.as_ref().unchecked_ref())); + let result = timeout( + async move { + receiver + .await + .map_err(|_| "WebRTC DataChannel closed while draining".to_string()) + }, + "WebRTC DataChannel drain timed out", + ) + .await; + channel.set_onbufferedamountlow(None); + drop(on_ready); + result +} + +async fn timeout(future: F, message: &'static str) -> Result +where + F: Future>, +{ + let operation = Box::pin(future); + let timer = Box::pin(TimeoutFuture::new(REQUEST_TIMEOUT_MS)); + match select(operation, timer).await { + Either::Left((result, _)) => result, + Either::Right(((), _)) => Err(message.to_string()), + } +} + +fn random_ice_credential() -> Result { + let mut random = [0u8; 32]; + getrandom::getrandom(&mut random) + .map_err(|error| format!("browser entropy failed: {error}"))?; + let mut credential = String::with_capacity(ICE_CREDENTIAL_PREFIX.len() + random.len()); + credential.push_str(ICE_CREDENTIAL_PREFIX); + for byte in random { + credential.push(char::from( + ICE_ALPHABET[usize::from(byte) % ICE_ALPHABET.len()], + )); + } + Ok(credential) +} + +fn js_error_message(value: JsValue) -> String { + value + .as_string() + .or_else(|| { + js_sys::Reflect::get(&value, &JsValue::from_str("message")) + .ok()? + .as_string() + }) + .unwrap_or_else(|| format!("browser WebRTC operation failed: {value:?}")) +} + +impl From for JsValue { + fn from(error: BrowserProtocolError) -> Self { + JsValue::from_str(&error.to_string()) + } +} diff --git a/ant-core/src/lib.rs b/ant-core/src/lib.rs index c39dea3b..9404d66b 100644 --- a/ant-core/src/lib.rs +++ b/ant-core/src/lib.rs @@ -1,4 +1,4 @@ -/// Browser-safe immutable-data primitives shared by native and WASM clients. +/// Cross-platform Autonomi client logic and browser WASM bindings. pub mod browser; #[cfg(feature = "native")] diff --git a/web/README.md b/web/README.md index 43d0cb92..4eb92d2a 100644 --- a/web/README.md +++ b/web/README.md @@ -3,11 +3,12 @@ This web application is the browser-facing client for ADR-0009. It loads a local testnet bootstrap manifest, connects directly to storage nodes over WebRTC Direct, and performs closest-node lookup through Saorsa's shared Rust -lookup engine. The portable part of `ant-core` is compiled to WASM and drives -the Kademlia walk, native self-encryption, public DataMap serialization, -reconstruction, and BLAKE3 content verification. The thin JavaScript layer -executes WebRTC Direct requests, retrieves and uploads records, uses the browser -wallet/payment APIs, and drives the page. +lookup engine. `ant-core` is compiled to WASM and owns the WebRTC peer +connections and data channels, wire framing, authenticated HELLO, connection +pool, Kademlia walk, native self-encryption, quote and commitment verification, +payment planning, record upload/download, public DataMap serialization, +reconstruction, and BLAKE3 content verification. JavaScript drives the page, +browser file/save APIs, and Ethers transaction submission. The node-side WebRTC Direct listener and testnet manifest API live in the `ant-node-web-support` sibling repository. No HTTP gateway performs lookup or @@ -23,7 +24,8 @@ proxies file bytes. extracts the stable certificate fingerprint from each node multiaddress. Nodes serialize these addresses from the native `saorsa_core::MultiAddr` -representation; the JavaScript parser consumes that canonical string form. +representation; the shared Rust client parser consumes that canonical string +form in WASM. Install the WASM build tools once if needed: @@ -89,14 +91,16 @@ manifest from port 25000 and fills in the default file: 1. **Load testnet** refreshes the manifest and direct multiaddress catalog. 2. **Connect** parses the first node multiaddress and performs a pinned WebRTC Direct `HELLO` and verifies its ML-DSA identity signature. -3. **Find closest** runs Saorsa's iterative lookup engine in WASM, with - JavaScript executing its WebRTC Direct query batches. +3. **Find closest** runs Saorsa's iterative lookup engine and WebRTC Direct + query batches entirely in Rust/WASM. 4. Under **Paid public file upload**, choose a file, paste the funded private key printed by ant-devnet, then select **Pay and upload file**. Rust/WASM - performs encryption and DataMap generation; quote/commitment verification, - approval, and payment happen locally in the browser. The key field is - cleared immediately and the resulting public DataMap address is placed in - the download field. + performs encryption, DataMap generation, closest-node selection, + quote/commitment verification, payment-total calculation, and storage. A + narrow JavaScript callback uses Ethers for token approval and the wallet + transaction; Rust verifies the callback's reported total before continuing. + The key field is cleared immediately and the resulting public DataMap + address is placed in the download field. 5. **Download and save file** opens the browser save flow, fetches the public DataMap and encrypted chunks from direct closest storage nodes, reconstructs the whole file, verifies BLAKE3, and retains a **Save again** link. @@ -122,7 +126,7 @@ boundary directly: ```bash cargo check -p ant-core --target wasm32-unknown-unknown \ --no-default-features --features browser-wasm -cargo test -p ant-core --lib browser::tests +cargo test -p ant-core --lib browser:: ``` The tests cover Saorsa's shared lookup engine and generic query driver, fixed-width IDs, @@ -135,15 +139,30 @@ downloads all public-file records through WebRTC Direct, reconstructs the original bytes, pays a real quote on local Anvil, uploads a fresh record through the ordinary node payment validator, and reads it back. -## Current boundary - -JavaScript currently owns native `RTCPeerConnection`/`RTCDataChannel` handling, Ethers payment -submission, ML-DSA quote verification, and DOM/save APIs. Rust/WASM owns the -complete iterative lookup loop; JavaScript implements only its transport batch -callback. -`ant-protocol 2.3.1` still enables native Saorsa/Tokio networking and cannot be -linked into a browser WASM build; a future transport-free feature can move the -remaining quote and multiaddress verification into Rust. +## Library boundary + +The reusable Autonomi behavior belongs to `ant-core`, the library crate in this +repository. Its cross-platform `browser` modules own bootstrap manifest and +public-file types, WebRTC Direct addresses and framing, HELLO identity +authentication, BLAKE3/self-encryption, Saorsa lookup, storage quote and native +commitment verification, pricing and payment planning, and complete public +upload/download workflows. The `browser-wasm` host adapter owns +`RTCPeerConnection` and `RTCDataChannel` through `web-sys`, allowing any web +application to use `BrowserNetworkClient` without copying the Autonomi protocol +into JavaScript. + +The application JavaScript owns only capabilities tied to the page or the +selected wallet stack: DOM events, browser `File`/save-picker APIs, and Ethers +contract calls. This is also the deliberate extension seam: another web app can +provide its own UI and wallet callback while sharing all network and Autonomi +logic from the Rust library. + +`ant-protocol 2.3.1` still couples its wire types to native Saorsa/Tokio +networking, so it cannot be linked into this WASM build. The browser verifier +therefore uses the same FIPS-204 ML-DSA-65 primitive directly. Splitting a +transport-free wire/crypto feature from `ant-protocol` would remove that final +dependency-level duplication and let native and WASM builds import the exact +same protocol types. The local testnet manifest is intentionally unsigned bootstrap material. A production deployment still needs ML-DSA-signed endpoint records, exceptional diff --git a/web/package-lock.json b/web/package-lock.json index 68109b22..68df99a4 100644 --- a/web/package-lock.json +++ b/web/package-lock.json @@ -8,9 +8,6 @@ "name": "ant-client-web", "version": "0.1.0", "dependencies": { - "@msgpack/msgpack": "^3.1.3", - "@noble/hashes": "2.2.0", - "@noble/post-quantum": "^0.6.1", "ethers": "^6.17.0" }, "devDependencies": { @@ -83,71 +80,6 @@ "@jridgewell/sourcemap-codec": "^1.4.14" } }, - "node_modules/@msgpack/msgpack": { - "version": "3.1.3", - "resolved": "https://registry.npmjs.org/@msgpack/msgpack/-/msgpack-3.1.3.tgz", - "integrity": "sha512-47XIizs9XZXvuJgoaJUIE2lFoID8ugvc0jzSHP+Ptfk8nTbnR8g788wv48N03Kx0UkAv559HWRQ3yzOgzlRNUA==", - "license": "ISC", - "engines": { - "node": ">= 18" - } - }, - "node_modules/@noble/ciphers": { - "version": "2.2.0", - "resolved": "https://registry.npmjs.org/@noble/ciphers/-/ciphers-2.2.0.tgz", - "integrity": "sha512-Z6pjIZ/8IJcCGzb2S/0Px5J81yij85xASuk1teLNeg75bfT07MV3a/O2Mtn1I2se43k3lkVEcFaR10N4cgQcZA==", - "license": "MIT", - "engines": { - "node": ">= 20.19.0" - }, - "funding": { - "url": "https://paulmillr.com/funding/" - } - }, - "node_modules/@noble/curves": { - "version": "2.2.0", - "resolved": "https://registry.npmjs.org/@noble/curves/-/curves-2.2.0.tgz", - "integrity": "sha512-T/BoHgFXirb0ENSPBquzX0rcjXeM6Lo892a2jlYJkqk83LqZx0l1Of7DzlKJ6jkpvMrkHSnAcgb5JegL8SeIkQ==", - "license": "MIT", - "dependencies": { - "@noble/hashes": "2.2.0" - }, - "engines": { - "node": ">= 20.19.0" - }, - "funding": { - "url": "https://paulmillr.com/funding/" - } - }, - "node_modules/@noble/hashes": { - "version": "2.2.0", - "resolved": "https://registry.npmjs.org/@noble/hashes/-/hashes-2.2.0.tgz", - "integrity": "sha512-IYqDGiTXab6FniAgnSdZwgWbomxpy9FtYvLKs7wCUs2a8RkITG+DFGO1DM9cr+E3/RgADRpFjrKVaJ1z6sjtEg==", - "license": "MIT", - "engines": { - "node": ">= 20.19.0" - }, - "funding": { - "url": "https://paulmillr.com/funding/" - } - }, - "node_modules/@noble/post-quantum": { - "version": "0.6.1", - "resolved": "https://registry.npmjs.org/@noble/post-quantum/-/post-quantum-0.6.1.tgz", - "integrity": "sha512-+pormrDZwjRw05U8ADK4JpHejo87+gBd+muRBB/ozztH5yhDLMDF4jHQWN3NQQAsu1zBNPWTG0ZwVI0CR29H0A==", - "license": "MIT", - "dependencies": { - "@noble/ciphers": "~2.2.0", - "@noble/curves": "~2.2.0", - "@noble/hashes": "~2.2.0" - }, - "engines": { - "node": ">= 20.19.0" - }, - "funding": { - "url": "https://paulmillr.com/funding/" - } - }, "node_modules/@oxc-project/types": { "version": "0.142.0", "resolved": "https://registry.npmjs.org/@oxc-project/types/-/types-0.142.0.tgz", @@ -817,9 +749,9 @@ } }, "node_modules/nanoid": { - "version": "3.3.17", - "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.17.tgz", - "integrity": "sha512-xQLf0A3HOMlgHq0n247/LRuAOYmB7dXJ/DvAxGvsSBij45XtBSmQycu+F8ODbHwns/XyFZagyL1+J0Offw1E0g==", + "version": "3.3.18", + "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.18.tgz", + "integrity": "sha512-DTg4MJbGMWkfi6VZFdNt2/caMbQy4Ou+Op/hJQvGEWcnVfoA1QA+xzRKAzw9jD6+GVOOeYr/mIcuDSdug6F6+w==", "dev": true, "funding": [ { diff --git a/web/package.json b/web/package.json index 4979f5d5..827ec5be 100644 --- a/web/package.json +++ b/web/package.json @@ -14,9 +14,6 @@ "test": "node --import ./setup-wasm.js --test" }, "dependencies": { - "@msgpack/msgpack": "^3.1.3", - "@noble/hashes": "2.2.0", - "@noble/post-quantum": "^0.6.1", "ethers": "^6.17.0" }, "devDependencies": { diff --git a/web/src/file.js b/web/src/file.js deleted file mode 100644 index bcb5b871..00000000 --- a/web/src/file.js +++ /dev/null @@ -1,97 +0,0 @@ -import { - decodePublicDataMap as decodePublicDataMapNative, - decryptPublicFile as decryptPublicFileNative, -} from "../pkg/ant_core.js"; -import { - BrowserNodeClientPool, - getChunkFromClosest, - hexToBytes, - verifyChunk, -} from "./protocol.js"; - -const MAX_DOWNLOAD_CONCURRENCY = 6; - -async function mapWithConcurrency(items, concurrency, operation) { - const results = new Array(items.length); - let next = 0; - const workerCount = Math.min(items.length, concurrency); - await Promise.all( - Array.from({ length: workerCount }, async () => { - while (next < items.length) { - const index = next; - next += 1; - results[index] = await operation(items[index], index); - } - }), - ); - return results; -} - -export async function downloadPublicFile( - seedEndpoints, - file, - { - concurrency = 3, - onProgress = () => {}, - downloadChunk = getChunkFromClosest, - decodeDataMap = decodePublicDataMapNative, - decrypt = decryptPublicFileNative, - } = {}, -) { - hexToBytes(file.address, 32); - hexToBytes(file.blake3, 32); - if (!Number.isSafeInteger(concurrency) || concurrency < 1) { - throw new Error("Download concurrency must be a positive integer"); - } - const boundedConcurrency = Math.min(concurrency, MAX_DOWNLOAD_CONCURRENCY); - - const clientPool = new BrowserNodeClientPool(); - try { - onProgress(`Fetching public DataMap ${file.address}`); - const dataMap = await downloadChunk(seedEndpoints, file.address, { - onProgress, - clientPool, - }); - if (dataMap.content.length !== file.data_map_size) { - throw new Error( - `Public DataMap has ${dataMap.content.length} bytes, expected ${file.data_map_size}`, - ); - } - onProgress(`Verified public DataMap (${dataMap.content.length} bytes)`); - const chunks = decodeDataMap(dataMap.content); - if (!Array.isArray(chunks) || chunks.length < 3) { - throw new Error("ant-core WASM returned an invalid public DataMap"); - } - - const encryptedChunks = await mapWithConcurrency( - chunks, - boundedConcurrency, - async (chunk) => { - onProgress( - `Fetching encrypted file chunk ${chunk.index + 1}/${chunks.length} (${chunk.dst_hash})`, - ); - const downloaded = await downloadChunk(seedEndpoints, chunk.dst_hash, { - onProgress, - clientPool, - }); - return downloaded.content; - }, - ); - - onProgress(`Reconstructing ${file.name} with native ant-core WASM`); - const content = decrypt(dataMap.content, encryptedChunks); - if (!(content instanceof Uint8Array)) { - throw new Error("ant-core WASM returned non-byte file content"); - } - if (content.length !== file.size) { - throw new Error( - `Reconstructed file has ${content.length} bytes, expected ${file.size}`, - ); - } - const hash = verifyChunk(file.blake3, content); - onProgress(`Verified complete ${file.name} as ${hash}`); - return { content, hash, dataMapNode: dataMap.node }; - } finally { - clientPool.close(); - } -} diff --git a/web/src/file.test.js b/web/src/file.test.js deleted file mode 100644 index 86200dab..00000000 --- a/web/src/file.test.js +++ /dev/null @@ -1,60 +0,0 @@ -import assert from "node:assert/strict"; -import test from "node:test"; -import { blake3 } from "@noble/hashes/blake3.js"; -import { downloadPublicFile } from "./file.js"; -import { bytesToHex } from "./protocol.js"; - -test("downloads records and delegates reconstruction to ant-core WASM", async () => { - const content = new TextEncoder().encode("browser whole-file fixture\n".repeat(160)); - const dataMapContent = Uint8Array.of(9, 8, 7); - const dataMapAddress = bytesToHex(blake3(dataMapContent)); - const chunks = [0, 1, 2].map((index) => ({ - index, - dst_hash: (index + 1).toString(16).padStart(2, "0").repeat(32), - src_hash: (index + 4).toString(16).padStart(2, "0").repeat(32), - src_size: content.length / 3, - })); - const encryptedByAddress = new Map( - chunks.map((chunk, index) => [chunk.dst_hash, Uint8Array.of(index)]), - ); - encryptedByAddress.set(dataMapAddress, dataMapContent); - const requested = []; - const downloadChunk = async (_seeds, address) => { - requested.push(address); - const bytes = encryptedByAddress.get(address); - if (!bytes) throw new Error(`No fixture record ${address}`); - return { content: bytes, node: { peer_id: "11".repeat(32) } }; - }; - - const result = await downloadPublicFile( - [], - { - name: "fixture.txt", - address: dataMapAddress, - size: content.length, - content_type: "text/plain", - blake3: bytesToHex(blake3(content)), - data_map_size: dataMapContent.length, - chunks, - }, - { - downloadChunk, - decodeDataMap: (receivedDataMap) => { - assert.deepEqual(receivedDataMap, dataMapContent); - return chunks; - }, - decrypt: (receivedDataMap, encryptedContents) => { - assert.deepEqual(receivedDataMap, dataMapContent); - assert.deepEqual( - encryptedContents, - chunks.map((_, index) => Uint8Array.of(index)), - ); - return content; - }, - }, - ); - - assert.deepEqual(result.content, content); - assert.equal(result.hash, bytesToHex(blake3(content))); - assert.deepEqual(new Set(requested), new Set([dataMapAddress, ...chunks.map((c) => c.dst_hash)])); -}); diff --git a/web/src/main.js b/web/src/main.js index 3ff53bc5..5bff2a2a 100644 --- a/web/src/main.js +++ b/web/src/main.js @@ -1,14 +1,11 @@ import "./style.css"; -import initAntCore from "../pkg/ant_core.js"; import { + BrowserNetworkClient, BrowserNodeClient, - bytesToHex, - hexToBytes, - iterativeFindClosest, -} from "./protocol.js"; -import { downloadPublicFile } from "./file.js"; + default as initAntCore, +} from "../pkg/ant_core.js"; import { fetchBrowserManifest } from "./manifest.js"; -import { uploadPublicFile } from "./upload.js"; +import { payForStorageQuotes } from "./payment.js"; await initAntCore(); @@ -44,6 +41,7 @@ const elements = { }; let client; +let networkClient; let browserManifest; let downloadObjectUrl; @@ -62,12 +60,6 @@ function endpointFromForm() { return elements.endpointMultiaddr.value.trim(); } -function seedEndpoints() { - return browserManifest?.endpoints?.length - ? browserManifest.endpoints - : [endpointFromForm()]; -} - async function loadManifest() { elements.manifestState.classList.remove("connected"); elements.manifestState.textContent = "Loading…"; @@ -75,6 +67,8 @@ async function loadManifest() { elements.manifestUrl.value.trim(), ); browserManifest = manifest; + networkClient?.close(); + networkClient = new BrowserNetworkClient(manifest.endpoints); const first = manifest.endpoints[0]; elements.endpointMultiaddr.value = first.multiaddr; @@ -150,18 +144,9 @@ elements.findClosest.addEventListener("click", async () => { const target = elements.lookupTarget.value.trim(); hexToBytes(target, 32); log(`Starting iterative lookup for ${target}`); - const result = await iterativeFindClosest(seedEndpoints(), target, { - onProgress: (message) => log(message), - }); - log("Closest nodes", { - nodes: result.nodes, - queried: result.queried, - failures: result.failures.map(({ peerId, error }) => ({ - peerId, - message: error.message, - })), - }); - if (result.ownsClientPool) result.clientPool.close(); + if (!networkClient) throw new Error("Load the browser testnet manifest first"); + const result = await networkClient.findClosest(target, (message) => log(message)); + log("Closest nodes", result); } catch (error) { reportError("Lookup", error); } @@ -182,17 +167,20 @@ elements.uploadFile.addEventListener("click", async () => { if (!walletSecret) throw new Error("Enter the paying wallet secret key"); log(`Starting paid public upload for ${file.name}`); - const result = await uploadPublicFile( - seedEndpoints(), + if (!networkClient) throw new Error("Browser network client is not ready"); + const content = new Uint8Array(await file.arrayBuffer()); + const onProgress = (message) => { + elements.uploadState.textContent = message; + log(message); + }; + const result = await networkClient.uploadPublicFile( + content, + file.name, + file.type, browserManifest.payment, - file, - walletSecret, - { - onProgress: (message) => { - elements.uploadState.textContent = message; - log(message); - }, - }, + (paymentNetwork, quotes) => + payForStorageQuotes(paymentNetwork, quotes, walletSecret, { onProgress }), + onProgress, ); browserManifest.files = [ @@ -245,12 +233,11 @@ elements.downloadFile.addEventListener("click", async () => { log( `Downloading complete public file ${published.name} (${published.size.toLocaleString()} bytes)`, ); - const { content, hash, dataMapNode } = await downloadPublicFile( - seedEndpoints(), + if (!networkClient) throw new Error("Browser network client is not ready"); + const { content, hash, dataMapNode } = await networkClient.downloadPublicFile( published, - { - onProgress: (message) => log(message), - }, + 3, + (message) => log(message), ); const savedDirectly = await exposeSavedFile(published, content, saveHandle); elements.downloadState.textContent = `${ @@ -313,8 +300,28 @@ async function exposeSavedFile(file, content, saveHandle) { window.addEventListener("beforeunload", () => { if (downloadObjectUrl) URL.revokeObjectURL(downloadObjectUrl); + client?.close(); + networkClient?.close(); }); +function bytesToHex(bytes) { + return Array.from(bytes, (byte) => byte.toString(16).padStart(2, "0")).join(""); +} + +function hexToBytes(value, expectedLength) { + const normalized = value.trim().replace(/^0x/i, "").replaceAll(":", ""); + if (!/^[0-9a-f]*$/i.test(normalized) || normalized.length % 2 !== 0) { + throw new Error("Expected an even-length hexadecimal value"); + } + const bytes = Uint8Array.from( + normalized.match(/.{2}/g)?.map((pair) => Number.parseInt(pair, 16)) ?? [], + ); + if (expectedLength !== undefined && bytes.length !== expectedLength) { + throw new Error(`Expected ${expectedLength} bytes, received ${bytes.length}`); + } + return bytes; +} + elements.randomTarget.click(); log("Ready. Loading the local browser testnet manifest…"); loadManifest().catch((error) => { diff --git a/web/src/manifest.js b/web/src/manifest.js index 63ba8c04..c78c4bb1 100644 --- a/web/src/manifest.js +++ b/web/src/manifest.js @@ -1,128 +1,6 @@ -import { hexToBytes, parseWebRtcDirectMultiaddr } from "./protocol.js"; +import { parseBrowserManifest } from "../pkg/ant_core.js"; -export const BROWSER_MANIFEST_VERSION = 5; -const MAX_PUBLIC_FILE_BYTES = 64 * 1024 * 1024; -const MAX_DATA_MAP_BYTES = 4 * 1024 * 1024; -const MAX_FILE_CHUNKS = 1024; - -export function parseBrowserManifest(value) { - if (!value || value.version !== BROWSER_MANIFEST_VERSION) { - throw new Error(`Unsupported browser manifest version ${value?.version}`); - } - if (typeof value.network_id !== "string" || value.network_id.length === 0) { - throw new Error("Browser manifest has no network ID"); - } - if (!Array.isArray(value.endpoints) || value.endpoints.length === 0) { - throw new Error("Browser manifest contains no WebRtcDirect endpoints"); - } - const endpoints = value.endpoints.map((endpoint) => { - const parsed = parseWebRtcDirectMultiaddr(endpoint); - return { multiaddr: parsed.multiaddr }; - }); - const payment = parsePaymentNetwork(value.payment); - - const files = (value.files ?? []).map((file) => { - if (!file || typeof file.name !== "string" || file.name.length === 0) { - throw new Error("Browser manifest file has no name"); - } - hexToBytes(file.address ?? "", 32); - if ( - !Number.isSafeInteger(file.size) || - file.size < 3 || - file.size > MAX_PUBLIC_FILE_BYTES - ) { - throw new Error(`Invalid public file size ${file.size}`); - } - hexToBytes(file.blake3 ?? "", 32); - if ( - !Number.isSafeInteger(file.data_map_size) || - file.data_map_size < 1 || - file.data_map_size > MAX_DATA_MAP_BYTES - ) { - throw new Error(`Invalid DataMap size ${file.data_map_size}`); - } - if ( - !Array.isArray(file.chunks) || - file.chunks.length < 3 || - file.chunks.length > MAX_FILE_CHUNKS - ) { - throw new Error("Public file has an invalid self-encryption chunk list"); - } - const chunks = file.chunks - .map((chunk) => { - if (!Number.isSafeInteger(chunk.index) || chunk.index < 0) { - throw new Error(`Invalid file chunk index ${chunk.index}`); - } - hexToBytes(chunk.dst_hash ?? "", 32); - hexToBytes(chunk.src_hash ?? "", 32); - if (!Number.isSafeInteger(chunk.src_size) || chunk.src_size < 1) { - throw new Error(`Invalid plaintext chunk size ${chunk.src_size}`); - } - return { - index: chunk.index, - dst_hash: chunk.dst_hash.toLowerCase(), - src_hash: chunk.src_hash.toLowerCase(), - src_size: chunk.src_size, - }; - }) - .sort((left, right) => left.index - right.index); - chunks.forEach((chunk, index) => { - if (chunk.index !== index) { - throw new Error("File chunk indices must be contiguous from zero"); - } - }); - const reconstructedSize = chunks.reduce((total, chunk) => total + chunk.src_size, 0); - if (reconstructedSize !== file.size) { - throw new Error( - `File chunk sizes total ${reconstructedSize}, expected ${file.size}`, - ); - } - return { - name: file.name, - address: file.address.toLowerCase(), - size: file.size, - content_type: file.content_type || "application/octet-stream", - blake3: file.blake3.toLowerCase(), - data_map_size: file.data_map_size, - chunks, - replicas: Number.isSafeInteger(file.replicas) ? file.replicas : 0, - }; - }); - - return { - version: value.version, - network_id: value.network_id, - created_at: value.created_at, - endpoints, - payment, - files, - }; -} - -export function parsePaymentNetwork(value) { - if (!value || typeof value !== "object") { - throw new Error("Browser manifest contains no payment network"); - } - let rpcUrl; - try { - rpcUrl = new URL(value.rpc_url); - } catch (error) { - throw new Error("Payment RPC URL is invalid", { cause: error }); - } - if (!["http:", "https:"].includes(rpcUrl.protocol)) { - throw new Error("Payment RPC URL must use HTTP or HTTPS"); - } - if (rpcUrl.username || rpcUrl.password) { - throw new Error("Payment RPC URL must not contain credentials"); - } - hexToBytes(value.payment_token_address ?? "", 20); - hexToBytes(value.payment_vault_address ?? "", 20); - return { - rpc_url: rpcUrl.toString(), - payment_token_address: `0x${value.payment_token_address.replace(/^0x/i, "").toLowerCase()}`, - payment_vault_address: `0x${value.payment_vault_address.replace(/^0x/i, "").toLowerCase()}`, - }; -} +export { parseBrowserManifest }; export async function fetchBrowserManifest(url) { const response = await fetch(url, { cache: "no-store" }); diff --git a/web/src/manifest.test.js b/web/src/manifest.test.js index 62efca80..57cdf8a9 100644 --- a/web/src/manifest.test.js +++ b/web/src/manifest.test.js @@ -77,7 +77,7 @@ test("browser manifest requires public payment contract configuration", () => { const endpoint = { multiaddr: webrtc_directMultiaddr("aa".repeat(32), 0xbb) }; assert.throws( () => parseBrowserManifest({ version: 5, network_id: "test", endpoints: [endpoint] }), - /no payment network/, + /missing field.*payment|payment network/i, ); assert.throws( () => diff --git a/web/src/payment.js b/web/src/payment.js index 7f2cc62d..7cd8b864 100644 --- a/web/src/payment.js +++ b/web/src/payment.js @@ -1,29 +1,10 @@ -import { ml_dsa65 } from "@noble/post-quantum/ml-dsa.js"; -import { decode } from "@msgpack/msgpack"; import { Contract, JsonRpcProvider, - keccak256, MaxUint256, NonceManager, Wallet, } from "ethers"; -import { contentAddress as contentAddressNative } from "../pkg/ant_core.js"; -import { bytesToHex, hexToBytes } from "./protocol.js"; - -const U256_MAX = (1n << 256n) - 1n; -const PAYMENT_MULTIPLIER = 3n; -const PRICE_BASELINE_WEI = 3_906_250_000_000_000n; -const PRICE_COEFFICIENT_WEI = 35_156_250_000_000_000n; -const DIVISOR_SQUARED = 6_000n * 6_000n; -const MAX_COMMITMENT_KEY_COUNT = 1_000_000; -const MAX_COMMITMENT_SIDECAR_BYTES = 8 * 1024; -const DOMAIN_COMMITMENT = new TextEncoder().encode( - "autonomi.ant.replication.storage_commitment.v1", -); -const DOMAIN_COMMITMENT_HASH = new TextEncoder().encode( - "autonomi.ant.replication.commitment_hash.v1", -); const TOKEN_ABI = [ "function allowance(address owner, address spender) view returns (uint256)", @@ -33,239 +14,6 @@ const VAULT_ABI = [ "function payForQuotes((address rewardsAddress,uint256 amount,bytes32 quoteHash)[] payments)", ]; -function concatBytes(...parts) { - const size = parts.reduce((total, part) => total + part.length, 0); - const output = new Uint8Array(size); - let offset = 0; - for (const part of parts) { - output.set(part, offset); - offset += part.length; - } - return output; -} - -function unsignedLittleEndian(value, length) { - let remaining = BigInt(value); - if (remaining < 0n || remaining >= 1n << BigInt(length * 8)) { - throw new Error(`Unsigned integer does not fit ${length} bytes`); - } - const result = new Uint8Array(length); - for (let index = 0; index < length; index += 1) { - result[index] = Number(remaining & 0xffn); - remaining >>= 8n; - } - return result; -} - -function postcardVarint(value) { - let remaining = BigInt(value); - const result = []; - do { - let byte = Number(remaining & 0x7fn); - remaining >>= 7n; - if (remaining > 0n) byte |= 0x80; - result.push(byte); - } while (remaining > 0n); - return Uint8Array.from(result); -} - -function canonicalQuoteBytes(quote) { - const content = hexToBytes(quote.content, 32); - const rewardsAddress = hexToBytes(quote.rewards_address, 20); - const price = parseAmount(quote.price, "quote price"); - const timestamp = parseAmount(String(quote.timestamp_secs), "quote timestamp"); - const count = quote.committed_key_count; - if (!Number.isSafeInteger(count) || count < 0 || count > MAX_COMMITMENT_KEY_COUNT) { - throw new Error(`Invalid committed key count ${count}`); - } - let pin = Uint8Array.of(0); - if (quote.commitment_pin !== null && quote.commitment_pin !== undefined) { - pin = concatBytes(Uint8Array.of(1), hexToBytes(quote.commitment_pin, 32)); - } - return concatBytes( - content, - unsignedLittleEndian(timestamp, 8), - unsignedLittleEndian(price, 32), - rewardsAddress, - unsignedLittleEndian(BigInt(count), 4), - pin, - ); -} - -export function paymentQuoteHash(signedBytes, publicKey, signature) { - // This is the EVM-facing PaymentQuote hash settled by the payment vault. - // Native evmlib deliberately uses Keccak-256 here, while ANT identities, - // chunk addresses, and commitment pins use BLAKE3. - return keccak256(concatBytes(signedBytes, publicKey, signature)).slice(2); -} - -function parseAmount(value, label) { - if (typeof value !== "string" || !/^(0|[1-9][0-9]*)$/.test(value)) { - throw new Error(`Invalid ${label}`); - } - const amount = BigInt(value); - if (amount > U256_MAX) throw new Error(`${label} exceeds uint256`); - return amount; -} - -function calculatePrice(keyCount) { - const count = BigInt(keyCount); - return PRICE_BASELINE_WEI + (count * count * PRICE_COEFFICIENT_WEI) / DIVISOR_SQUARED; -} - -function equalBytes(left, right) { - if (left.length !== right.length) return false; - return left.every((byte, index) => byte === right[index]); -} - -function verifyEncodedCommitment(commitment, normalized) { - const encoded = hexToBytes(commitment.encoded); - if (encoded.length > MAX_COMMITMENT_SIDECAR_BYTES) { - throw new Error("Storage commitment sidecar exceeds the protocol limit"); - } - let fields; - try { - fields = decode(encoded); - } catch (error) { - throw new Error(`Storage commitment sidecar is not valid MessagePack: ${error.message}`, { - cause: error, - }); - } - if (!Array.isArray(fields) || fields.length !== 5) { - throw new Error("Storage commitment sidecar has an invalid native shape"); - } - const [root, keyCount, peerId, publicKey, signature] = fields; - if ( - keyCount !== normalized.keyCount || - !equalBytes(fixedCommitmentBytes(root, 32), normalized.root) || - !equalBytes(fixedCommitmentBytes(peerId, 32), normalized.peerId) || - !equalBytes(fixedCommitmentBytes(publicKey), normalized.publicKey) || - !equalBytes(fixedCommitmentBytes(signature), normalized.signature) - ) { - throw new Error("Storage commitment sidecar differs from the verified commitment"); - } -} - -function fixedCommitmentBytes(value, length) { - const bytes = value instanceof Uint8Array ? value : Uint8Array.from(value ?? []); - if (length !== undefined && bytes.length !== length) { - throw new Error(`Storage commitment field must contain ${length} bytes`); - } - return bytes; -} - -function verifyCommitment(commitment, quote) { - if (!commitment || typeof commitment !== "object") { - throw new Error("Bound quote omitted its storage commitment"); - } - const root = hexToBytes(commitment.root, 32); - const peerId = hexToBytes(commitment.sender_peer_id, 32); - const publicKey = hexToBytes(commitment.sender_public_key); - const signature = hexToBytes(commitment.signature); - const keyCount = commitment.key_count; - if ( - publicKey.length !== ml_dsa65.lengths.publicKey || - signature.length !== ml_dsa65.lengths.signature - ) { - throw new Error("Storage commitment has invalid ML-DSA-65 field lengths"); - } - if (!Number.isSafeInteger(keyCount) || keyCount !== quote.committed_key_count) { - throw new Error("Storage commitment key count does not match quote"); - } - if (contentAddressNative(publicKey) !== quote.peer_id.toLowerCase()) { - throw new Error("Storage commitment public key is not bound to quote peer"); - } - if (bytesToHex(peerId) !== quote.peer_id.toLowerCase()) { - throw new Error("Storage commitment belongs to a different peer"); - } - verifyEncodedCommitment(commitment, { - root, - keyCount, - peerId, - publicKey, - signature, - }); - const signedPayload = concatBytes( - root, - unsignedLittleEndian(BigInt(keyCount), 4), - peerId, - unsignedLittleEndian(BigInt(publicKey.length), 4), - publicKey, - ); - if ( - !ml_dsa65.verify(signature, signedPayload, publicKey, { - context: DOMAIN_COMMITMENT, - }) - ) { - throw new Error("Storage commitment has an invalid ML-DSA-65 signature"); - } - - const postcard = concatBytes( - root, - postcardVarint(keyCount), - peerId, - postcardVarint(publicKey.length), - publicKey, - postcardVarint(signature.length), - signature, - ); - const pin = contentAddressNative(concatBytes(DOMAIN_COMMITMENT_HASH, postcard)); - if (pin !== quote.commitment_pin.toLowerCase()) { - throw new Error("Storage commitment does not resolve the quote pin"); - } - return true; -} - -export function verifyStorageQuote(quote, expectedAddress, expectedPeerId) { - if (!quote || typeof quote !== "object") throw new Error("Node returned no quote"); - hexToBytes(expectedAddress, 32); - hexToBytes(expectedPeerId, 32); - if (quote.content?.toLowerCase() !== expectedAddress.toLowerCase()) { - throw new Error("Storage quote is for a different chunk"); - } - if (quote.peer_id?.toLowerCase() !== expectedPeerId.toLowerCase()) { - throw new Error("Storage quote belongs to a different WebRtcDirect peer"); - } - const publicKey = hexToBytes(quote.public_key); - const signature = hexToBytes(quote.signature); - if (publicKey.length !== ml_dsa65.lengths.publicKey) { - throw new Error(`Storage quote has a ${publicKey.length}-byte public key`); - } - if (signature.length !== ml_dsa65.lengths.signature) { - throw new Error(`Storage quote has a ${signature.length}-byte signature`); - } - if (contentAddressNative(publicKey) !== quote.peer_id.toLowerCase()) { - throw new Error("Storage quote public key is not bound to its peer ID"); - } - const signedBytes = canonicalQuoteBytes(quote); - if (!ml_dsa65.verify(signature, signedBytes, publicKey)) { - throw new Error("Storage quote has an invalid ML-DSA-65 signature"); - } - const quoteHash = paymentQuoteHash(signedBytes, publicKey, signature); - if (quoteHash !== quote.quote_hash?.toLowerCase()) { - throw new Error("Storage quote hash does not match its signed fields"); - } - const price = parseAmount(quote.price, "quote price"); - if (price !== calculatePrice(quote.committed_key_count)) { - throw new Error("Storage quote price is not bound to its committed key count"); - } - if (quote.committed_key_count === 0) { - if (quote.commitment_pin !== null || quote.commitment !== null) { - throw new Error("Baseline storage quote has an incoherent commitment"); - } - } else { - if (!quote.commitment_pin) throw new Error("Bound storage quote omitted its pin"); - verifyCommitment(quote.commitment, quote); - } - hexToBytes(quote.rewards_address, 20); - return { - quote, - quoteHash, - rewardsAddress: `0x${quote.rewards_address.replace(/^0x/i, "")}`, - amount: price * PAYMENT_MULTIPLIER, - }; -} - export async function payForStorageQuotes( paymentNetwork, verifiedQuotes, @@ -283,7 +31,10 @@ export async function payForStorageQuotes( throw new Error("Wallet secret key is invalid", { cause: error }); } const signer = new NonceManager(wallet); - const totalAmount = verifiedQuotes.reduce((total, quote) => total + quote.amount, 0n); + const totalAmount = verifiedQuotes.reduce( + (total, quote) => total + BigInt(quote.amount), + 0n, + ); const token = new Contract(paymentNetwork.payment_token_address, TOKEN_ABI, signer); const vault = new Contract(paymentNetwork.payment_vault_address, VAULT_ABI, signer); const allowance = await token.allowance(wallet.address, paymentNetwork.payment_vault_address); @@ -302,5 +53,9 @@ export async function payForStorageQuotes( const receipt = await transaction.wait(); if (!receipt || receipt.status !== 1) throw new Error("Storage payment transaction reverted"); onProgress(`Payment confirmed in ${transaction.hash}`); - return { transactionHash: transaction.hash, walletAddress: wallet.address, totalAmount }; + return { + transactionHash: transaction.hash, + walletAddress: wallet.address, + totalAmount: totalAmount.toString(), + }; } diff --git a/web/src/payment.test.js b/web/src/payment.test.js deleted file mode 100644 index 84a4f680..00000000 --- a/web/src/payment.test.js +++ /dev/null @@ -1,235 +0,0 @@ -import assert from "node:assert/strict"; -import test from "node:test"; -import { blake3 } from "@noble/hashes/blake3.js"; -import { ml_dsa65 } from "@noble/post-quantum/ml-dsa.js"; -import { encode } from "@msgpack/msgpack"; -import { keccak256 } from "ethers"; -import { bytesToHex } from "./protocol.js"; -import { paymentQuoteHash, verifyStorageQuote } from "./payment.js"; - -const BASELINE_PRICE = 3_906_250_000_000_000n; -const PRICE_COEFFICIENT = 35_156_250_000_000_000n; -const COMMITMENT_CONTEXT = new TextEncoder().encode( - "autonomi.ant.replication.storage_commitment.v1", -); -const COMMITMENT_HASH_DOMAIN = new TextEncoder().encode( - "autonomi.ant.replication.commitment_hash.v1", -); - -function concatBytes(...parts) { - const output = new Uint8Array(parts.reduce((total, part) => total + part.length, 0)); - let offset = 0; - for (const part of parts) { - output.set(part, offset); - offset += part.length; - } - return output; -} - -function littleEndian(value, length) { - let remaining = BigInt(value); - const result = new Uint8Array(length); - for (let index = 0; index < length; index += 1) { - result[index] = Number(remaining & 0xffn); - remaining >>= 8n; - } - return result; -} - -function postcardVarint(value) { - let remaining = BigInt(value); - const bytes = []; - do { - let byte = Number(remaining & 0x7fn); - remaining >>= 7n; - if (remaining > 0n) byte |= 0x80; - bytes.push(byte); - } while (remaining > 0n); - return Uint8Array.from(bytes); -} - -function signedBaselineQuote() { - const content = Uint8Array.from({ length: 32 }, (_, index) => index); - const rewards = new Uint8Array(20).fill(0x44); - const timestamp = 1_775_000_000n; - const { publicKey, secretKey } = ml_dsa65.keygen(new Uint8Array(32).fill(0x17)); - const payload = concatBytes( - content, - littleEndian(timestamp, 8), - littleEndian(BASELINE_PRICE, 32), - rewards, - littleEndian(0n, 4), - Uint8Array.of(0), - ); - const signature = ml_dsa65.sign(payload, secretKey); - const quoteHash = keccak256(concatBytes(payload, publicKey, signature)).slice(2); - const peerId = bytesToHex(blake3(publicKey)); - return { - peerId, - address: bytesToHex(content), - quote: { - peer_id: peerId, - content: bytesToHex(content), - timestamp_secs: Number(timestamp), - price: BASELINE_PRICE.toString(), - rewards_address: bytesToHex(rewards), - public_key: bytesToHex(publicKey), - signature: bytesToHex(signature), - committed_key_count: 0, - commitment_pin: null, - quote_hash: quoteHash, - commitment: null, - }, - }; -} - -function signedBoundQuote() { - const content = new Uint8Array(32).fill(0x31); - const rewards = new Uint8Array(20).fill(0x42); - const root = new Uint8Array(32).fill(0x53); - const keyCount = 23; - const timestamp = 1_775_000_001n; - const { publicKey, secretKey } = ml_dsa65.keygen(new Uint8Array(32).fill(0x29)); - const peerId = blake3(publicKey); - const commitmentPayload = concatBytes( - root, - littleEndian(keyCount, 4), - peerId, - littleEndian(publicKey.length, 4), - publicKey, - ); - const commitmentSignature = ml_dsa65.sign(commitmentPayload, secretKey, { - context: COMMITMENT_CONTEXT, - }); - const postcard = concatBytes( - root, - postcardVarint(keyCount), - peerId, - postcardVarint(publicKey.length), - publicKey, - postcardVarint(commitmentSignature.length), - commitmentSignature, - ); - const pin = blake3(concatBytes(COMMITMENT_HASH_DOMAIN, postcard)); - const price = - BASELINE_PRICE + - (BigInt(keyCount) * BigInt(keyCount) * PRICE_COEFFICIENT) / (6_000n * 6_000n); - const quotePayload = concatBytes( - content, - littleEndian(timestamp, 8), - littleEndian(price, 32), - rewards, - littleEndian(keyCount, 4), - Uint8Array.of(1), - pin, - ); - const quoteSignature = ml_dsa65.sign(quotePayload, secretKey); - const quoteHash = keccak256( - concatBytes(quotePayload, publicKey, quoteSignature), - ).slice(2); - const encodedCommitment = encode([ - Array.from(root), - keyCount, - Array.from(peerId), - Array.from(publicKey), - Array.from(commitmentSignature), - ]); - const commitment = { - encoded: bytesToHex(encodedCommitment), - root: bytesToHex(root), - key_count: keyCount, - sender_peer_id: bytesToHex(peerId), - sender_public_key: bytesToHex(publicKey), - signature: bytesToHex(commitmentSignature), - }; - return { - address: bytesToHex(content), - peerId: bytesToHex(peerId), - quote: { - peer_id: bytesToHex(peerId), - content: bytesToHex(content), - timestamp_secs: Number(timestamp), - price: price.toString(), - rewards_address: bytesToHex(rewards), - public_key: bytesToHex(publicKey), - signature: bytesToHex(quoteSignature), - committed_key_count: keyCount, - commitment_pin: bytesToHex(pin), - quote_hash: quoteHash, - commitment, - }, - }; -} - -test("accepts a correctly bound and signed native-shaped storage quote", () => { - const { quote, address, peerId } = signedBaselineQuote(); - const verified = verifyStorageQuote(quote, address, peerId); - assert.equal(verified.quoteHash, quote.quote_hash); - assert.equal(verified.amount, BASELINE_PRICE * 3n); - assert.equal(verified.rewardsAddress, `0x${quote.rewards_address}`); -}); - -test("uses evmlib's Keccak-256 PaymentQuote hash", () => { - // Fixed native hash-vector input split as bytes-for-signing, public key, - // and signature. PaymentQuote::hash() concatenates these exact byte slices. - assert.equal( - paymentQuoteHash(Uint8Array.of(0, 1), Uint8Array.of(2), Uint8Array.of(3)), - "d98f2e8134922f73748703c8e7084d42f13d2fa1439936ef5a3abcf5646fe83f", - ); -}); - -test("rejects quote field tampering before any payment", () => { - const fixture = signedBaselineQuote(); - assert.throws( - () => - verifyStorageQuote( - { ...fixture.quote, price: (BASELINE_PRICE + 1n).toString() }, - fixture.address, - fixture.peerId, - ), - /invalid ML-DSA-65 signature|price/, - ); - assert.throws( - () => verifyStorageQuote(fixture.quote, fixture.address, "ff".repeat(32)), - /different WebRtcDirect peer/, - ); - const signature = Uint8Array.from(Buffer.from(fixture.quote.signature, "hex")); - signature[100] ^= 1; - assert.throws( - () => - verifyStorageQuote( - { ...fixture.quote, signature: bytesToHex(signature) }, - fixture.address, - fixture.peerId, - ), - /invalid ML-DSA-65 signature/, - ); -}); - -test("verifies a bound commitment and the exact native sidecar before payment", () => { - const fixture = signedBoundQuote(); - assert.doesNotThrow(() => - verifyStorageQuote(fixture.quote, fixture.address, fixture.peerId), - ); - - const decodedShape = [ - Array(32).fill(0x99), - fixture.quote.committed_key_count, - Array.from(Buffer.from(fixture.quote.commitment.sender_peer_id, "hex")), - Array.from(Buffer.from(fixture.quote.commitment.sender_public_key, "hex")), - Array.from(Buffer.from(fixture.quote.commitment.signature, "hex")), - ]; - const mismatchedSidecar = { - ...fixture.quote.commitment, - encoded: bytesToHex(encode(decodedShape)), - }; - assert.throws( - () => - verifyStorageQuote( - { ...fixture.quote, commitment: mismatchedSidecar }, - fixture.address, - fixture.peerId, - ), - /sidecar differs/, - ); -}); diff --git a/web/src/protocol.js b/web/src/protocol.js deleted file mode 100644 index bfab9f99..00000000 --- a/web/src/protocol.js +++ /dev/null @@ -1,1025 +0,0 @@ -import { bytesToHex as nobleBytesToHex } from "@noble/hashes/utils.js"; -import { blake3 } from "@noble/hashes/blake3.js"; -import { ml_dsa65 } from "@noble/post-quantum/ml-dsa.js"; -import { - BrowserIterativeLookup as BrowserIterativeLookupNative, - verifyRecord as verifyRecordNative, -} from "../pkg/ant_core.js"; - -export const PROTOCOL_VERSION = 3; -export const PROTOCOL_NAME = "autonomi.web.poc.v3"; -export const WEBRTC_DIRECT_DATA_CHANNEL = "autonomi.web.v3"; -export const MAX_CHUNK_SIZE = 4 * 1024 * 1024; -export const MAX_RESPONSE_HEADER_BYTES = 64 * 1024; -const MAX_RESPONSE_BYTES = 4 + MAX_RESPONSE_HEADER_BYTES + MAX_CHUNK_SIZE; -const MAX_WEBRTC_DIRECT_MULTIADDR_LENGTH = 2048; -const WEBRTC_WRITE_CHUNK_BYTES = 16 * 1024; -const MAX_BUFFERED_AMOUNT = 2 * 1024 * 1024; -const REQUEST_TIMEOUT_MS = 10_000; -const ICE_CREDENTIAL_PREFIX = "saorsa+webrtc+v1/"; -const ICE_RANDOM_LENGTH = 32; -const ICE_ALPHABET = - "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789"; -const SHA2_256_MULTIHASH_CODE = 0x12; -const SHA2_256_MULTIHASH_LENGTH = 32; -const encoder = new TextEncoder(); -const decoder = new TextDecoder("utf-8", { fatal: true }); -const HELLO_DOMAIN = encoder.encode("autonomi-webrtc-direct-hello-v1\0"); - -let nextRequestId = 1; - -export function hexToBytes(value, expectedLength) { - const normalized = value.trim().replace(/^0x/i, "").replaceAll(":", ""); - if (!/^[0-9a-f]*$/i.test(normalized) || normalized.length % 2 !== 0) { - throw new Error("Expected an even-length hexadecimal value"); - } - const bytes = Uint8Array.from( - normalized.match(/.{2}/g)?.map((pair) => Number.parseInt(pair, 16)) ?? [], - ); - if (expectedLength !== undefined && bytes.length !== expectedLength) { - throw new Error( - `Expected ${expectedLength} bytes, received ${bytes.length}`, - ); - } - return bytes; -} - -export function bytesToHex(bytes) { - return nobleBytesToHex(bytes); -} - -function concatBytes(...parts) { - const result = new Uint8Array( - parts.reduce((total, part) => total + part.length, 0), - ); - let offset = 0; - for (const part of parts) { - result.set(part, offset); - offset += part.length; - } - return result; -} - -export function verifyChunk(address, content) { - const expected = address.trim().replace(/^0x/i, "").toLowerCase(); - hexToBytes(expected, 32); - return verifyRecordNative(expected, content); -} - -export function parseResponseFrame(frame) { - const { header, contentOffset, frameLength } = parseResponseHeader(frame); - if (frame.length !== frameLength) { - throw new Error( - `Response length mismatch: declared ${header.content_length} content bytes`, - ); - } - return { header, content: frame.slice(contentOffset) }; -} - -function parseResponseHeader(frame) { - if (!(frame instanceof Uint8Array) || frame.length < 4) { - throw new Error("Response ended before its four-byte header length"); - } - const view = new DataView(frame.buffer, frame.byteOffset, frame.byteLength); - const headerLength = view.getUint32(0, false); - if (headerLength === 0 || headerLength > MAX_RESPONSE_HEADER_BYTES) { - throw new Error(`Invalid response header length ${headerLength}`); - } - const contentOffset = 4 + headerLength; - if (contentOffset > frame.length) { - throw new Error("Response ended inside its JSON header"); - } - - let header; - try { - header = JSON.parse(decoder.decode(frame.subarray(4, contentOffset))); - } catch (error) { - throw new Error(`Invalid response JSON: ${error.message}`, { - cause: error, - }); - } - if (header.version !== PROTOCOL_VERSION) { - throw new Error(`Unsupported response version ${header.version}`); - } - if ( - !Number.isSafeInteger(header.content_length) || - header.content_length < 0 || - header.content_length > MAX_CHUNK_SIZE - ) { - throw new Error(`Invalid response content length ${header.content_length}`); - } - return { - header, - contentOffset, - frameLength: contentOffset + header.content_length, - }; -} - -export async function readResponseFrame(stream, limit = MAX_RESPONSE_BYTES) { - if (!Number.isSafeInteger(limit) || limit < 4) { - throw new Error(`Invalid response limit ${limit}`); - } - let frame = new Uint8Array(Math.min(limit, 8 * 1024)); - let total = 0; - let expectedLength; - for await (const value of stream) { - const chunk = value instanceof Uint8Array ? value : value.subarray(); - const nextTotal = total + chunk.length; - if (nextTotal > limit) { - throw new Error(`Response exceeded the ${limit}-byte client limit`); - } - if (nextTotal > frame.length) { - let capacity = frame.length; - while (capacity < nextTotal) capacity = Math.min(limit, capacity * 2); - const grown = new Uint8Array(capacity); - grown.set(frame.subarray(0, total)); - frame = grown; - } - frame.set(chunk, total); - total = nextTotal; - - if (expectedLength === undefined && total >= 4) { - const headerLength = new DataView( - frame.buffer, - frame.byteOffset, - total, - ).getUint32(0, false); - if (headerLength === 0 || headerLength > MAX_RESPONSE_HEADER_BYTES) { - throw new Error(`Invalid response header length ${headerLength}`); - } - if (total >= 4 + headerLength) { - expectedLength = parseResponseHeader( - frame.subarray(0, total), - ).frameLength; - if (expectedLength > limit) { - throw new Error(`Response exceeded the ${limit}-byte client limit`); - } - } - } - - if (expectedLength !== undefined && total >= expectedLength) { - if (total !== expectedLength) { - throw new Error("Response contains bytes after its declared frame"); - } - return frame.slice(0, total); - } - } - throw new Error("Response ended before its declared frame was complete"); -} - -function decodeBase64Url(value) { - if (!/^[A-Za-z0-9_-]+$/.test(value)) { - throw new Error("Certificate multihash is not valid unpadded base64url"); - } - const padding = "=".repeat((4 - (value.length % 4)) % 4); - let binary; - try { - binary = atob(value.replaceAll("-", "+").replaceAll("_", "/") + padding); - } catch (error) { - throw new Error("Certificate multihash is not valid unpadded base64url", { - cause: error, - }); - } - return Uint8Array.from(binary, (character) => character.charCodeAt(0)); -} - -function decodeCertificateMultihash(value) { - if (typeof value !== "string" || !value.startsWith("u")) { - throw new Error("Certificate multihash must use base64url multibase (`u`)"); - } - const decoded = decodeBase64Url(value.slice(1)); - if ( - decoded.length !== 34 || - decoded[0] !== SHA2_256_MULTIHASH_CODE || - decoded[1] !== SHA2_256_MULTIHASH_LENGTH - ) { - throw new Error( - "Certificate multihash must contain a 32-byte SHA-256 digest", - ); - } - return decoded.slice(2); -} - -function validateIpv4(value) { - const octets = value.split("."); - if ( - octets.length !== 4 || - octets.some( - (octet) => - !/^(0|[1-9][0-9]{0,2})$/.test(octet) || - Number.parseInt(octet, 10) > 255, - ) - ) { - throw new Error(`Invalid IPv4 address ${value}`); - } - return value; -} - -function endpointMultiaddr(endpoint) { - if (typeof endpoint === "string") return endpoint; - if (endpoint && typeof endpoint.multiaddr === "string") - return endpoint.multiaddr; - throw new Error("A WebRtcDirect multiaddress is required"); -} - -export function parseWebRtcDirectMultiaddr(endpoint) { - const multiaddr = endpointMultiaddr(endpoint).trim(); - if ( - multiaddr.length === 0 || - multiaddr.length > MAX_WEBRTC_DIRECT_MULTIADDR_LENGTH || - !multiaddr.startsWith("/") - ) { - throw new Error("Invalid WebRtcDirect multiaddress length or prefix"); - } - const segments = multiaddr.split("/"); - if (segments.length !== 10) { - throw new Error("WebRtcDirect multiaddress is incomplete"); - } - - const hostProtocol = segments[1]; - const hostValue = segments[2]; - if (!hostValue) throw new Error("WebRtcDirect multiaddress host is empty"); - if (hostProtocol === "ip4") { - validateIpv4(hostValue); - } else if (hostProtocol === "ip6") { - if (!hostValue.includes(":")) - throw new Error(`Invalid IPv6 address ${hostValue}`); - } else { - throw new Error( - "WebRTC Direct multiaddresses must use a literal IP address", - ); - } - if (segments[3] !== "udp") { - throw new Error("WebRtcDirect multiaddress must use UDP"); - } - if (!/^[0-9]{1,5}$/.test(segments[4])) { - throw new Error("WebRtcDirect multiaddress has an invalid UDP port"); - } - const port = Number.parseInt(segments[4], 10); - if (port < 1 || port > 65535) { - throw new Error("WebRtcDirect multiaddress has an invalid UDP port"); - } - if (segments[5] !== "webrtc-direct") { - throw new Error("WebRTC Direct multiaddress must contain /webrtc-direct"); - } - if (segments[6] !== "certhash" || !segments[7]) { - throw new Error( - "WebRTC Direct multiaddress must contain exactly one certhash", - ); - } - const certificateHash = decodeCertificateMultihash(segments[7]); - if (segments[8] !== "p2p") { - throw new Error("WebRtcDirect multiaddress must end with /p2p/"); - } - const peerId = segments[9]?.toLowerCase() ?? ""; - hexToBytes(peerId, 32); - return { - multiaddr, - hostProtocol, - host: hostValue, - port, - peerId, - certificateHash, - }; -} - -const normalizeEndpoint = parseWebRtcDirectMultiaddr; - -export function verifyHelloIdentity(header, expectedEndpoint, challengeBytes) { - const endpoint = normalizeEndpoint(expectedEndpoint); - if (!(challengeBytes instanceof Uint8Array) || challengeBytes.length !== 32) { - throw new Error("HELLO verification requires a 32-byte challenge"); - } - if (header.type !== "hello") throw new Error("Expected a HELLO response"); - if (header.protocol !== PROTOCOL_NAME) { - throw new Error(`Unsupported browser protocol ${header.protocol}`); - } - hexToBytes(header.peer_id, 32); - if (header.challenge?.toLowerCase() !== bytesToHex(challengeBytes)) { - throw new Error("Node signed a different HELLO challenge"); - } - const advertisedEndpoint = normalizeEndpoint(header.endpoint); - if ( - advertisedEndpoint.multiaddr !== endpoint.multiaddr || - advertisedEndpoint.peerId !== header.peer_id.toLowerCase() - ) { - throw new Error("Node advertised a different WebRTC Direct endpoint"); - } - if (header.peer_id.toLowerCase() !== endpoint.peerId.toLowerCase()) { - throw new Error( - `Endpoint identity mismatch: expected ${endpoint.peerId}, received ${header.peer_id}`, - ); - } - const publicKey = hexToBytes(header.public_key); - const signature = hexToBytes(header.signature); - if (publicKey.length !== ml_dsa65.lengths.publicKey) { - throw new Error(`HELLO has a ${publicKey.length}-byte public key`); - } - if (signature.length !== ml_dsa65.lengths.signature) { - throw new Error(`HELLO has a ${signature.length}-byte signature`); - } - if (bytesToHex(blake3(publicKey)) !== header.peer_id.toLowerCase()) { - throw new Error("HELLO public key is not bound to the ANT peer ID"); - } - const transcript = concatBytes( - HELLO_DOMAIN, - challengeBytes, - encoder.encode(header.peer_id.toLowerCase()), - encoder.encode(advertisedEndpoint.multiaddr), - ); - if (!ml_dsa65.verify(signature, transcript, publicKey)) { - throw new Error("HELLO has an invalid ML-DSA-65 signature"); - } - return header.peer_id.toLowerCase(); -} - -function randomIceCredential() { - const random = crypto.getRandomValues(new Uint8Array(ICE_RANDOM_LENGTH)); - let suffix = ""; - for (const byte of random) suffix += ICE_ALPHABET[byte % ICE_ALPHABET.length]; - return ICE_CREDENTIAL_PREFIX + suffix; -} - -function certificateFingerprint(certificateHash) { - return Array.from(certificateHash, (byte) => - byte.toString(16).padStart(2, "0").toUpperCase(), - ).join(":"); -} - -export function serverAnswerFromEndpoint(endpoint, iceCredential) { - const normalized = normalizeEndpoint(endpoint); - if ( - typeof iceCredential !== "string" || - !iceCredential.startsWith(ICE_CREDENTIAL_PREFIX) || - !/^[a-zA-Z0-9+/]{22,256}$/.test(iceCredential) - ) { - throw new Error("Invalid Saorsa WebRTC Direct ICE credential"); - } - const ipVersion = normalized.hostProtocol === "ip4" ? "IP4" : "IP6"; - return { - type: "answer", - sdp: `v=0\r -o=- 0 0 IN ${ipVersion} ${normalized.host}\r -s=-\r -t=0 0\r -a=ice-lite\r -m=application ${normalized.port} UDP/DTLS/SCTP webrtc-datachannel\r -c=IN ${ipVersion} ${normalized.host}\r -a=mid:0\r -a=ice-options:ice2\r -a=ice-ufrag:${iceCredential}\r -a=ice-pwd:${iceCredential}\r -a=fingerprint:sha-256 ${certificateFingerprint(normalized.certificateHash)}\r -a=setup:passive\r -a=sctp-port:5000\r -a=max-message-size:${WEBRTC_WRITE_CHUNK_BYTES}\r -a=candidate:1467250027 1 UDP 1467250027 ${normalized.host} ${normalized.port} typ host\r -a=end-of-candidates\r -`, - }; -} - -export function mungeOfferIceCredentials(offer, iceCredential) { - if (!offer?.sdp) throw new Error("Browser created an empty WebRTC offer"); - const sdp = offer.sdp - .replace(/a=ice-ufrag:[^\r\n]+/, `a=ice-ufrag:${iceCredential}`) - .replace(/a=ice-pwd:[^\r\n]+/, `a=ice-pwd:${iceCredential}`); - if ( - !sdp.includes(`a=ice-ufrag:${iceCredential}`) || - !sdp.includes(`a=ice-pwd:${iceCredential}`) - ) { - throw new Error("Browser offer did not contain ICE credentials"); - } - return { type: "offer", sdp }; -} - -class DataChannelInbox { - constructor(channel) { - this.queue = []; - this.waiters = []; - this.closed = false; - this.error = undefined; - channel.binaryType = "arraybuffer"; - channel.addEventListener("message", ({ data }) => { - try { - let message; - if (data instanceof ArrayBuffer) { - message = new Uint8Array(data); - } else if (ArrayBuffer.isView(data)) { - message = new Uint8Array( - data.buffer, - data.byteOffset, - data.byteLength, - ); - } else { - throw new Error("Node sent a non-binary DataChannel message"); - } - this.push(message); - } catch (error) { - this.fail(error); - } - }); - channel.addEventListener("error", (event) => { - this.fail(event.error ?? new Error("WebRTC DataChannel failed")); - }); - channel.addEventListener("close", () => this.finish()); - } - - push(message) { - const waiter = this.waiters.shift(); - if (waiter) waiter.resolve({ value: message, done: false }); - else this.queue.push(message); - } - - finish() { - this.closed = true; - for (const waiter of this.waiters.splice(0)) waiter.resolve({ done: true }); - } - - fail(error) { - this.error = error; - for (const waiter of this.waiters.splice(0)) waiter.reject(error); - } - - next() { - if (this.queue.length > 0) { - return Promise.resolve({ value: this.queue.shift(), done: false }); - } - if (this.error) return Promise.reject(this.error); - if (this.closed) return Promise.resolve({ done: true }); - return new Promise((resolve, reject) => - this.waiters.push({ resolve, reject }), - ); - } - - [Symbol.asyncIterator]() { - return this; - } -} - -function waitForDataChannelOpen(channel, timeoutMs = REQUEST_TIMEOUT_MS) { - if (channel.readyState === "open") return Promise.resolve(); - return new Promise((resolve, reject) => { - const timeout = setTimeout(() => { - cleanup(); - reject(new Error("WebRTC DataChannel opening timed out")); - }, timeoutMs); - const cleanup = () => { - clearTimeout(timeout); - channel.removeEventListener("open", opened); - channel.removeEventListener("close", closed); - channel.removeEventListener("error", failed); - }; - const opened = () => { - cleanup(); - resolve(); - }; - const closed = () => { - cleanup(); - reject(new Error("WebRTC DataChannel closed before opening")); - }; - const failed = (event) => { - cleanup(); - reject( - event.error ?? new Error("WebRTC DataChannel failed while opening"), - ); - }; - channel.addEventListener("open", opened, { once: true }); - channel.addEventListener("close", closed, { once: true }); - channel.addEventListener("error", failed, { once: true }); - }); -} - -async function waitForDataChannelCapacity(channel) { - if (channel.bufferedAmount <= MAX_BUFFERED_AMOUNT) return; - channel.bufferedAmountLowThreshold = MAX_BUFFERED_AMOUNT / 2; - await new Promise((resolve, reject) => { - const drained = () => { - cleanup(); - resolve(); - }; - const closed = () => { - cleanup(); - reject(new Error("WebRTC DataChannel closed while draining")); - }; - const cleanup = () => { - channel.removeEventListener("bufferedamountlow", drained); - channel.removeEventListener("close", closed); - }; - channel.addEventListener("bufferedamountlow", drained, { once: true }); - channel.addEventListener("close", closed, { once: true }); - }); -} - -export class BrowserNodeClient { - constructor(endpoint) { - this.endpoint = normalizeEndpoint(endpoint); - this.peerConnection = undefined; - this.dataChannel = undefined; - this.inbox = undefined; - this.connectPromise = undefined; - this.requestTail = Promise.resolve(); - this.peerId = undefined; - this.helloResponse = undefined; - this.helloPromise = undefined; - } - - async connect() { - if (this.dataChannel?.readyState === "open") return; - if (this.connectPromise) return this.connectPromise; - if (typeof RTCPeerConnection === "undefined") { - throw new Error("This browser does not expose the RTCPeerConnection API"); - } - this.connectPromise = this.openDirectConnection(); - try { - await this.connectPromise; - } catch (error) { - this.close(); - throw error; - } finally { - this.connectPromise = undefined; - } - } - - async request(type, fields = {}, content = new Uint8Array()) { - const operation = this.requestTail.then(() => - this.requestDirect(type, fields, content), - ); - this.requestTail = operation.then( - () => undefined, - () => undefined, - ); - return operation; - } - - async openDirectConnection() { - const configuration = { iceServers: [] }; - if (typeof RTCPeerConnection.generateCertificate === "function") { - configuration.certificates = [ - await RTCPeerConnection.generateCertificate({ - name: "ECDSA", - namedCurve: "P-256", - }), - ]; - } - const peerConnection = new RTCPeerConnection(configuration); - const dataChannel = peerConnection.createDataChannel( - WEBRTC_DIRECT_DATA_CHANNEL, - { - ordered: true, - }, - ); - const inbox = new DataChannelInbox(dataChannel); - this.peerConnection = peerConnection; - this.dataChannel = dataChannel; - this.inbox = inbox; - const iceCredential = randomIceCredential(); - const offer = mungeOfferIceCredentials( - await peerConnection.createOffer(), - iceCredential, - ); - await peerConnection.setLocalDescription(offer); - await peerConnection.setRemoteDescription( - serverAnswerFromEndpoint(this.endpoint, iceCredential), - ); - await waitForDataChannelOpen(dataChannel); - } - - async requestDirect(type, fields, content) { - await this.connect(); - if (!(content instanceof Uint8Array) || content.length > MAX_CHUNK_SIZE) { - throw new Error( - `Request content must be at most ${MAX_CHUNK_SIZE} bytes`, - ); - } - const requestId = nextRequestId; - nextRequestId += 1; - const header = encoder.encode( - JSON.stringify({ - version: PROTOCOL_VERSION, - request_id: requestId, - content_length: content.length, - type, - ...fields, - }), - ); - if (header.length === 0 || header.length > MAX_RESPONSE_HEADER_BYTES) { - throw new Error(`Request header is ${header.length} bytes`); - } - const frame = new Uint8Array(4 + header.length + content.length); - new DataView(frame.buffer).setUint32(0, header.length, false); - frame.set(header, 4); - frame.set(content, 4 + header.length); - for ( - let offset = 0; - offset < frame.length; - offset += WEBRTC_WRITE_CHUNK_BYTES - ) { - await waitForDataChannelCapacity(this.dataChannel); - this.dataChannel.send( - frame.subarray(offset, offset + WEBRTC_WRITE_CHUNK_BYTES), - ); - } - let timeout; - let responseFrame; - try { - responseFrame = await Promise.race([ - readResponseFrame(this.inbox), - new Promise((_, reject) => { - timeout = setTimeout( - () => reject(new Error("WebRTC request timed out")), - REQUEST_TIMEOUT_MS, - ); - }), - ]); - } catch (error) { - this.close(); - throw error; - } finally { - clearTimeout(timeout); - } - - const response = parseResponseFrame(responseFrame); - if (response.header.request_id !== requestId) { - throw new Error( - `Response ID ${response.header.request_id} does not match request ${requestId}`, - ); - } - if (response.header.status === "error") { - const error = new Error( - response.header.message ?? "Node returned an error", - ); - error.code = response.header.code; - throw error; - } - return response; - } - - async hello() { - if (this.helloResponse && this.dataChannel?.readyState === "open") { - return this.helloResponse; - } - if (this.helloPromise) return this.helloPromise; - this.helloPromise = (async () => { - const challengeBytes = crypto.getRandomValues(new Uint8Array(32)); - const challenge = bytesToHex(challengeBytes); - const { header } = await this.request("hello", { challenge }); - this.peerId = verifyHelloIdentity(header, this.endpoint, challengeBytes); - this.helloResponse = header; - return header; - })(); - try { - return await this.helloPromise; - } finally { - this.helloPromise = undefined; - } - } - - async findNode(target, count = 20) { - hexToBytes(target, 32); - const { header } = await this.request("find_node", { target, count }); - if (header.type !== "nodes") throw new Error("Expected a NODES response"); - if (header.target.toLowerCase() !== target.toLowerCase()) { - throw new Error("Node returned results for a different lookup target"); - } - if (!Array.isArray(header.nodes)) { - throw new Error("Node returned an invalid node list"); - } - for (const node of header.nodes) { - hexToBytes(node.peer_id, 32); - if (node.webrtc_direct) { - const parsed = normalizeEndpoint(node.webrtc_direct); - if (parsed.peerId !== node.peer_id.toLowerCase()) { - throw new Error( - `Node ${node.peer_id} advertised another peer's endpoint`, - ); - } - } - } - return header.nodes; - } - - async getChunk(address) { - hexToBytes(address, 32); - const response = await this.request("get_chunk", { address }); - if (response.header.status === "not_found") { - const error = new Error(`Chunk ${address} was not found on this node`); - error.code = "not_found"; - throw error; - } - if (response.header.type !== "chunk") { - throw new Error("Expected a CHUNK response"); - } - if (response.header.address.toLowerCase() !== address.toLowerCase()) { - throw new Error("Node returned a different chunk address"); - } - if (response.header.size !== response.content.length) { - throw new Error("Chunk metadata size does not match its content"); - } - const hash = verifyChunk(address, response.content); - return { content: response.content, hash }; - } - - async quoteChunk(address, size) { - hexToBytes(address, 32); - if (!Number.isSafeInteger(size) || size < 0 || size > MAX_CHUNK_SIZE) { - throw new Error(`Invalid chunk size ${size}`); - } - const { header } = await this.request("quote_chunk", { address, size }); - if (header.type !== "storage_quote") { - throw new Error("Expected a STORAGE_QUOTE response"); - } - if (header.address.toLowerCase() !== address.toLowerCase()) { - throw new Error("Node returned a quote for a different chunk address"); - } - return { - quote: header.quote, - alreadyStored: Boolean(header.already_stored), - }; - } - - async putChunk(address, content, quote, transactionHash) { - hexToBytes(address, 32); - hexToBytes(transactionHash, 32); - verifyChunk(address, content); - const { header } = await this.request( - "put_chunk", - { address, quote, transaction_hash: transactionHash }, - content, - ); - if (header.type !== "chunk_stored") { - throw new Error("Expected a CHUNK_STORED response"); - } - if (header.address.toLowerCase() !== address.toLowerCase()) { - throw new Error("Node stored a different chunk address"); - } - return { - address: header.address, - alreadyStored: Boolean(header.already_stored), - }; - } - - close() { - this.dataChannel?.close(); - this.peerConnection?.close(); - this.dataChannel = undefined; - this.peerConnection = undefined; - this.inbox = undefined; - this.connectPromise = undefined; - this.peerId = undefined; - this.helloResponse = undefined; - this.helloPromise = undefined; - } -} - -function endpointKey(endpoint) { - const normalized = normalizeEndpoint(endpoint); - return normalized.multiaddr; -} - -const DEFAULT_MAX_POOLED_CLIENTS = 10; - -/** - * A bounded set of reusable browser-to-node WebRTC associations. - * - * In Safari, the PoC exhausted WebRTC resources when it rapidly replaced every - * seed connection for each chunk, even though callers invoked `close()`. - * Keeping authenticated, persistent DataChannels avoids relying on prompt - * reclamation and avoids repeating ICE, DTLS, SCTP, and HELLO for each lookup - * and record. - */ -export class BrowserNodeClientPool { - constructor({ - maxClients = DEFAULT_MAX_POOLED_CLIENTS, - clientFactory = (endpoint) => new BrowserNodeClient(endpoint), - } = {}) { - if (!Number.isSafeInteger(maxClients) || maxClients < 1) { - throw new Error("WebRTC client pool size must be a positive integer"); - } - if (typeof clientFactory !== "function") { - throw new Error("WebRTC client pool factory must be a function"); - } - this.maxClients = maxClients; - this.clientFactory = clientFactory; - this.entries = new Map(); - this.waiters = []; - this.clock = 0; - this.closed = false; - } - - get size() { - return this.entries.size; - } - - async withClient(endpoint, operation) { - if (typeof operation !== "function") { - throw new Error("WebRTC client pool operation must be a function"); - } - const entry = await this.acquire(endpoint); - try { - return await operation(entry.client); - } finally { - this.release(entry); - } - } - - async acquire(endpoint) { - const key = endpointKey(endpoint); - for (;;) { - if (this.closed) throw new Error("WebRTC client pool is closed"); - - const existing = this.entries.get(key); - if (existing) { - existing.active += 1; - existing.lastUsed = ++this.clock; - return existing; - } - - if (this.entries.size < this.maxClients) { - const entry = { - key, - client: this.clientFactory(endpoint), - active: 1, - lastUsed: ++this.clock, - }; - this.entries.set(key, entry); - return entry; - } - - let oldestIdle; - for (const candidate of this.entries.values()) { - if ( - candidate.active === 0 && - (!oldestIdle || candidate.lastUsed < oldestIdle.lastUsed) - ) { - oldestIdle = candidate; - } - } - if (oldestIdle) { - this.entries.delete(oldestIdle.key); - oldestIdle.client.close(); - continue; - } - - await new Promise((resolve) => this.waiters.push(resolve)); - } - } - - release(entry) { - entry.active = Math.max(0, entry.active - 1); - entry.lastUsed = ++this.clock; - this.waiters.shift()?.(); - } - - close() { - if (this.closed) return; - this.closed = true; - for (const entry of this.entries.values()) entry.client.close(); - this.entries.clear(); - for (const wake of this.waiters.splice(0)) wake(); - } -} - -export async function iterativeFindClosest( - seedEndpoints, - target, - { - k = 20, - alpha = 3, - maxIterations = 20, - onProgress = () => {}, - clientPool, - } = {}, -) { - hexToBytes(target, 32); - if (!Array.isArray(seedEndpoints) || seedEndpoints.length === 0) { - throw new Error("At least one seed endpoint is required"); - } - - const ownsClientPool = clientPool === undefined; - const pool = clientPool ?? new BrowserNodeClientPool(); - const failures = []; - const seedNodes = []; - - await Promise.all( - seedEndpoints.map(async (endpoint) => { - const seedName = - typeof endpoint === "string" - ? endpoint - : (endpoint?.multiaddr ?? "seed"); - try { - const hello = await pool.withClient(endpoint, (client) => - client.hello(), - ); - seedNodes.push({ - peer_id: hello.peer_id, - native_addresses: [], - reliability: 1, - webrtc_direct: hello.endpoint, - }); - onProgress(`Connected seed ${hello.peer_id}`); - } catch (error) { - failures.push({ peerId: seedName, error }); - onProgress(`Seed ${seedName} failed: ${error.message}`); - } - }), - ); - if (seedNodes.length === 0) { - if (ownsClientPool) pool.close(); - const detail = failures.map(({ error }) => error.message).join("; "); - throw new Error(`Could not connect to any WebRtcDirect seed: ${detail}`); - } - - const lookup = new BrowserIterativeLookupNative( - target, - k, - alpha, - maxIterations, - ); - lookup.addCandidates(seedNodes); - try { - await lookup.run( - async ({ target: lookupTarget, count, iteration, candidates }) => - Promise.all( - candidates.map(async (candidate) => { - try { - const nodes = await pool.withClient( - candidate.webrtc_direct, - async (client) => { - if (!client.peerId) await client.hello(); - return client.findNode(lookupTarget, count); - }, - ); - onProgress( - `Iteration ${iteration}: ${candidate.peer_id} returned ${nodes.length} nodes`, - ); - return { - status: "succeeded", - responder: candidate.peer_id, - candidates: nodes, - }; - } catch (error) { - failures.push({ peerId: candidate.peer_id, error }); - onProgress(`Query ${candidate.peer_id} failed: ${error.message}`); - return { status: "failed", responder: candidate.peer_id }; - } - }), - ), - ); - - return { - nodes: lookup.results(), - queried: lookup.queriedPeers(), - failures, - clientPool: pool, - ownsClientPool, - }; - } catch (error) { - if (ownsClientPool) pool.close(); - throw error; - } -} - -export async function getChunkFromClosest( - seedEndpoints, - address, - { onProgress = () => {}, ...lookupOptions } = {}, -) { - hexToBytes(address, 32); - const lookup = await iterativeFindClosest(seedEndpoints, address, { - ...lookupOptions, - onProgress, - }); - const attempted = []; - - try { - for (const node of lookup.nodes) { - if (!node.webrtc_direct) continue; - try { - onProgress(`Requesting ${address} from ${node.peer_id}`); - const chunk = await lookup.clientPool.withClient( - node.webrtc_direct, - async (client) => { - if (!client.peerId) await client.hello(); - return client.getChunk(address); - }, - ); - return { ...chunk, node, lookup }; - } catch (error) { - attempted.push({ peerId: node.peer_id, error }); - onProgress( - `Node ${node.peer_id} did not return the file: ${error.message}`, - ); - } - } - } finally { - if (lookup.ownsClientPool) lookup.clientPool.close(); - } - - const detail = attempted - .map(({ peerId, error }) => `${peerId}: ${error.message}`) - .join("; "); - throw new Error( - `No closest WebRtcDirect node returned chunk ${address}${detail ? ` (${detail})` : ""}`, - ); -} diff --git a/web/src/protocol.test.js b/web/src/protocol.test.js index dbc7998a..ff6e2f20 100644 --- a/web/src/protocol.test.js +++ b/web/src/protocol.test.js @@ -1,28 +1,40 @@ import assert from "node:assert/strict"; import test from "node:test"; -import { blake3 } from "@noble/hashes/blake3.js"; -import { ml_dsa65 } from "@noble/post-quantum/ml-dsa.js"; import { - BrowserNodeClientPool, - bytesToHex, - hexToBytes, + BrowserNetworkClient, + BrowserNodeClient, mungeOfferIceCredentials, parseResponseFrame, parseWebRtcDirectMultiaddr, - readResponseFrame, + paymentQuoteHash, serverAnswerFromEndpoint, - verifyChunk, - verifyHelloIdentity, -} from "./protocol.js"; +} from "../pkg/ant_core.js"; -test("hex conversion enforces fixed widths", () => { - const value = "ab".repeat(32); - assert.equal(bytesToHex(hexToBytes(value, 32)), value); - assert.throws(() => hexToBytes("abcd", 32), /Expected 32 bytes/); - assert.throws(() => hexToBytes("zz", 1), /hexadecimal/); +test("Rust/WASM parses stable certificate-pinned WebRTC Direct addresses", () => { + const peerId = "ab".repeat(32); + const multiaddr = webRtcDirectMultiaddr(peerId, 0x11); + const parsed = parseWebRtcDirectMultiaddr(multiaddr); + + assert.equal(parsed.hostProtocol, "ip4"); + assert.equal(parsed.host, "127.0.0.1"); + assert.equal(parsed.port, 24000); + assert.equal(parsed.peerId, peerId); + assert.deepEqual([...parsed.certificateHash], Array(32).fill(0x11)); + assert.throws( + () => + parseWebRtcDirectMultiaddr( + `/dns/node.example/udp/24000/webrtc-direct/certhash/${certificateMultihash(0x11)}/p2p/${peerId}`, + ), + /literal IP/, + ); + + const node = new BrowserNodeClient(multiaddr); + node.close(); + const network = new BrowserNetworkClient([{ multiaddr }]); + network.close(); }); -test("response framing preserves a raw binary body", () => { +test("Rust/WASM validates response framing with a raw binary body", () => { const header = new TextEncoder().encode( JSON.stringify({ version: 3, @@ -44,75 +56,8 @@ test("response framing preserves a raw binary body", () => { assert.deepEqual([...parsed.content], [1, 2, 3]); }); -test("response framing completes without waiting for stream EOF", async () => { - const header = new TextEncoder().encode( - JSON.stringify({ - version: 3, - request_id: 10, - status: "ok", - content_length: 3, - type: "chunk", - }), - ); - const frame = new Uint8Array(4 + header.length + 3); - new DataView(frame.buffer).setUint32(0, header.length, false); - frame.set(header, 4); - frame.set([4, 5, 6], 4 + header.length); - let requestedAnotherChunk = false; - const stream = { - async *[Symbol.asyncIterator]() { - yield frame.subarray(0, 2); - yield frame.subarray(2, frame.length); - requestedAnotherChunk = true; - await new Promise(() => {}); - }, - }; - - const received = await readResponseFrame(stream); - assert.deepEqual(received, frame); - assert.equal(requestedAnotherChunk, false); -}); - -test("WebRTC Direct multiaddresses carry one stable certificate hash", () => { - const peerId = "ab".repeat(32); - const multiaddr = webrtc_directMultiaddr( - "ip4", - "127.0.0.1", - 24000, - peerId, - 0x11, - ); - const parsed = parseWebRtcDirectMultiaddr(multiaddr); - - assert.equal(parsed.hostProtocol, "ip4"); - assert.equal(parsed.host, "127.0.0.1"); - assert.equal(parsed.port, 24000); - assert.equal(parsed.peerId, peerId); - assert.deepEqual([...parsed.certificateHash], Array(32).fill(0x11)); - assert.throws( - () => - parseWebRtcDirectMultiaddr( - `/ip4/127.0.0.1/udp/24000/webrtc-direct/p2p/${peerId}`, - ), - /certhash|incomplete/, - ); - assert.throws( - () => - parseWebRtcDirectMultiaddr( - `/dns/node.example/udp/24000/webrtc-direct/certhash/${certificateMultihash(0x11)}/p2p/${peerId}`, - ), - /literal IP/, - ); -}); - -test("the browser synthesizes a certificate-pinned ICE-lite answer", () => { - const endpoint = webrtc_directMultiaddr( - "ip4", - "127.0.0.1", - 24000, - "ab".repeat(32), - 0x11, - ); +test("Rust/WASM synthesizes the pinned answer and shared ICE credentials", () => { + const endpoint = webRtcDirectMultiaddr("ab".repeat(32), 0x11); const credential = `saorsa+webrtc+v1/${"a".repeat(32)}`; const answer = serverAnswerFromEndpoint(endpoint, credential); @@ -122,164 +67,23 @@ test("the browser synthesizes a certificate-pinned ICE-lite answer", () => { answer.sdp, /m=application 24000 UDP\/DTLS\/SCTP webrtc-datachannel/, ); - assert.match( - answer.sdp, - new RegExp(`a=ice-ufrag:${credential.replaceAll("+", "\\+")}`), - ); assert.match(answer.sdp, /a=fingerprint:sha-256 11:11:11:11/); - assert.match(answer.sdp, /a=setup:passive/); -}); -test("the browser offer uses the Saorsa credential as ufrag and password", () => { - const credential = `saorsa+webrtc+v1/${"b".repeat(32)}`; const offer = mungeOfferIceCredentials( { type: "offer", - sdp: "v=0\r\na=ice-ufrag:browser-generated\r\na=ice-pwd:browser-secret\r\n", + sdp: "v=0\r\na=ice-ufrag:old\r\na=ice-pwd:secret\r\n", }, credential, ); - assert.match( - offer.sdp, - new RegExp(`a=ice-ufrag:${credential.replaceAll("+", "\\+")}`), - ); - assert.match( - offer.sdp, - new RegExp(`a=ice-pwd:${credential.replaceAll("+", "\\+")}`), - ); -}); - -test("the browser reuses a bounded pool of WebRTC node connections", async () => { - const endpoints = [0x11, 0x22, 0x33].map((hashByte, index) => - webrtc_directMultiaddr( - "ip4", - "127.0.0.1", - 24000 + index, - (index + 1).toString(16).padStart(2, "0").repeat(32), - hashByte, - ), - ); - const created = []; - const closed = []; - const pool = new BrowserNodeClientPool({ - maxClients: 2, - clientFactory: (endpoint) => { - const client = { endpoint, close: () => closed.push(endpoint) }; - created.push(client); - return client; - }, - }); - - let firstClient; - await pool.withClient(endpoints[0], async (client) => { - firstClient = client; - }); - await pool.withClient(endpoints[0], async (client) => { - assert.equal(client, firstClient); - }); - await pool.withClient(endpoints[1], async () => {}); - await pool.withClient(endpoints[2], async () => {}); - - assert.equal(created.length, 3); - assert.equal(pool.size, 2); - assert.deepEqual(closed, [endpoints[0]]); - pool.close(); - assert.equal(closed.length, 3); -}); - -test("the WebRTC client pool waits instead of exceeding its connection cap", async () => { - const firstEndpoint = webrtc_directMultiaddr( - "ip4", - "127.0.0.1", - 24000, - "11".repeat(32), - 0x11, - ); - const secondEndpoint = webrtc_directMultiaddr( - "ip4", - "127.0.0.1", - 24001, - "22".repeat(32), - 0x22, - ); - const created = []; - let releaseFirst; - let secondEntered = false; - const pool = new BrowserNodeClientPool({ - maxClients: 1, - clientFactory: (endpoint) => { - const client = { endpoint, close() {} }; - created.push(client); - return client; - }, - }); - const first = pool.withClient( - firstEndpoint, - () => new Promise((resolve) => (releaseFirst = resolve)), - ); - const second = pool.withClient(secondEndpoint, async () => { - secondEntered = true; - }); - - await Promise.resolve(); - await Promise.resolve(); - assert.equal(created.length, 1); - assert.equal(secondEntered, false); - releaseFirst(); - await Promise.all([first, second]); - assert.equal(created.length, 2); - assert.equal(secondEntered, true); - pool.close(); -}); - -test("HELLO binds the persistent ANT identity to the WebRTC endpoint", () => { - const challenge = new Uint8Array(32).fill(0x22); - const { publicKey, secretKey } = ml_dsa65.keygen( - new Uint8Array(32).fill(0x33), - ); - const peerId = bytesToHex(blake3(publicKey)); - const multiaddr = webrtc_directMultiaddr( - "ip4", - "127.0.0.1", - 24000, - peerId, - 0x44, - ); - const transcript = concatBytes( - new TextEncoder().encode("autonomi-webrtc-direct-hello-v1\0"), - challenge, - new TextEncoder().encode(peerId), - new TextEncoder().encode(multiaddr), - ); - const header = { - type: "hello", - protocol: "autonomi.web.poc.v3", - challenge: bytesToHex(challenge), - peer_id: peerId, - public_key: bytesToHex(publicKey), - signature: bytesToHex(ml_dsa65.sign(transcript, secretKey)), - endpoint: { multiaddr }, - }; - - assert.equal(verifyHelloIdentity(header, multiaddr, challenge), peerId); - assert.throws( - () => - verifyHelloIdentity( - { ...header, challenge: "00".repeat(32) }, - multiaddr, - challenge, - ), - /different HELLO challenge/, - ); + assert.match(offer.sdp, new RegExp(`a=ice-ufrag:${escapeRegex(credential)}`)); + assert.match(offer.sdp, new RegExp(`a=ice-pwd:${escapeRegex(credential)}`)); }); -test("BLAKE3 verification accepts the canonical empty hash", () => { - const emptyHash = - "af1349b9f5f9a1a6a0404dea36dcc9499bcb25c9adc112b7cc9a93cae41f3262"; - assert.equal(verifyChunk(emptyHash, new Uint8Array()), emptyHash); - assert.throws( - () => verifyChunk("00".repeat(32), new Uint8Array()), - /BLAKE3 mismatch/, +test("Rust/WASM uses the native EVM PaymentQuote Keccak hash", () => { + assert.equal( + paymentQuoteHash(Uint8Array.of(0, 1), Uint8Array.of(2), Uint8Array.of(3)), + "d98f2e8134922f73748703c8e7084d42f13d2fa1439936ef5a3abcf5646fe83f", ); }); @@ -288,18 +92,10 @@ function certificateMultihash(byte) { return `u${Buffer.from(multihash).toString("base64url")}`; } -function concatBytes(...parts) { - const output = new Uint8Array( - parts.reduce((length, part) => length + part.length, 0), - ); - let offset = 0; - for (const part of parts) { - output.set(part, offset); - offset += part.length; - } - return output; +function webRtcDirectMultiaddr(peerId, certificateByte) { + return `/ip4/127.0.0.1/udp/24000/webrtc-direct/certhash/${certificateMultihash(certificateByte)}/p2p/${peerId}`; } -function webrtc_directMultiaddr(hostProtocol, host, port, peerId, hashByte) { - return `/${hostProtocol}/${host}/udp/${port}/webrtc-direct/certhash/${certificateMultihash(hashByte)}/p2p/${peerId}`; +function escapeRegex(value) { + return value.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); } diff --git a/web/src/upload.js b/web/src/upload.js deleted file mode 100644 index 45cdf7d1..00000000 --- a/web/src/upload.js +++ /dev/null @@ -1,232 +0,0 @@ -import { encryptPublicFile as encryptPublicFileNative } from "../pkg/ant_core.js"; -import { BrowserNodeClientPool, iterativeFindClosest } from "./protocol.js"; -import { payForStorageQuotes, verifyStorageQuote } from "./payment.js"; - -export const MAX_BROWSER_UPLOAD_BYTES = 64 * 1024 * 1024; -const MAX_STORE_TARGETS = 7; - -export async function encryptPublicFile( - content, - { - name = "upload.bin", - contentType = "application/octet-stream", - encrypt = encryptPublicFileNative, - } = {}, -) { - if (!(content instanceof Uint8Array)) - throw new Error("Upload content must be bytes"); - if (content.length < 3) - throw new Error("Self-encryption requires a file of at least 3 bytes"); - if (content.length > MAX_BROWSER_UPLOAD_BYTES) { - throw new Error( - `Browser uploads are limited to ${MAX_BROWSER_UPLOAD_BYTES} bytes`, - ); - } - const encrypted = encrypt(content); - return { - descriptor: { - name, - address: encrypted.address, - size: content.length, - content_type: contentType || "application/octet-stream", - blake3: encrypted.blake3, - data_map_size: encrypted.data_map_size, - chunks: encrypted.chunks, - replicas: 0, - }, - records: encrypted.records, - }; -} - -function assertUploadNode(hello, paymentNetwork) { - if ( - !Array.isArray(hello.capabilities) || - !hello.capabilities.includes("quote_chunk") || - !hello.capabilities.includes("put_chunk") - ) { - throw new Error("Node does not advertise paid browser uploads"); - } - const advertised = hello.payment; - if ( - !advertised || - new URL(advertised.rpc_url).toString() !== - new URL(paymentNetwork.rpc_url).toString() || - advertised.payment_token_address?.toLowerCase() !== - paymentNetwork.payment_token_address.toLowerCase() || - advertised.payment_vault_address?.toLowerCase() !== - paymentNetwork.payment_vault_address.toLowerCase() - ) { - throw new Error( - "Node advertises a different payment network than the manifest", - ); - } -} - -async function prepareRecord( - seedEndpoints, - paymentNetwork, - record, - clientPool, - onProgress, -) { - onProgress(`Finding closest nodes for ${record.address}`); - const lookup = await iterativeFindClosest(seedEndpoints, record.address, { - onProgress, - clientPool, - }); - const endpoints = lookup.nodes - .filter((node) => node.webrtc_direct) - .slice(0, MAX_STORE_TARGETS) - .map((node) => ({ peerId: node.peer_id, endpoint: node.webrtc_direct })); - const failures = []; - for (const target of endpoints) { - try { - const prepared = await clientPool.withClient( - target.endpoint, - async (client) => { - assertUploadNode(await client.hello(), paymentNetwork); - const response = await client.quoteChunk( - record.address, - record.content.length, - ); - const verified = verifyStorageQuote( - response.quote, - record.address, - target.peerId, - ); - if (response.alreadyStored) { - onProgress( - `Chunk ${record.address} is already stored; skipping payment`, - ); - return { record, alreadyStored: true, targets: endpoints }; - } - onProgress( - `Verified storage quote ${verified.quoteHash} from ${target.peerId}`, - ); - return { - record, - alreadyStored: false, - targets: [ - target, - ...endpoints.filter((candidate) => candidate !== target), - ], - verified, - }; - }, - ); - return prepared; - } catch (error) { - failures.push(`${target.peerId}: ${error.message}`); - } - } - throw new Error( - `No closest node supplied a valid quote (${failures.join("; ")})`, - ); -} - -async function storePrepared( - prepared, - paymentNetwork, - transactionHash, - clientPool, - onProgress, -) { - if (prepared.alreadyStored) return 1; - const attempts = await Promise.allSettled( - prepared.targets.map(async (target) => { - return clientPool.withClient(target.endpoint, async (client) => { - assertUploadNode(await client.hello(), paymentNetwork); - const result = await client.putChunk( - prepared.record.address, - prepared.record.content, - prepared.verified.quote, - transactionHash, - ); - onProgress( - `${result.alreadyStored ? "Confirmed" : "Stored"} ${prepared.record.address} on ${target.peerId}`, - ); - return result; - }); - }), - ); - const stored = attempts.filter( - (attempt) => attempt.status === "fulfilled", - ).length; - if (stored === 0) { - const failures = attempts - .filter((attempt) => attempt.status === "rejected") - .map((attempt) => attempt.reason?.message ?? String(attempt.reason)); - throw new Error( - `Paid chunk was rejected by every closest node: ${failures.join("; ")}`, - ); - } - return stored; -} - -export async function uploadPublicFile( - seedEndpoints, - paymentNetwork, - file, - walletSecret, - { onProgress = () => {}, encrypt = encryptPublicFileNative } = {}, -) { - const content = new Uint8Array(await file.arrayBuffer()); - onProgress( - `Self-encrypting ${file.name} with native ant-core WASM (${content.length.toLocaleString()} bytes)`, - ); - const encrypted = await encryptPublicFile(content, { - name: file.name, - contentType: file.type, - encrypt, - }); - const clientPool = new BrowserNodeClientPool(); - try { - const prepared = []; - for (let index = 0; index < encrypted.records.length; index += 1) { - onProgress(`Preparing record ${index + 1}/${encrypted.records.length}`); - prepared.push( - await prepareRecord( - seedEndpoints, - paymentNetwork, - encrypted.records[index], - clientPool, - onProgress, - ), - ); - } - const payable = prepared.filter((record) => !record.alreadyStored); - let transactionHash; - let totalAmount = 0n; - if (payable.length > 0) { - const payment = await payForStorageQuotes( - paymentNetwork, - payable.map((record) => record.verified), - walletSecret, - { onProgress }, - ); - transactionHash = payment.transactionHash; - totalAmount = payment.totalAmount; - } - let replicas = Number.POSITIVE_INFINITY; - for (let index = 0; index < prepared.length; index += 1) { - onProgress(`Storing record ${index + 1}/${prepared.length}`); - const stored = await storePrepared( - prepared[index], - paymentNetwork, - transactionHash, - clientPool, - onProgress, - ); - replicas = Math.min(replicas, stored); - } - encrypted.descriptor.replicas = Number.isFinite(replicas) ? replicas : 0; - return { - file: encrypted.descriptor, - transactionHash, - storageCostAtto: totalAmount.toString(), - records: encrypted.records.length, - }; - } finally { - clientPool.close(); - } -} diff --git a/web/src/upload.test.js b/web/src/upload.test.js deleted file mode 100644 index e53ec6ae..00000000 --- a/web/src/upload.test.js +++ /dev/null @@ -1,38 +0,0 @@ -import assert from "node:assert/strict"; -import test from "node:test"; -import { encryptPublicFile } from "./upload.js"; - -test("adapts native ant-core WASM output to browser file metadata", async () => { - const content = new TextEncoder().encode("browser fixture"); - const dataMap = Uint8Array.of(7, 8, 9); - const encryptedRecord = Uint8Array.of(1, 2, 3); - const encrypted = await encryptPublicFile(content, { - name: "fixture.txt", - contentType: "text/plain", - encrypt: () => ({ - address: "44".repeat(32), - blake3: "55".repeat(32), - data_map_size: dataMap.length, - chunks: [ - { - index: 0, - dst_hash: "66".repeat(32), - src_hash: "77".repeat(32), - src_size: content.length, - }, - ], - records: [ - { address: "66".repeat(32), content: encryptedRecord }, - { address: "44".repeat(32), content: dataMap }, - ], - }), - }); - - assert.equal(encrypted.descriptor.name, "fixture.txt"); - assert.equal(encrypted.descriptor.address, "44".repeat(32)); - assert.equal(encrypted.descriptor.content_type, "text/plain"); - assert.equal(encrypted.descriptor.size, content.length); - assert.equal(encrypted.records.length, 2); - assert.deepEqual(encrypted.records[0].content, encryptedRecord); - assert.deepEqual(encrypted.records[1].content, dataMap); -}); From 1e23045ce641eba28fd1bedd4c91c0246a9128d0 Mon Sep 17 00:00:00 2001 From: Mick van Dijke <12992260+mickvandijke@users.noreply.github.com> Date: Wed, 26 Aug 2026 14:11:06 +0200 Subject: [PATCH 08/31] feat(web): share protocol logic and automate browser smoke test --- .gitignore | 3 + Cargo.lock | 7 +- Cargo.toml | 1 + ant-core/Cargo.toml | 14 +-- ant-core/src/browser.rs | 1 - ant-core/src/browser/crypto.rs | 71 ------------- ant-core/src/browser/payment.rs | 133 +++++++++---------------- ant-core/src/browser/protocol.rs | 2 +- ant-core/src/browser/wasm_transport.rs | 11 +- web/README.md | 25 +++-- web/e2e/webrtc-direct.spec.js | 28 ++++++ web/package-lock.json | 64 ++++++++++++ web/package.json | 6 +- web/playwright.config.js | 69 +++++++++++++ web/src/main.js | 5 + 15 files changed, 258 insertions(+), 182 deletions(-) delete mode 100644 ant-core/src/browser/crypto.rs create mode 100644 web/e2e/webrtc-direct.spec.js create mode 100644 web/playwright.config.js diff --git a/.gitignore b/.gitignore index 0b6b5d47..238a2f15 100644 --- a/.gitignore +++ b/.gitignore @@ -2,6 +2,9 @@ /web/node_modules/ /web/dist/ /web/pkg/ +/web/test-results/ +/web/playwright-report/ +/web/.playwright-devnet/ .cargo/config.toml .claude/plans/ .claude/scheduled_tasks.lock diff --git a/Cargo.lock b/Cargo.lock index 6c34de84..6624000a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -847,7 +847,6 @@ dependencies = [ "blake3", "bytes", "console_error_panic_hook", - "fips204", "flate2", "fs2", "futures", @@ -861,7 +860,6 @@ dependencies = [ "libc", "lru", "openssl", - "postcard", "rand 0.8.6", "reqwest 0.12.28", "rmp-serde", @@ -878,7 +876,6 @@ dependencies = [ "tar", "tempfile", "thiserror 2.0.18", - "tiny-keccak", "tokio", "tokio-test", "tokio-util", @@ -958,18 +955,18 @@ dependencies = [ [[package]] name = "ant-protocol" version = "2.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3081ee130dd45e8166bc8ac4d789aac17d143e2e7f54f1c3006503dc63cf3edb" dependencies = [ "blake3", "bytes", "evmlib", + "fips204", "hex", "postcard", "rmp-serde", "saorsa-core", "saorsa-pqc 0.5.1", "serde", + "tiny-keccak", "tokio", "tracing", ] diff --git a/Cargo.toml b/Cargo.toml index 07d8c6e7..e2df0d2c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -3,5 +3,6 @@ members = ["ant-core", "ant-cli"] resolver = "2" [patch.crates-io] +ant-protocol = { path = "../ant-protocol-web-support" } saorsa-core = { path = "../saorsa-core-web-support" } saorsa-transport = { path = "../saorsa-transport-web-support" } diff --git a/ant-core/Cargo.toml b/ant-core/Cargo.toml index 48721471..79480fcc 100644 --- a/ant-core/Cargo.toml +++ b/ant-core/Cargo.toml @@ -18,7 +18,6 @@ blake3 = "1" base64 = "0.22" bytes = "1" hex = "0.4" -postcard = { version = "1.1.3", features = ["use-std"] } rmp-serde = "1" serde = { version = "1", features = ["derive"] } serde_bytes = "0.11" @@ -26,7 +25,6 @@ serde_json = "1" self_encryption = "0.36" saorsa-dht-lookup = { version = "0.1.0", path = "../../saorsa-core-web-support/crates/saorsa-dht-lookup" } thiserror = "2" -tiny-keccak = { version = "2", features = ["keccak"] } url = "2" # Node management @@ -55,7 +53,7 @@ tower-http = { version = "0.6.8", features = ["cors"], optional = true } # under `ant_protocol::{evm, transport, pqc}`. This is the ONE pin for # those three deps — do not add direct evmlib/saorsa-core/saorsa-pqc # deps here or the version can skew between ant-client and ant-node. -ant-protocol = { version = "2.3.1", optional = true } +ant-protocol = { version = "2.3.1", default-features = false, optional = true } xor_name = { version = "5", optional = true } futures = { version = "0.3", optional = true } tracing = { version = "0.1", optional = true } @@ -100,11 +98,6 @@ web-sys = { version = "0.3", optional = true, features = [ "RtcSessionDescriptionInit", ] } gloo-timers = { version = "0.3", features = ["futures"], optional = true } -# `ant-protocol` currently pulls native Tokio networking on wasm32. Use the -# same FIPS-204 primitive as `saorsa-pqc` for verification-only browser builds -# until ant-protocol's pure wire layer can be built independently of transport. -fips204 = { version = "0.4.6", default-features = false, features = ["ml-dsa-65"], optional = true } - [target.'cfg(target_arch = "wasm32")'.dependencies] # self_encryption/rand use getrandom 0.2. Browser entropy is supplied by the # Web Crypto API through its `js` feature. @@ -127,6 +120,8 @@ windows-sys = { version = "0.61", features = ["Win32_Foundation", "Win32_System_ default = ["native"] native = [ "dep:ant-protocol", + "ant-protocol/native", + "ant-protocol/logging", "dep:async-stream", "dep:axum", "dep:flate2", @@ -153,12 +148,13 @@ native = [ "dep:zip", ] browser-wasm = [ + "dep:ant-protocol", + "ant-protocol/portable", "dep:console_error_panic_hook", "dep:futures-channel", "dep:futures-util", "dep:gloo-timers", "dep:js-sys", - "dep:fips204", "dep:serde-wasm-bindgen", "dep:wasm-bindgen", "dep:wasm-bindgen-futures", diff --git a/ant-core/src/browser.rs b/ant-core/src/browser.rs index 8cf4f6e1..68acadd1 100644 --- a/ant-core/src/browser.rs +++ b/ant-core/src/browser.rs @@ -6,7 +6,6 @@ //! host adapter; only DOM, file-save, and wallet transaction submission remain //! in JavaScript. -mod crypto; pub mod manifest; pub mod payment; pub mod protocol; diff --git a/ant-core/src/browser/crypto.rs b/ant-core/src/browser/crypto.rs deleted file mode 100644 index acdb442b..00000000 --- a/ant-core/src/browser/crypto.rs +++ /dev/null @@ -1,71 +0,0 @@ -#[cfg(feature = "native")] -use ant_protocol::pqc::api::{ml_dsa_65, MlDsaPublicKey, MlDsaSignature, MlDsaVariant}; -#[cfg(all(not(feature = "native"), feature = "browser-wasm"))] -use fips204::{ - ml_dsa_65, - traits::{SerDes as _, Verifier as _}, -}; -use tiny_keccak::{Hasher as _, Keccak}; - -pub(crate) fn verify_ml_dsa_65( - public_key: &[u8], - signature: &[u8], - message: &[u8], - context: &[u8], -) -> bool { - verify_ml_dsa_65_inner(public_key, signature, message, context).unwrap_or(false) -} - -#[cfg(feature = "native")] -fn verify_ml_dsa_65_inner( - public_key: &[u8], - signature: &[u8], - message: &[u8], - context: &[u8], -) -> Result { - let public_key = MlDsaPublicKey::from_bytes(MlDsaVariant::MlDsa65, public_key) - .map_err(|error| error.to_string())?; - let signature = MlDsaSignature::from_bytes(MlDsaVariant::MlDsa65, signature) - .map_err(|error| error.to_string())?; - ml_dsa_65() - .verify_with_context(&public_key, message, &signature, context) - .map_err(|error| error.to_string()) -} - -#[cfg(all(not(feature = "native"), feature = "browser-wasm"))] -fn verify_ml_dsa_65_inner( - public_key: &[u8], - signature: &[u8], - message: &[u8], - context: &[u8], -) -> Result { - let public_key: [u8; ml_dsa_65::PK_LEN] = public_key - .try_into() - .map_err(|_| "invalid ML-DSA-65 public key length".to_string())?; - let signature: [u8; ml_dsa_65::SIG_LEN] = signature - .try_into() - .map_err(|_| "invalid ML-DSA-65 signature length".to_string())?; - let public_key = - ml_dsa_65::PublicKey::try_from_bytes(public_key).map_err(ToString::to_string)?; - Ok(public_key.verify(message, &signature, context)) -} - -#[cfg(not(any(feature = "native", feature = "browser-wasm")))] -fn verify_ml_dsa_65_inner( - _public_key: &[u8], - _signature: &[u8], - _message: &[u8], - _context: &[u8], -) -> Result { - Err("ML-DSA verification requires the native or browser-wasm feature".to_string()) -} - -pub(crate) fn keccak256(parts: &[&[u8]]) -> [u8; 32] { - let mut hasher = Keccak::v256(); - for part in parts { - hasher.update(part); - } - let mut output = [0u8; 32]; - hasher.finalize(&mut output); - output -} diff --git a/ant-core/src/browser/payment.rs b/ant-core/src/browser/payment.rs index 0cede796..a986ae79 100644 --- a/ant-core/src/browser/payment.rs +++ b/ant-core/src/browser/payment.rs @@ -1,17 +1,19 @@ //! Verification and payment planning shared by native and browser clients. -use super::crypto::{keccak256, verify_ml_dsa_65}; use super::protocol::normalize_hex; +use ant_protocol::crypto::verify_ml_dsa_65; +use ant_protocol::payment::{ + calculate_price_wei, commitment_hash, payment_quote_bytes_for_signing, + verify_commitment_signature, StorageCommitment, MAX_COMMITMENT_KEY_COUNT, + MAX_COMMITMENT_SIDECAR_BYTES, +}; use serde::{Deserialize, Serialize}; +pub use ant_protocol::payment::payment_quote_hash; + const PAYMENT_MULTIPLIER: u128 = 3; +#[cfg(test)] const PRICE_BASELINE_WEI: u128 = 3_906_250_000_000_000; -const PRICE_COEFFICIENT_WEI: u128 = 35_156_250_000_000_000; -const PRICE_DIVISOR_SQUARED: u128 = 6_000 * 6_000; -const MAX_COMMITMENT_KEY_COUNT: u32 = 1_000_000; -const MAX_COMMITMENT_SIDECAR_BYTES: usize = 8 * 1024; -const DOMAIN_COMMITMENT: &[u8] = b"autonomi.ant.replication.storage_commitment.v1"; -const DOMAIN_COMMITMENT_HASH: &[u8] = b"autonomi.ant.replication.commitment_hash.v1"; /// JSON-safe form of a node's signed storage commitment. #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] @@ -77,21 +79,6 @@ pub struct VerifiedStorageQuote { #[error("invalid storage quote: {0}")] pub struct StorageQuoteError(pub String); -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -struct NativeStorageCommitment { - root: [u8; 32], - key_count: u32, - sender_peer_id: [u8; 32], - sender_public_key: Vec, - signature: Vec, -} - -/// Compute the EVM-facing `PaymentQuote` Keccak-256 hash. -#[must_use] -pub fn payment_quote_hash(signed_bytes: &[u8], public_key: &[u8], signature: &[u8]) -> [u8; 32] { - keccak256(&[signed_bytes, public_key, signature]) -} - /// Sum verified decimal quote amounts without exposing integer arithmetic to /// JavaScript or a wallet adapter. pub fn storage_payment_total(quotes: &[VerifiedStorageQuote]) -> Result { @@ -140,7 +127,7 @@ pub fn verify_storage_quote( )); } let price = parse_decimal_u128("e.price, "quote price")?; - let expected_price = calculate_price(quote.committed_key_count); + let expected_price = calculate_price_wei(quote.committed_key_count); if price != expected_price { return Err(StorageQuoteError( "storage quote price is not bound to its committed key count".to_string(), @@ -204,23 +191,19 @@ fn canonical_quote_bytes( rewards: &str, commitment_pin: Option<&str>, ) -> Result, StorageQuoteError> { - let content = - hex::decode("e.content).map_err(|error| StorageQuoteError(error.to_string()))?; - let rewards = hex::decode(rewards).map_err(|error| StorageQuoteError(error.to_string()))?; - let mut bytes = Vec::with_capacity(32 + 8 + 32 + 20 + 4 + 33); - bytes.extend_from_slice(&content); - bytes.extend_from_slice("e.timestamp_secs.to_le_bytes()); - bytes.extend_from_slice(&price.to_le_bytes()); - bytes.extend_from_slice(&[0u8; 16]); - bytes.extend_from_slice(&rewards); - bytes.extend_from_slice("e.committed_key_count.to_le_bytes()); - if let Some(pin) = commitment_pin { - bytes.push(1); - bytes.extend(hex::decode(pin).map_err(|error| StorageQuoteError(error.to_string()))?); - } else { - bytes.push(0); - } - Ok(bytes) + let content = decode_hex_array::<32>("e.content, "quote content")?; + let rewards = decode_hex_array::<20>(rewards, "quote rewards address")?; + let commitment_pin = commitment_pin + .map(|pin| decode_hex_array::<32>(pin, "storage commitment pin")) + .transpose()?; + Ok(payment_quote_bytes_for_signing( + &content, + quote.timestamp_secs, + price, + &rewards, + quote.committed_key_count, + commitment_pin.as_ref(), + )) } fn verify_commitment( @@ -235,7 +218,7 @@ fn verify_commitment( "storage commitment sidecar exceeds the protocol limit".to_string(), )); } - let commitment: NativeStorageCommitment = rmp_serde::from_slice(&encoded).map_err(|error| { + let commitment: StorageCommitment = rmp_serde::from_slice(&encoded).map_err(|error| { StorageQuoteError(format!( "storage commitment sidecar is not valid MessagePack: {error}" )) @@ -245,9 +228,10 @@ fn verify_commitment( let public_key = decode_unbounded_hex(&artifact.sender_public_key, "storage commitment public key")?; let signature = decode_unbounded_hex(&artifact.signature, "storage commitment signature")?; - if commitment.root != decode_array_32(&root)? + if commitment.root != decode_hex_array::<32>(&root, "storage commitment root")? || commitment.key_count != artifact.key_count - || commitment.sender_peer_id != decode_array_32(&peer_id)? + || commitment.sender_peer_id + != decode_hex_array::<32>(&peer_id, "storage commitment peer ID")? || commitment.sender_public_key != public_key || commitment.signature != signature { @@ -272,18 +256,12 @@ fn verify_commitment( "storage commitment belongs to a different peer".to_string(), )); } - let signed_payload = commitment_signed_payload(&commitment)?; - if !verify_ml_dsa_65(&public_key, &signature, &signed_payload, DOMAIN_COMMITMENT) { + if !verify_commitment_signature(&commitment) { return Err(StorageQuoteError( "storage commitment has an invalid ML-DSA-65 signature".to_string(), )); } - let postcard = - postcard::to_allocvec(&commitment).map_err(|error| StorageQuoteError(error.to_string()))?; - let mut hasher = blake3::Hasher::new(); - hasher.update(DOMAIN_COMMITMENT_HASH); - hasher.update(&postcard); - if hasher.finalize().to_hex().as_str() != expected_pin { + if commitment_hash(&commitment).map(hex::encode).as_deref() != Some(expected_pin) { return Err(StorageQuoteError( "storage commitment does not resolve the quote pin".to_string(), )); @@ -291,29 +269,6 @@ fn verify_commitment( Ok(()) } -fn commitment_signed_payload( - commitment: &NativeStorageCommitment, -) -> Result, StorageQuoteError> { - let key_length = u32::try_from(commitment.sender_public_key.len()) - .map_err(|_| StorageQuoteError("storage commitment public key is too large".to_string()))?; - let mut payload = Vec::with_capacity(32 + 4 + 32 + 4 + commitment.sender_public_key.len()); - payload.extend_from_slice(&commitment.root); - payload.extend_from_slice(&commitment.key_count.to_le_bytes()); - payload.extend_from_slice(&commitment.sender_peer_id); - payload.extend_from_slice(&key_length.to_le_bytes()); - payload.extend_from_slice(&commitment.sender_public_key); - Ok(payload) -} - -fn calculate_price(key_count: u32) -> u128 { - let count = u128::from(key_count); - PRICE_BASELINE_WEI - + count - .saturating_mul(count) - .saturating_mul(PRICE_COEFFICIENT_WEI) - / PRICE_DIVISOR_SQUARED -} - fn parse_decimal_u128(value: &str, label: &str) -> Result { if value.is_empty() || (value.len() > 1 && value.starts_with('0')) @@ -334,16 +289,23 @@ fn decode_unbounded_hex(value: &str, label: &str) -> Result, StorageQuot hex::decode(value).map_err(|error| StorageQuoteError(format!("invalid {label}: {error}"))) } -fn decode_array_32(value: &str) -> Result<[u8; 32], StorageQuoteError> { +fn decode_hex_array( + value: &str, + label: &str, +) -> Result<[u8; LENGTH], StorageQuoteError> { let decoded = hex::decode(value).map_err(|error| StorageQuoteError(error.to_string()))?; decoded.try_into().map_err(|bytes: Vec| { - StorageQuoteError(format!("expected 32 bytes, received {}", bytes.len())) + StorageQuoteError(format!( + "expected {LENGTH} bytes for {label}, received {}", + bytes.len() + )) }) } #[cfg(test)] mod tests { use super::*; + use ant_protocol::payment::{storage_commitment_bytes_for_signing, DOMAIN_COMMITMENT}; use ant_protocol::pqc::api::ml_dsa_65; fn baseline_quote() -> (BrowserQuoteArtifact, String, String) { @@ -387,26 +349,27 @@ mod tests { let (public_key, secret_key) = ml_dsa_65().generate_keypair().expect("keypair"); let public_key = public_key.to_bytes(); let peer_id = blake3::hash(&public_key).into(); - let mut commitment = NativeStorageCommitment { + let mut commitment = StorageCommitment { root, key_count, sender_peer_id: peer_id, sender_public_key: public_key.clone(), signature: Vec::new(), }; - let commitment_payload = commitment_signed_payload(&commitment).expect("payload"); + let commitment_payload = storage_commitment_bytes_for_signing( + &commitment.root, + commitment.key_count, + &commitment.sender_peer_id, + &commitment.sender_public_key, + ); commitment.signature = ml_dsa_65() .sign_with_context(&secret_key, &commitment_payload, DOMAIN_COMMITMENT) .expect("commitment signature") .to_bytes(); let encoded = rmp_serde::to_vec(&commitment).expect("MessagePack commitment"); - let postcard = postcard::to_allocvec(&commitment).expect("postcard commitment"); - let mut pin_hasher = blake3::Hasher::new(); - pin_hasher.update(DOMAIN_COMMITMENT_HASH); - pin_hasher.update(&postcard); - let pin = pin_hasher.finalize().to_hex().to_string(); + let pin = hex::encode(commitment_hash(&commitment).expect("commitment hash")); let peer_id = hex::encode(peer_id); - let price = calculate_price(key_count); + let price = calculate_price_wei(key_count); let mut quote = BrowserQuoteArtifact { peer_id: peer_id.clone(), content: hex::encode(content), @@ -464,7 +427,7 @@ mod tests { verify_storage_quote(quote.clone(), &content, &peer_id).expect("valid bound quote"); assert_eq!( storage_payment_total(&[verified]).expect("payment total"), - (calculate_price(23) * PAYMENT_MULTIPLIER).to_string() + (calculate_price_wei(23) * PAYMENT_MULTIPLIER).to_string() ); let mut tampered = quote; diff --git a/ant-core/src/browser/protocol.rs b/ant-core/src/browser/protocol.rs index 5daf8acc..56803d61 100644 --- a/ant-core/src/browser/protocol.rs +++ b/ant-core/src/browser/protocol.rs @@ -1,6 +1,6 @@ //! Browser-facing WebRTC Direct wire profile. -use super::crypto::verify_ml_dsa_65; +use ant_protocol::crypto::verify_ml_dsa_65; use base64::engine::general_purpose::URL_SAFE_NO_PAD; use base64::Engine as _; use serde::{Deserialize, Serialize}; diff --git a/ant-core/src/browser/wasm_transport.rs b/ant-core/src/browser/wasm_transport.rs index 98f52d55..f22aab71 100644 --- a/ant-core/src/browser/wasm_transport.rs +++ b/ant-core/src/browser/wasm_transport.rs @@ -339,9 +339,14 @@ impl Connection { let offer = JsFuture::from(connection.peer_connection.create_offer()) .await .map_err(js_error_message)?; - let offer: RtcSessionDescriptionInit = offer.dyn_into().map_err(js_error_message)?; - let offer_sdp = offer - .get_sdp() + // `RTCSessionDescriptionInit` is a Web IDL dictionary, not a branded + // interface. Chromium returns a plain object here, so `dyn_into` can + // reject a perfectly valid offer because there is no `instanceof` + // identity to test. Read the dictionary member structurally instead. + let offer_sdp = js_sys::Reflect::get(&offer, &JsValue::from_str("sdp")) + .map_err(js_error_message)? + .as_string() + .filter(|sdp| !sdp.is_empty()) .ok_or_else(|| "browser created an empty WebRTC offer".to_string())?; let munged_sdp = munge_offer_ice_credentials(&offer_sdp, &credential) .map_err(|error| error.to_string())?; diff --git a/web/README.md b/web/README.md index 4eb92d2a..b2be4a15 100644 --- a/web/README.md +++ b/web/README.md @@ -120,6 +120,20 @@ npm test npm run build ``` +Run the real-browser WebRTC Direct smoke test with: + +```bash +npm run test:browser +``` + +The first run downloads Playwright's pinned headless Chromium build. The test +then starts the sibling `ant-node-web-support` minimal devnet and local Anvil on +dedicated test ports, serves this application, loads its real manifest, and +requires an authenticated HELLO over a browser `RTCDataChannel`. Set +`ANT_NODE_DIR` when the patched node checkout is not at the default sibling +path. Set `ANT_WEBRTC_SMOKE_LOG=warn` (or another tracing level) to include +native devnet diagnostics while troubleshooting a failure. + Both commands build the WASM package automatically. To validate the Rust boundary directly: @@ -157,12 +171,11 @@ contract calls. This is also the deliberate extension seam: another web app can provide its own UI and wallet callback while sharing all network and Autonomi logic from the Rust library. -`ant-protocol 2.3.1` still couples its wire types to native Saorsa/Tokio -networking, so it cannot be linked into this WASM build. The browser verifier -therefore uses the same FIPS-204 ML-DSA-65 primitive directly. Splitting a -transport-free wire/crypto feature from `ant-protocol` would remove that final -dependency-level duplication and let native and WASM builds import the exact -same protocol types. +The patched `ant-protocol 2.3.1` exposes a `portable` feature that omits Tokio, +EVM, Saorsa transport, and `saorsa-pqc`. Both native and WASM clients now import +the same storage commitment type, signing encodings, quote hash, price curve, +and ML-DSA-65 verifier from `ant-protocol`; only that crate selects the native +or FIPS-204 verification backend. The local testnet manifest is intentionally unsigned bootstrap material. A production deployment still needs ML-DSA-signed endpoint records, exceptional diff --git a/web/e2e/webrtc-direct.spec.js b/web/e2e/webrtc-direct.spec.js new file mode 100644 index 00000000..ada8e2b4 --- /dev/null +++ b/web/e2e/webrtc-direct.spec.js @@ -0,0 +1,28 @@ +import { expect, test } from "@playwright/test"; + +test("authenticates a real ant-node over WebRTC Direct", async ({ page }) => { + const applicationErrors = []; + page.on("console", (message) => { + if (message.type() === "error") applicationErrors.push(message.text()); + }); + + const manifestUrl = "http://127.0.0.1:35000/api/browser-manifest.json"; + await page.goto(`/?manifest=${encodeURIComponent(manifestUrl)}`); + await expect(page.locator("#manifest-state")).toContainText("direct nodes", { + timeout: 120_000, + }); + await expect(page.locator("#endpoint-multiaddr")).toHaveValue( + /\/webrtc-direct\/certhash\/.+\/p2p\//, + ); + + await page.getByRole("button", { name: "Connect", exact: true }).click(); + + try { + await expect(page.locator("#connection-state")).toContainText("Connected"); + } catch (error) { + const protocolLog = await page.locator("#log").textContent(); + throw new Error(`${error.message}\n\nBrowser protocol log:\n${protocolLog}`); + } + await expect(page.locator("#log")).toContainText("HELLO"); + expect(applicationErrors).toEqual([]); +}); diff --git a/web/package-lock.json b/web/package-lock.json index 68df99a4..6860ec2e 100644 --- a/web/package-lock.json +++ b/web/package-lock.json @@ -11,6 +11,7 @@ "ethers": "^6.17.0" }, "devDependencies": { + "@playwright/test": "^1.62.1", "vite": "8.2.0" } }, @@ -90,6 +91,22 @@ "url": "https://github.com/sponsors/Boshen" } }, + "node_modules/@playwright/test": { + "version": "1.62.1", + "resolved": "https://registry.npmjs.org/@playwright/test/-/test-1.62.1.tgz", + "integrity": "sha512-DTcUc8qii+cpHvtOwggMtBRMjKZHXYWdw8syRYu2vtzuq4Wxphqq4NfCs5Zt44L6mA8rfDfj+PHnxFc/FeK6mQ==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "playwright": "1.62.1" + }, + "bin": { + "playwright": "cli.js" + }, + "engines": { + "node": ">=20" + } + }, "node_modules/@rolldown/binding-android-arm64": { "version": "1.2.2", "resolved": "https://registry.npmjs.org/@rolldown/binding-android-arm64/-/binding-android-arm64-1.2.2.tgz", @@ -787,6 +804,53 @@ "url": "https://github.com/sponsors/jonschlinkert" } }, + "node_modules/playwright": { + "version": "1.62.1", + "resolved": "https://registry.npmjs.org/playwright/-/playwright-1.62.1.tgz", + "integrity": "sha512-0M+L3LAD8/nm554LOla9Ayx0j0tmFZ0FBcoQ7F1VuVHpM/XpiC8RcDzBQB8W5+hA8L22THxELzeF+2WcUzvcLg==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "playwright-core": "1.62.1" + }, + "bin": { + "playwright": "cli.js" + }, + "engines": { + "node": ">=20" + }, + "optionalDependencies": { + "fsevents": "2.3.2" + } + }, + "node_modules/playwright-core": { + "version": "1.62.1", + "resolved": "https://registry.npmjs.org/playwright-core/-/playwright-core-1.62.1.tgz", + "integrity": "sha512-wPYSwEBJY9GHraISXqyqtx0na0LpO3XEX7jNDhntbex7tzUS7kLnZsOlFruFJB4Hi/rhDMjXGqHewDZ68nYZVw==", + "dev": true, + "license": "Apache-2.0", + "bin": { + "playwright-core": "cli.js" + }, + "engines": { + "node": ">=20" + } + }, + "node_modules/playwright/node_modules/fsevents": { + "version": "2.3.2", + "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.2.tgz", + "integrity": "sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA==", + "dev": true, + "hasInstallScript": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^8.16.0 || ^10.6.0 || >=11.0.0" + } + }, "node_modules/postcss": { "version": "8.5.25", "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.5.25.tgz", diff --git a/web/package.json b/web/package.json index 827ec5be..121770ca 100644 --- a/web/package.json +++ b/web/package.json @@ -11,12 +11,16 @@ "prebuild": "npm run wasm:release", "build": "vite build", "pretest": "npm run wasm:dev", - "test": "node --import ./setup-wasm.js --test" + "test": "node --import ./setup-wasm.js --test", + "test:browser:install": "playwright install chromium", + "pretest:browser": "playwright install chromium", + "test:browser": "playwright test" }, "dependencies": { "ethers": "^6.17.0" }, "devDependencies": { + "@playwright/test": "^1.62.1", "vite": "8.2.0" } } diff --git a/web/playwright.config.js b/web/playwright.config.js new file mode 100644 index 00000000..d1c20d1f --- /dev/null +++ b/web/playwright.config.js @@ -0,0 +1,69 @@ +import { defineConfig } from "@playwright/test"; +import { dirname, resolve } from "node:path"; +import { fileURLToPath } from "node:url"; + +const webDirectory = dirname(fileURLToPath(import.meta.url)); +const antNodeDirectory = process.env.ANT_NODE_DIR + ? resolve(process.env.ANT_NODE_DIR) + : resolve(webDirectory, "../../ant-node-web-support"); +const manifestUrl = "http://127.0.0.1:35000/api/browser-manifest.json"; +const devnetLogLevel = process.env.ANT_WEBRTC_SMOKE_LOG; +if ( + devnetLogLevel && + !["error", "warn", "info", "debug", "trace"].includes(devnetLogLevel) +) { + throw new Error("ANT_WEBRTC_SMOKE_LOG must be error, warn, info, debug, or trace"); +} + +// JSON string quoting is accepted by the shells used by Cargo's supported +// desktop platforms and keeps workspace paths containing spaces intact. +const nodeManifest = JSON.stringify(resolve(antNodeDirectory, "Cargo.toml")); +const devnetData = JSON.stringify(resolve(webDirectory, ".playwright-devnet")); +const devnetCommand = [ + "cargo run", + `--manifest-path ${nodeManifest}`, + "--features webrtc-direct", + "--bin ant-devnet --", + "--preset minimal", + `--data-dir ${devnetData}`, + "--base-port 33000", + "--webrtc-direct", + "--webrtc-direct-base-port 34000", + "--serve-port 35000", + "--enable-evm", + ...(devnetLogLevel + ? ["--enable-logging", `--log-level ${devnetLogLevel}`] + : []), +].join(" "); + +export default defineConfig({ + testDir: "./e2e", + fullyParallel: false, + workers: 1, + timeout: 60_000, + expect: { timeout: 30_000 }, + reporter: "line", + use: { + baseURL: "http://127.0.0.1:35173", + browserName: "chromium", + headless: true, + }, + webServer: [ + { + command: devnetCommand, + url: manifestUrl, + timeout: 300_000, + reuseExistingServer: false, + stdout: "pipe", + stderr: "pipe", + }, + { + command: "npm run dev -- --port 35173", + url: "http://127.0.0.1:35173", + timeout: 180_000, + reuseExistingServer: false, + stdout: "pipe", + stderr: "pipe", + }, + ], +}); diff --git a/web/src/main.js b/web/src/main.js index 5bff2a2a..4d98ffef 100644 --- a/web/src/main.js +++ b/web/src/main.js @@ -40,6 +40,11 @@ const elements = { log: document.querySelector("#log"), }; +const manifestOverride = new URLSearchParams(window.location.search).get( + "manifest", +); +if (manifestOverride) elements.manifestUrl.value = manifestOverride; + let client; let networkClient; let browserManifest; From e89f5616ac85068e5741f72483dd18cfda6929bc Mon Sep 17 00:00:00 2001 From: Mick van Dijke <12992260+mickvandijke@users.noreply.github.com> Date: Wed, 26 Aug 2026 17:19:07 +0200 Subject: [PATCH 09/31] feat(web): support large nested DataMap files --- ant-core/src/browser.rs | 127 +++++++++++------- ant-core/src/browser/wasm_transport.rs | 8 +- .../ADR-0003-direct-browser-read-client.md | 5 +- web/README.md | 9 +- web/e2e/webrtc-direct.spec.js | 54 +++++++- web/src/main.js | 21 ++- web/src/wasm.test.js | 25 +++- 7 files changed, 187 insertions(+), 62 deletions(-) diff --git a/ant-core/src/browser.rs b/ant-core/src/browser.rs index 68acadd1..b6b8bd3a 100644 --- a/ant-core/src/browser.rs +++ b/ant-core/src/browser.rs @@ -28,9 +28,14 @@ mod wasm_transport; use bytes::Bytes; use self_encryption::{DataMap, EncryptedChunk}; use serde::{Deserialize, Serialize}; +use std::collections::HashSet; -/// Maximum file size accepted by the in-memory browser demo. -pub const MAX_BROWSER_FILE_BYTES: usize = 64 * 1024 * 1024; +/// Maximum file size accepted by the in-memory browser demo (1 GB decimal). +/// +/// The current browser path retains the complete plaintext and encrypted +/// records in memory, so reaching this protocol limit still depends on the +/// browser's available memory. +pub const MAX_BROWSER_FILE_BYTES: usize = 1_000_000_000; /// One native self-encryption chunk descriptor exposed to the browser. #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] @@ -126,32 +131,39 @@ pub fn encrypt_public_file(content: &[u8]) -> Result>(); + let mut get_local_chunk = |address: self_encryption::XorName| { + encrypted_by_address + .get(&address.0) + .map(|content| (*content).clone()) + .ok_or_else(|| { + self_encryption::Error::Generic(format!( + "self-encryption output omitted DataMap chunk {}", + hex::encode(address.0) + )) + }) + }; + self_encryption::get_root_data_map(published_data_map.clone(), &mut get_local_chunk) + .map_err(|error| BrowserError::SelfEncryption(error.to_string()))? + }; + let chunks = chunk_infos(&root_data_map); let mut records: Vec = encrypted_chunks .into_iter() - .zip(&chunks) - .map(|(chunk, info)| BrowserRecord { - address: info.dst_hash.clone(), + .map(|chunk| BrowserRecord { + address: content_address(&chunk.content), content: chunk.content.to_vec(), }) .collect(); - let encoded_data_map = - rmp_serde::to_vec(&data_map).map_err(|error| BrowserError::DataMap(error.to_string()))?; + let encoded_data_map = rmp_serde::to_vec(&published_data_map) + .map_err(|error| BrowserError::DataMap(error.to_string()))?; let address = content_address(&encoded_data_map); let data_map_size = encoded_data_map.len(); records.push(BrowserRecord { @@ -172,11 +184,6 @@ pub fn encrypt_public_file(content: &[u8]) -> Result Result, BrowserError> { let data_map: DataMap = rmp_serde::from_slice(content).map_err(|error| BrowserError::DataMap(error.to_string()))?; - if data_map.is_child() { - return Err(BrowserError::Invalid( - "nested DataMaps are not supported by the browser client".to_string(), - )); - } Ok(chunk_infos(&data_map)) } @@ -188,30 +195,24 @@ pub fn decrypt_public_file( ) -> Result, BrowserError> { let data_map: DataMap = rmp_serde::from_slice(data_map_content) .map_err(|error| BrowserError::DataMap(error.to_string()))?; - if data_map.is_child() { - return Err(BrowserError::Invalid( - "nested DataMaps are not supported by the browser client".to_string(), - )); - } - if encrypted_contents.len() != data_map.infos().len() { - return Err(BrowserError::Invalid(format!( - "received {} encrypted records for {} DataMap entries", - encrypted_contents.len(), - data_map.infos().len() - ))); + let available = encrypted_contents + .iter() + .map(|content| *blake3::hash(content).as_bytes()) + .collect::>(); + for info in data_map.infos() { + if !available.contains(&info.dst_hash.0) { + return Err(BrowserError::Invalid(format!( + "record set does not contain DataMap chunk {}; a record may be missing or corrupt", + hex::encode(info.dst_hash.0) + ))); + } } - - let encrypted_chunks = data_map - .infos() + let encrypted_chunks = encrypted_contents .iter() - .zip(encrypted_contents) - .map(|(info, content)| { - verify_record(&hex::encode(info.dst_hash.0), content)?; - Ok(EncryptedChunk { - content: Bytes::copy_from_slice(content), - }) + .map(|content| EncryptedChunk { + content: Bytes::copy_from_slice(content), }) - .collect::, BrowserError>>()?; + .collect::>(); self_encryption::decrypt(&data_map, &encrypted_chunks) .map(|bytes| bytes.to_vec()) .map_err(|error| BrowserError::SelfEncryption(error.to_string())) @@ -743,4 +744,34 @@ mod tests { tampered[0][0] ^= 1; assert!(decrypt_public_file(data_map, &tampered).is_err()); } + + #[test] + fn nested_data_map_round_trip() { + let size = 3 * self_encryption::MAX_CHUNK_SIZE + 1; + let content = (0..size).map(|index| index as u8).collect::>(); + let encrypted = encrypt_public_file(&content).expect("encrypt nested fixture"); + assert_eq!(encrypted.chunks.len(), 4); + assert!(encrypted.records.len() > encrypted.chunks.len() + 1); + + let data_map = &encrypted.records.last().expect("DataMap record").content; + let published: DataMap = rmp_serde::from_slice(data_map).expect("decode published map"); + assert!(published.is_child()); + + let mut required_addresses = decode_public_data_map(data_map) + .expect("decode child map") + .into_iter() + .map(|chunk| chunk.dst_hash) + .collect::>(); + required_addresses.extend(encrypted.chunks.iter().map(|chunk| chunk.dst_hash.clone())); + let records = encrypted.records[..encrypted.records.len() - 1] + .iter() + .filter(|record| required_addresses.contains(&record.address)) + .map(|record| record.content.clone()) + .collect::>(); + assert_eq!(records.len(), encrypted.records.len() - 1); + assert_eq!( + decrypt_public_file(data_map, &records).expect("decrypt nested fixture"), + content + ); + } } diff --git a/ant-core/src/browser/wasm_transport.rs b/ant-core/src/browser/wasm_transport.rs index f22aab71..0c378dfe 100644 --- a/ant-core/src/browser/wasm_transport.rs +++ b/ant-core/src/browser/wasm_transport.rs @@ -1071,7 +1071,11 @@ impl BrowserNetworkClient { "Verified public DataMap ({} bytes)", data_map.len() )); - let chunks = super::decode_public_data_map(&data_map).map_err(|error| error.to_string())?; + let mut chunks = + super::decode_public_data_map(&data_map).map_err(|error| error.to_string())?; + chunks.extend(file.chunks.iter().cloned()); + let mut seen = std::collections::HashSet::new(); + chunks.retain(|chunk| seen.insert(chunk.dst_hash.clone())); if chunks.len() < 3 { return Err("ant-core returned an invalid public DataMap".to_string()); } @@ -1083,7 +1087,7 @@ impl BrowserNetworkClient { async move { progress.report(&format!( "Fetching encrypted file chunk {}/{} ({})", - chunk.index + 1, + position + 1, total, chunk.dst_hash )); diff --git a/docs/adr/ADR-0003-direct-browser-read-client.md b/docs/adr/ADR-0003-direct-browser-read-client.md index 32e2bceb..5f445184 100644 --- a/docs/adr/ADR-0003-direct-browser-read-client.md +++ b/docs/adr/ADR-0003-direct-browser-read-client.md @@ -167,8 +167,9 @@ metadata chain. ### Negative / Trade-offs - The current client reconstructs files in memory and the local launcher caps - public files at 64 MiB; upload encryption and reconstruction are not yet - streaming. + public files at 1 GB (1,000,000,000 bytes); upload encryption and + reconstruction are not yet streaming, so the practical limit depends on + available browser memory. - JavaScript quote-verification behavior must remain aligned with native protocol rules until transport-free `ant-protocol` APIs exist. - The initial WASM module is approximately 1.4 MiB uncompressed and browser diff --git a/web/README.md b/web/README.md index b2be4a15..36341dde 100644 --- a/web/README.md +++ b/web/README.md @@ -70,7 +70,9 @@ Confirm that `HELLO.payment.rpc_url` is a loopback Anvil URL. An Pass `--public-file /path/to/file` to publish another file instead. The built-in file is generated as 5 MiB so the demo exercises whole-file reconstruction. A -custom file may be up to 64 MiB in this local launcher. +custom file may be up to 1 GB (1,000,000,000 bytes) in this local launcher. +Uploads and downloads are currently processed in memory, so large files still +depend on the browser having sufficient available memory. ## Run the site @@ -150,8 +152,9 @@ quote verification including the native Keccak-256 EVM quote hash, and a Rust verification. Cross-repository live verification additionally starts the node testnet, downloads all public-file records through WebRTC Direct, reconstructs the -original bytes, pays a real quote on local Anvil, uploads a fresh record -through the ordinary node payment validator, and reads it back. +original bytes, pays real quotes on local Anvil, uploads an incompressible file +large enough to require a nested DataMap through the ordinary node payment +validator, and downloads and verifies it again. ## Library boundary diff --git a/web/e2e/webrtc-direct.spec.js b/web/e2e/webrtc-direct.spec.js index ada8e2b4..a7cf3ef1 100644 --- a/web/e2e/webrtc-direct.spec.js +++ b/web/e2e/webrtc-direct.spec.js @@ -1,10 +1,19 @@ import { expect, test } from "@playwright/test"; -test("authenticates a real ant-node over WebRTC Direct", async ({ page }) => { +test("uploads and downloads a nested-DataMap file over WebRTC Direct", async ({ + page, +}) => { + test.setTimeout(180_000); const applicationErrors = []; page.on("console", (message) => { if (message.type() === "error") applicationErrors.push(message.text()); }); + await page.addInitScript(() => { + Object.defineProperty(globalThis, "showSaveFilePicker", { + value: undefined, + configurable: true, + }); + }); const manifestUrl = "http://127.0.0.1:35000/api/browser-manifest.json"; await page.goto(`/?manifest=${encodeURIComponent(manifestUrl)}`); @@ -24,5 +33,48 @@ test("authenticates a real ant-node over WebRTC Direct", async ({ page }) => { throw new Error(`${error.message}\n\nBrowser protocol log:\n${protocolLog}`); } await expect(page.locator("#log")).toContainText("HELLO"); + + const maxChunkSize = 4_190_208; + const content = Buffer.allocUnsafe(3 * maxChunkSize + 1); + let state = 0x9e3779b9; + for (let index = 0; index < content.length; index += 1) { + state ^= state << 13; + state ^= state >>> 17; + state ^= state << 5; + content[index] = state; + } + await page.locator("#upload-file-input").setInputFiles({ + name: "nested-datamap.bin", + mimeType: "application/octet-stream", + buffer: content, + }); + await page + .locator("#wallet-secret") + .fill("0xac0974bec39a17e36ba4a6b4d238ff944bacb478cbed5efcae784d7bf4f2ff80"); + await page.getByRole("button", { name: "Pay and upload file" }).click(); + + try { + await expect(page.locator("#upload-state")).toContainText("Uploaded", { + timeout: 150_000, + }); + } catch (error) { + const protocolLog = await page.locator("#log").textContent(); + throw new Error(`${error.message}\n\nBrowser protocol log:\n${protocolLog}`); + } + await expect(page.locator("#upload-result-records")).toContainText("8 records"); + + const downloadStarted = page.waitForEvent("download"); + await page.getByRole("button", { name: "Download and save file" }).click(); + try { + await expect(page.locator("#download-state")).toContainText( + "Browser download started", + { timeout: 120_000 }, + ); + } catch (error) { + const protocolLog = await page.locator("#log").textContent(); + throw new Error(`${error.message}\n\nBrowser protocol log:\n${protocolLog}`); + } + const download = await downloadStarted; + expect(download.suggestedFilename()).toBe("nested-datamap.bin"); expect(applicationErrors).toEqual([]); }); diff --git a/web/src/main.js b/web/src/main.js index 4d98ffef..c2b0c844 100644 --- a/web/src/main.js +++ b/web/src/main.js @@ -61,6 +61,17 @@ function log(message, value) { elements.log.scrollTop = elements.log.scrollHeight; } +function errorMessage(error) { + if (error instanceof Error) return error.message; + if (typeof error === "string") return error; + if (error === undefined) return "unknown error"; + try { + return JSON.stringify(error); + } catch { + return String(error); + } +} + function endpointFromForm() { return elements.endpointMultiaddr.value.trim(); } @@ -113,7 +124,7 @@ async function connectedClient() { function reportError(context, error) { elements.connectionState.textContent = `${context} failed`; - log(`${context} failed: ${error.message}`); + log(`${context} failed: ${errorMessage(error)}`); console.error(error); } @@ -127,7 +138,7 @@ elements.loadManifest.addEventListener("click", async () => { await loadManifest(); } catch (error) { elements.manifestState.textContent = "Load failed"; - log(`Manifest load failed: ${error.message}`); + log(`Manifest load failed: ${errorMessage(error)}`); console.error(error); } }); @@ -210,7 +221,7 @@ elements.uploadFile.addEventListener("click", async () => { ); } catch (error) { elements.uploadState.textContent = "Upload failed"; - log(`File upload failed: ${error.message}`); + log(`File upload failed: ${errorMessage(error)}`); console.error(error); } finally { walletSecret = ""; @@ -258,7 +269,7 @@ elements.downloadFile.addEventListener("click", async () => { } catch (error) { elements.downloadState.textContent = error.name === "AbortError" ? "Save cancelled" : "Failed"; - log(`File download failed: ${error.message}`); + log(`File download failed: ${errorMessage(error)}`); console.error(error); } finally { elements.downloadFile.disabled = false; @@ -331,5 +342,5 @@ elements.randomTarget.click(); log("Ready. Loading the local browser testnet manifest…"); loadManifest().catch((error) => { elements.manifestState.textContent = "Not running"; - log(`Local manifest not available yet: ${error.message}`); + log(`Local manifest not available yet: ${errorMessage(error)}`); }); diff --git a/web/src/wasm.test.js b/web/src/wasm.test.js index 8aa261f2..fd9de5ac 100644 --- a/web/src/wasm.test.js +++ b/web/src/wasm.test.js @@ -97,5 +97,28 @@ test("generated ant-core WASM matches the native self-encryption vector", () => const tampered = chunks.map((chunk) => chunk.slice()); tampered[0][0] ^= 1; - assert.throws(() => decryptPublicFile(dataMap, tampered), /BLAKE3 mismatch/); + assert.throws( + () => decryptPublicFile(dataMap, tampered), + /record may be missing or corrupt/, + ); +}); + +test("generated ant-core WASM supports nested DataMaps", () => { + const maxChunkSize = 4_190_208; + const content = new Uint8Array(3 * maxChunkSize + 1); + for (let index = 0; index < content.length; index += 1) { + content[index] = index; + } + + const encrypted = encryptPublicFile(content); + assert.equal(encrypted.chunks.length, 4); + assert.equal(decodePublicDataMap(encrypted.records.at(-1).content).length, 3); + assert(encrypted.records.length > encrypted.chunks.length + 1); + + const decrypted = decryptPublicFile( + encrypted.records.at(-1).content, + encrypted.records.slice(0, -1).map((record) => record.content), + ); + assert.equal(decrypted.length, content.length); + assert.equal(verifyRecord(encrypted.blake3, decrypted), encrypted.blake3); }); From b42fa6cb73c1c9297812ae98233f27fa2847c658 Mon Sep 17 00:00:00 2001 From: Mick van Dijke <12992260+mickvandijke@users.noreply.github.com> Date: Thu, 27 Aug 2026 17:52:27 +0200 Subject: [PATCH 10/31] feat(web): stream files through reusable WebRTC sessions --- ant-core/src/browser/wasm_transport.rs | 368 +++++++++++++++++- .../ADR-0003-direct-browser-read-client.md | 41 +- web/README.md | 54 ++- web/e2e/webrtc-direct.spec.js | 105 ++++- web/index.html | 23 +- web/src/main.js | 181 ++++++++- web/src/style.css | 13 + 7 files changed, 743 insertions(+), 42 deletions(-) diff --git a/ant-core/src/browser/wasm_transport.rs b/ant-core/src/browser/wasm_transport.rs index 0c378dfe..0b9861e0 100644 --- a/ant-core/src/browser/wasm_transport.rs +++ b/ant-core/src/browser/wasm_transport.rs @@ -28,7 +28,7 @@ use saorsa_dht_lookup::{ use serde::{Deserialize, Serialize}; use serde_json::{Map, Value}; use std::cell::{Cell, RefCell}; -use std::collections::HashMap; +use std::collections::{HashMap, HashSet}; use std::future::Future; use std::ops::Deref; use std::rc::Rc; @@ -49,6 +49,8 @@ const DEFAULT_LOOKUP_ALPHA: usize = 3; const DEFAULT_MAX_LOOKUP_ITERATIONS: usize = 20; const MAX_STORE_TARGETS: usize = 7; const MAX_DOWNLOAD_CONCURRENCY: usize = 6; +const MAX_BROWSER_RANGE_BYTES: usize = 4 * 1024 * 1024; +const MAX_RANGE_CACHE_BYTES: usize = 32 * 1024 * 1024; type ResponseInbox = Rc, String>>>>; @@ -945,6 +947,239 @@ struct BrowserUploadResult { records: usize, } +struct CachedRangeRecord { + content: bytes::Bytes, + last_used: u64, +} + +#[derive(Default)] +struct BrowserRangeCache { + entries: HashMap<[u8; 32], CachedRangeRecord>, + total_bytes: usize, + clock: u64, +} + +impl BrowserRangeCache { + fn contains(&self, address: &[u8; 32]) -> bool { + self.entries.contains_key(address) + } + + fn get(&mut self, address: &[u8; 32]) -> Option { + self.clock = self.clock.wrapping_add(1); + let entry = self.entries.get_mut(address)?; + entry.last_used = self.clock; + Some(entry.content.clone()) + } + + fn insert(&mut self, address: [u8; 32], content: Vec) { + self.clock = self.clock.wrapping_add(1); + let content = bytes::Bytes::from(content); + if let Some(previous) = self.entries.remove(&address) { + self.total_bytes = self.total_bytes.saturating_sub(previous.content.len()); + } + self.total_bytes = self.total_bytes.saturating_add(content.len()); + self.entries.insert( + address, + CachedRangeRecord { + content, + last_used: self.clock, + }, + ); + while self.total_bytes > MAX_RANGE_CACHE_BYTES && self.entries.len() > 1 { + let Some(oldest) = self + .entries + .iter() + .min_by_key(|(_, entry)| entry.last_used) + .map(|(address, _)| *address) + else { + break; + }; + if let Some(removed) = self.entries.remove(&oldest) { + self.total_bytes = self.total_bytes.saturating_sub(removed.content.len()); + } + } + } + + fn clear(&mut self) { + self.entries.clear(); + self.total_bytes = 0; + } +} + +/// Random-access public-file reader for media playback and bounded downloads. +#[wasm_bindgen(js_name = BrowserFileReader)] +pub struct BrowserFileReader { + inner: Rc, + file: PublicFileDescriptor, + root_data_map: self_encryption::DataMap, + cache: RefCell, + progress: ProgressReporter, + closed: Cell, +} + +#[wasm_bindgen(js_class = BrowserFileReader)] +impl BrowserFileReader { + /// Plaintext file size in bytes. + #[wasm_bindgen(getter)] + pub fn size(&self) -> usize { + self.file.size + } + + /// Browser MIME type advertised by the file descriptor. + #[wasm_bindgen(getter, js_name = contentType)] + pub fn content_type(&self) -> String { + self.file.content_type.clone() + } + + /// Display filename advertised by the file descriptor. + #[wasm_bindgen(getter)] + pub fn name(&self) -> String { + self.file.name.clone() + } + + /// Fetch and decrypt one plaintext byte range without reconstructing the file. + #[wasm_bindgen(js_name = readRange)] + pub async fn read_range(&self, start: usize, length: usize) -> Result { + let content = self + .read_range_inner(start, length) + .await + .map_err(|error| JsValue::from_str(&error))?; + Ok(Uint8Array::from(content.as_slice())) + } + + /// Release cached encrypted records held for playback read-ahead and seeks. + pub fn close(&self) { + self.closed.set(true); + self.cache.borrow_mut().clear(); + } +} + +impl BrowserFileReader { + async fn read_range_inner(&self, start: usize, length: usize) -> Result, String> { + if self.closed.get() { + return Err("browser file reader is closed".to_string()); + } + if length > MAX_BROWSER_RANGE_BYTES { + return Err(format!( + "browser range reads are limited to {MAX_BROWSER_RANGE_BYTES} bytes" + )); + } + if length == 0 || start >= self.file.size { + return Ok(Vec::new()); + } + let end = start.saturating_add(length).min(self.file.size); + let required = required_range_records(&self.root_data_map, start, end)?; + if required.is_empty() { + return Err("DataMap contains no records for the requested range".to_string()); + } + + let missing = { + let cache = self.cache.borrow(); + required + .iter() + .filter(|(_, address)| !cache.contains(address)) + .copied() + .collect::>() + }; + if !missing.is_empty() { + let downloads = stream::iter(missing) + .map(|(index, address)| { + let inner = Rc::clone(&self.inner); + let progress = self.progress.clone(); + async move { + let encoded = hex::encode(address); + progress.report(&format!( + "Streaming encrypted chunk {} ({encoded})", + index + 1 + )); + inner + .get_chunk_from_closest(&encoded, &progress) + .await + .map(|(content, _)| (address, content)) + } + }) + .buffer_unordered(MAX_DOWNLOAD_CONCURRENCY) + .collect::>() + .await; + let mut cache = self.cache.borrow_mut(); + for download in downloads { + let (address, content) = download?; + cache.insert(address, content); + } + } + + let available = { + let mut cache = self.cache.borrow_mut(); + required + .iter() + .map(|(_, address)| { + cache + .get(address) + .map(|content| (*address, content)) + .ok_or_else(|| { + format!("streaming cache omitted record {}", hex::encode(address)) + }) + }) + .collect::, _>>()? + }; + let fetch_cached = |requested: &[(usize, self_encryption::XorName)]| { + requested + .iter() + .map(|(index, address)| { + available + .get(&address.0) + .cloned() + .map(|content| (*index, content)) + .ok_or_else(|| { + self_encryption::Error::Generic(format!( + "streaming range omitted record {}", + hex::encode(address.0) + )) + }) + }) + .collect::, _>>() + }; + let stream = self_encryption::streaming_decrypt_with_batch_size( + &self.root_data_map, + fetch_cached, + required.len(), + ) + .map_err(|error| format!("could not initialize range decryption: {error}"))?; + let plaintext = stream + .get_range(start, end - start) + .map_err(|error| format!("could not decrypt requested range: {error}"))?; + if plaintext.len() != end - start { + return Err(format!( + "range decryption returned {} bytes, expected {}", + plaintext.len(), + end - start + )); + } + Ok(plaintext.to_vec()) + } +} + +fn required_range_records( + data_map: &self_encryption::DataMap, + start: usize, + end: usize, +) -> Result, String> { + let mut infos = data_map.infos().to_vec(); + infos.sort_by_key(|info| info.index); + let mut cursor = 0usize; + let mut required = Vec::new(); + for info in infos { + let chunk_end = cursor + .checked_add(info.src_size) + .ok_or_else(|| "DataMap plaintext size overflow".to_string())?; + if cursor < end && chunk_end > start { + required.push((info.index, info.dst_hash.0)); + } + cursor = chunk_end; + } + Ok(required) +} + /// Stateful Autonomi browser client sharing Rust lookup and data workflows. #[wasm_bindgen(js_name = BrowserNetworkClient)] pub struct BrowserNetworkClient { @@ -1005,6 +1240,21 @@ impl BrowserNetworkClient { serde_wasm_bindgen::to_value(&result).map_err(|error| JsValue::from_str(&error.to_string())) } + /// Resolve and validate a public file for random-access range reads. + #[wasm_bindgen(js_name = openPublicFile)] + pub async fn open_public_file( + &self, + file: JsValue, + on_progress: Option, + ) -> Result { + let file: PublicFileDescriptor = serde_wasm_bindgen::from_value(file) + .map_err(|error| JsValue::from_str(&error.to_string()))?; + let progress = ProgressReporter::from_js(on_progress); + self.open_public_file_inner(file, progress) + .await + .map_err(|error| JsValue::from_str(&error)) + } + /// Self-encrypt, quote, pay through a wallet callback, and store a public file. #[wasm_bindgen(js_name = uploadPublicFile)] pub async fn upload_public_file( @@ -1043,6 +1293,120 @@ impl BrowserNetworkClient { } impl BrowserNetworkClient { + async fn open_public_file_inner( + &self, + mut file: PublicFileDescriptor, + progress: ProgressReporter, + ) -> Result { + file.address = super::protocol::normalize_hex(&file.address, 32)?; + file.blake3 = super::protocol::normalize_hex(&file.blake3, 32)?; + if file.name.is_empty() { + return Err("public file has no name".to_string()); + } + if file.size == 0 || file.size > super::MAX_BROWSER_FILE_BYTES { + return Err(format!("invalid public file size {}", file.size)); + } + progress.report(&format!( + "Opening {} for random-access streaming", + file.name + )); + let (encoded_data_map, _) = self + .inner + .get_chunk_from_closest(&file.address, &progress) + .await?; + if encoded_data_map.len() != file.data_map_size { + return Err(format!( + "public DataMap has {} bytes, expected {}", + encoded_data_map.len(), + file.data_map_size + )); + } + let published_data_map: self_encryption::DataMap = rmp_serde::from_slice(&encoded_data_map) + .map_err(|error| format!("could not decode public DataMap: {error}"))?; + let root_data_map = if published_data_map.is_child() { + let child_infos = published_data_map.infos().to_vec(); + let downloads = stream::iter(child_infos.iter().cloned()) + .map(|info| { + let inner = Rc::clone(&self.inner); + let progress = progress.clone(); + async move { + let address = hex::encode(info.dst_hash.0); + progress.report(&format!( + "Resolving nested DataMap record {}", + info.index + 1 + )); + inner + .get_chunk_from_closest(&address, &progress) + .await + .map(|(content, _)| (info.dst_hash.0, bytes::Bytes::from(content))) + } + }) + .buffer_unordered(MAX_DOWNLOAD_CONCURRENCY) + .collect::>() + .await; + let mut child_records = HashMap::with_capacity(downloads.len()); + for download in downloads { + let (address, content) = download?; + child_records.insert(address, content); + } + let mut get_child = |address: self_encryption::XorName| { + child_records.get(&address.0).cloned().ok_or_else(|| { + self_encryption::Error::Generic(format!( + "nested DataMap resolution requested unavailable record {}", + hex::encode(address.0) + )) + }) + }; + self_encryption::get_root_data_map(published_data_map, &mut get_child) + .map_err(|error| format!("could not resolve root DataMap: {error}"))? + } else { + published_data_map + }; + + let actual_chunks = super::chunk_infos(&root_data_map); + let mut expected_chunks = file + .chunks + .iter() + .map(|chunk| super::BrowserChunkInfo { + index: chunk.index, + dst_hash: chunk.dst_hash.to_ascii_lowercase(), + src_hash: chunk.src_hash.to_ascii_lowercase(), + src_size: chunk.src_size, + }) + .collect::>(); + expected_chunks.sort_by_key(|chunk| chunk.index); + if actual_chunks != expected_chunks { + return Err( + "resolved root DataMap does not match the public file descriptor".to_string(), + ); + } + let resolved_size = actual_chunks.iter().try_fold(0usize, |total, chunk| { + total + .checked_add(chunk.src_size) + .ok_or_else(|| "resolved public file size overflow".to_string()) + })?; + if resolved_size != file.size { + return Err(format!( + "resolved public file has {resolved_size} bytes, expected {}", + file.size + )); + } + progress.report(&format!( + "Ready to stream {} ({} bytes, {} chunks)", + file.name, + file.size, + actual_chunks.len() + )); + Ok(BrowserFileReader { + inner: Rc::clone(&self.inner), + file, + root_data_map, + cache: RefCell::new(BrowserRangeCache::default()), + progress, + closed: Cell::new(false), + }) + } + async fn download_public_file_inner( &self, file: PublicFileDescriptor, @@ -1074,7 +1438,7 @@ impl BrowserNetworkClient { let mut chunks = super::decode_public_data_map(&data_map).map_err(|error| error.to_string())?; chunks.extend(file.chunks.iter().cloned()); - let mut seen = std::collections::HashSet::new(); + let mut seen = HashSet::new(); chunks.retain(|chunk| seen.insert(chunk.dst_hash.clone())); if chunks.len() < 3 { return Err("ant-core returned an invalid public DataMap".to_string()); diff --git a/docs/adr/ADR-0003-direct-browser-read-client.md b/docs/adr/ADR-0003-direct-browser-read-client.md index 5f445184..bc8649d5 100644 --- a/docs/adr/ADR-0003-direct-browser-read-client.md +++ b/docs/adr/ADR-0003-direct-browser-read-client.md @@ -69,7 +69,9 @@ The Rust/WASM core will: - encode and decode the native MessagePack `DataMap` representation; - calculate and verify BLAKE3 content addresses; - verify every encrypted record against its DataMap destination address; and -- authenticate, decompress, and reconstruct complete public files. +- authenticate, decompress, and reconstruct complete public files; and +- resolve nested DataMaps and decrypt bounded plaintext byte ranges after + asynchronously retrieving only the overlapping encrypted records. The `web/` package will remain responsible for browser-specific orchestration: @@ -103,13 +105,21 @@ The `web/` package will remain responsible for browser-specific orchestration: then pass those records to the Rust/WASM reconstruction API; - verify the final file size and use the Rust/WASM BLAKE3 verifier before allowing a save; +- expose a bounded `BrowserFileReader` that retains a small encrypted-record + cache and serves independent ranges for playback and seeking; +- bridge native media-element requests to that reader through a same-origin + service worker returning standard `200`/`206`, `Content-Length`, + `Content-Range`, and `Accept-Ranges` responses; and - expose a small test site that loads the local testnet manifest, displays the - startup-published file, uploads paid files, and downloads through the browser - save flow. + startup-published file, uploads paid files, downloads through the browser + save flow, and stream-watches browser-supported video files. -The local browser manifest is bootstrap metadata, not a gateway. Production -clients will replace its unsigned endpoint list with the ML-DSA-signed records -defined by ant-node ADR-0009. +The local browser manifest is bootstrap metadata, not a gateway. Independently +deployed nodes publish their WebRTC Direct multiaddresses through Saorsa's +transport-authenticated DHT address sets, allowing a production client to +discover peers after dialing one configured bootstrap address. ADR-0009 keeps +an independently cacheable ML-DSA-signed endpoint record as a possible later +hardening step. The JavaScript API and demo never accept a certificate hash separately from an endpoint. Their sole dialing input is a canonical address of the form @@ -161,15 +171,17 @@ metadata chain. content verification. - Self-encryption, DataMap serialization, reconstruction, and content addressing have one Rust implementation across native and browser clients. +- Video playback and seeks retrieve and decrypt only the required records; + playback memory is bounded independently of the complete file size. - The browser application keeps direct control of WebRTC, wallet, and DOM APIs without pulling native runtime dependencies into WASM. ### Negative / Trade-offs -- The current client reconstructs files in memory and the local launcher caps - public files at 1 GB (1,000,000,000 bytes); upload encryption and - reconstruction are not yet streaming, so the practical limit depends on - available browser memory. +- Whole-file upload and save still process complete files in memory and the + local launcher caps public files at 1 GB (1,000,000,000 bytes). Video range + playback is bounded, but it does not remove those separate upload/save + memory constraints. - JavaScript quote-verification behavior must remain aligned with native protocol rules until transport-free `ant-protocol` APIs exist. - The initial WASM module is approximately 1.4 MiB uncompressed and browser @@ -192,6 +204,8 @@ metadata chain. - The manifest HTTP service carries only small bootstrap metadata. - Browser WebRTC APIs still require a secure browser context; localhost qualifies for development. +- The range-response service worker also requires a secure context and the + controlling page must remain open to own WebRTC and the Rust range reader. - Node and client repositories must run compatible browser protocol versions. ## Validation @@ -217,10 +231,13 @@ metadata chain. self-contained multiaddress, retrieves every record, and reconstructs the exact file, pays a real signed quote, accepts a paid binary PUT through the ordinary node verifier, and reads the stored record back. +- The real-browser test uploads a nested-DataMap file, opens it through the + Rust range reader, verifies exact disjoint and suffix HTTP ranges through the + service worker, and then performs complete reconstruction as a separate path. - Before acceptance, run interactive tests on current Chrome, Firefox, and Safari and add shared lookup convergence vectors with the native client. -- Revisit this decision when streaming file reconstruction or production signed - endpoint discovery is implemented. +- Revisit this decision when relayed WebRTC or independently cacheable signed + endpoint records are implemented. ## Notes for AI-assisted work diff --git a/web/README.md b/web/README.md index 36341dde..26cdfed3 100644 --- a/web/README.md +++ b/web/README.md @@ -7,8 +7,9 @@ lookup engine. `ant-core` is compiled to WASM and owns the WebRTC peer connections and data channels, wire framing, authenticated HELLO, connection pool, Kademlia walk, native self-encryption, quote and commitment verification, payment planning, record upload/download, public DataMap serialization, -reconstruction, and BLAKE3 content verification. JavaScript drives the page, -browser file/save APIs, and Ethers transaction submission. +whole-file reconstruction, random-access range decryption, and BLAKE3 content +verification. JavaScript drives the page, browser file/save and service-worker +APIs, and Ethers transaction submission. The node-side WebRTC Direct listener and testnet manifest API live in the `ant-node-web-support` sibling repository. No HTTP gateway performs lookup or @@ -16,7 +17,7 @@ proxies file bytes. ## Requirements -- Rust 1.88 or newer for the node's optional Saorsa WebRTC transport. +- Rust 1.88 or newer for the node's Saorsa WebRTC transport. - The `wasm32-unknown-unknown` Rust target. - `wasm-pack` 0.15. - Node.js 20.19+ or 22.12+. @@ -39,7 +40,7 @@ cargo install wasm-pack --version 0.15.0 --locked From `ant-node-web-support`: ```bash -cargo run --features webrtc-direct --bin ant-devnet -- \ +cargo run --bin ant-devnet -- \ --preset minimal \ --base-port 23000 \ --webrtc-direct \ @@ -91,8 +92,9 @@ Open `http://127.0.0.1:5173`. The page automatically loads the testnet manifest from port 25000 and fills in the default file: 1. **Load testnet** refreshes the manifest and direct multiaddress catalog. -2. **Connect** parses the first node multiaddress and performs a pinned - WebRTC Direct `HELLO` and verifies its ML-DSA identity signature. +2. **Connect and use as bootstrap** parses the first node multiaddress, + performs a pinned WebRTC Direct `HELLO`, and verifies its ML-DSA identity + signature. 3. **Find closest** runs Saorsa's iterative lookup engine and WebRTC Direct query batches entirely in Rust/WASM. 4. Under **Paid public file upload**, choose a file, paste the funded private @@ -106,10 +108,37 @@ manifest from port 25000 and fills in the default file: 5. **Download and save file** opens the browser save flow, fetches the public DataMap and encrypted chunks from direct closest storage nodes, reconstructs the whole file, verifies BLAKE3, and retains a **Save again** link. +6. For a browser-supported video, select **Prepare video stream** and then use + the native video controls. A Rust `BrowserFileReader` resolves the root + DataMap and fetches only records overlapping each requested byte range. A + thin service-worker adapter presents those decrypted ranges to the native + `
+
+

Stream video directly

+

+ Opens the public file as a seekable byte-range source. Playback fetches, + verifies, and decrypts only the encrypted chunks needed by the video element. + The public DataMap address from the download field above is used. +

+
+ + +
+ +
+

Protocol log


diff --git a/web/src/main.js b/web/src/main.js
index c2b0c844..6e1b416b 100644
--- a/web/src/main.js
+++ b/web/src/main.js
@@ -37,18 +37,24 @@ const elements = {
   downloadFile: document.querySelector("#download-file"),
   downloadState: document.querySelector("#download-state"),
   downloadLink: document.querySelector("#download-link"),
+  streamFile: document.querySelector("#stream-file"),
+  streamState: document.querySelector("#stream-state"),
+  streamVideo: document.querySelector("#stream-video"),
   log: document.querySelector("#log"),
 };
 
-const manifestOverride = new URLSearchParams(window.location.search).get(
-  "manifest",
-);
+const pageParameters = new URLSearchParams(window.location.search);
+const manifestOverride = pageParameters.get("manifest");
 if (manifestOverride) elements.manifestUrl.value = manifestOverride;
+const endpointOverride = pageParameters.get("endpoint");
+if (endpointOverride) elements.endpointMultiaddr.value = endpointOverride;
 
 let client;
 let networkClient;
 let browserManifest;
 let downloadObjectUrl;
+const videoReaders = new Map();
+let activeVideoSession;
 
 function timestamp() {
   return new Date().toLocaleTimeString();
@@ -76,6 +82,103 @@ function endpointFromForm() {
   return elements.endpointMultiaddr.value.trim();
 }
 
+function useEndpointAsBootstrap(multiaddr, hello) {
+  const endpointWasInManifest = browserManifest?.endpoints.some(
+    (endpoint) => endpoint.multiaddr === multiaddr,
+  );
+  const files = endpointWasInManifest ? browserManifest.files : [];
+  if (!endpointWasInManifest) {
+    stopVideoStream();
+    elements.fileAddress.value = "";
+    elements.publicFile.hidden = true;
+  }
+  networkClient?.close();
+  networkClient = new BrowserNetworkClient([{ multiaddr }]);
+  browserManifest = {
+    version: browserManifest?.version ?? 5,
+    network_id: endpointWasInManifest
+      ? browserManifest.network_id
+      : `manual-seed-${hello.peer_id}`,
+    created_at: endpointWasInManifest
+      ? browserManifest.created_at
+      : new Date().toISOString(),
+    endpoints: [{ multiaddr }],
+    payment: hello.payment,
+    files,
+  };
+  elements.manifestState.textContent = "Manual bootstrap · 1 direct node";
+  elements.manifestState.classList.add("connected");
+  log(`Using ${hello.peer_id} as the Rust network bootstrap seed`);
+}
+
+async function ensureVideoStreamWorker() {
+  if (!("serviceWorker" in navigator)) {
+    throw new Error("This browser does not support service workers");
+  }
+  await navigator.serviceWorker.register("/video-stream-sw.js", { scope: "/" });
+  await navigator.serviceWorker.ready;
+  if (navigator.serviceWorker.controller) return;
+  await new Promise((resolve, reject) => {
+    const timeout = setTimeout(
+      () => reject(new Error("The video streaming service worker did not take control")),
+      10_000,
+    );
+    navigator.serviceWorker.addEventListener(
+      "controllerchange",
+      () => {
+        clearTimeout(timeout);
+        resolve();
+      },
+      { once: true },
+    );
+  });
+}
+
+function stopVideoStream() {
+  elements.streamVideo.pause();
+  elements.streamVideo.removeAttribute("src");
+  elements.streamVideo.load();
+  elements.streamVideo.hidden = true;
+  if (!activeVideoSession) return;
+  videoReaders.get(activeVideoSession)?.close();
+  videoReaders.delete(activeVideoSession);
+  activeVideoSession = undefined;
+}
+
+function streamingUrl(sessionId, file) {
+  const url = new URL(`/__autonomi_stream/${sessionId}/video`, location.origin);
+  url.searchParams.set("size", String(file.size));
+  url.searchParams.set("type", file.content_type || "application/octet-stream");
+  url.searchParams.set("name", file.name);
+  return url.href;
+}
+
+navigator.serviceWorker?.addEventListener("message", async (event) => {
+  if (event.data?.type !== "autonomi-video-range") return;
+  const port = event.ports[0];
+  if (!port) return;
+  try {
+    const { sessionId, start, length } = event.data;
+    const reader = videoReaders.get(sessionId);
+    if (!reader) throw new Error("The requested video stream is no longer open");
+    if (
+      !Number.isSafeInteger(start) ||
+      !Number.isSafeInteger(length) ||
+      start < 0 ||
+      length < 0
+    ) {
+      throw new Error("The service worker requested an invalid video range");
+    }
+    const bytes = await reader.readRange(start, length);
+    const owned = bytes.byteOffset === 0 && bytes.byteLength === bytes.buffer.byteLength
+      ? bytes
+      : bytes.slice();
+    port.postMessage({ ok: true, bytes: owned.buffer }, [owned.buffer]);
+  } catch (error) {
+    port.postMessage({ ok: false, error: errorMessage(error) });
+  }
+});
+
 async function loadManifest() {
   elements.manifestState.classList.remove("connected");
   elements.manifestState.textContent = "Loading…";
@@ -83,6 +186,7 @@ async function loadManifest() {
     elements.manifestUrl.value.trim(),
   );
   browserManifest = manifest;
+  stopVideoStream();
   networkClient?.close();
   networkClient = new BrowserNetworkClient(manifest.endpoints);
 
@@ -113,9 +217,11 @@ async function loadManifest() {
 
 async function connectedClient() {
   if (client) return client;
-  const next = new BrowserNodeClient(endpointFromForm());
+  const multiaddr = endpointFromForm();
+  const next = new BrowserNodeClient(multiaddr);
   const hello = await next.hello();
   client = next;
+  useEndpointAsBootstrap(multiaddr, hello);
   elements.connectionState.textContent = `Connected · ${hello.peer_id.slice(0, 16)}…`;
   elements.connectionState.classList.add("connected");
   log("HELLO", hello);
@@ -276,6 +382,57 @@ elements.downloadFile.addEventListener("click", async () => {
   }
 });
 
+elements.streamFile.addEventListener("click", async () => {
+  elements.streamState.classList.remove("connected");
+  elements.streamState.textContent = "Opening stream…";
+  elements.streamFile.disabled = true;
+  try {
+    const address = elements.fileAddress.value.trim().toLowerCase();
+    hexToBytes(address, 32);
+    const published = browserManifest?.files.find(
+      (file) => file.address === address,
+    );
+    if (!published) {
+      throw new Error(
+        "That public file address is not described by the loaded testnet manifest",
+      );
+    }
+    if (!networkClient) throw new Error("Browser network client is not ready");
+    stopVideoStream();
+    await ensureVideoStreamWorker();
+    const reader = await networkClient.openPublicFile(published, (message) => {
+      elements.streamState.textContent = message;
+      log(message);
+    });
+    const sessionId = bytesToHex(crypto.getRandomValues(new Uint8Array(16)));
+    videoReaders.set(sessionId, reader);
+    activeVideoSession = sessionId;
+    elements.streamVideo.src = streamingUrl(sessionId, published);
+    elements.streamVideo.hidden = false;
+    elements.streamState.textContent = "Ready · press play to stream";
+    elements.streamState.classList.add("connected");
+    log(`Prepared random-access video stream for ${published.name}`, {
+      size: published.size,
+      content_type: published.content_type,
+      session_id: sessionId,
+    });
+  } catch (error) {
+    stopVideoStream();
+    elements.streamState.textContent = "Stream failed";
+    log(`Video stream failed: ${errorMessage(error)}`);
+    console.error(error);
+  } finally {
+    elements.streamFile.disabled = false;
+  }
+});
+
+elements.streamVideo.addEventListener("error", () => {
+  const mediaError = elements.streamVideo.error;
+  if (!mediaError || !activeVideoSession) return;
+  elements.streamState.textContent = `Playback error (${mediaError.code})`;
+  log(`Video element could not decode the selected file (media error ${mediaError.code})`);
+});
+
 async function chooseSaveHandle(name) {
   if (typeof window.showSaveFilePicker !== "function") return undefined;
   return window.showSaveFilePicker({ suggestedName: name });
@@ -316,6 +473,7 @@ async function exposeSavedFile(file, content, saveHandle) {
 
 window.addEventListener("beforeunload", () => {
   if (downloadObjectUrl) URL.revokeObjectURL(downloadObjectUrl);
+  stopVideoStream();
   client?.close();
   networkClient?.close();
 });
@@ -339,8 +497,13 @@ function hexToBytes(value, expectedLength) {
 }
 
 elements.randomTarget.click();
-log("Ready. Loading the local browser testnet manifest…");
-loadManifest().catch((error) => {
-  elements.manifestState.textContent = "Not running";
-  log(`Local manifest not available yet: ${errorMessage(error)}`);
-});
+if (endpointOverride) {
+  elements.manifestState.textContent = "Manual endpoint · manifest skipped";
+  log("Ready. Using the WebRTC Direct endpoint from the page URL.");
+} else {
+  log("Ready. Loading the local browser testnet manifest…");
+  loadManifest().catch((error) => {
+    elements.manifestState.textContent = "Not running";
+    log(`Local manifest not available yet: ${errorMessage(error)}`);
+  });
+}
diff --git a/web/src/style.css b/web/src/style.css
index 301225fb..6e322766 100644
--- a/web/src/style.css
+++ b/web/src/style.css
@@ -191,6 +191,19 @@ a {
   font-weight: 700;
 }
 
+video {
+  display: block;
+  width: 100%;
+  max-height: 70vh;
+  margin-top: 1rem;
+  border-radius: 10px;
+  background: #000;
+}
+
+video[hidden] {
+  display: none;
+}
+
 @media (prefers-color-scheme: dark) {
   :root {
     color: #e8eee9;

From f2c039e065a09f94723fa6acb23f920439825181 Mon Sep 17 00:00:00 2001
From: Mick van Dijke <12992260+mickvandijke@users.noreply.github.com>
Date: Mon, 31 Aug 2026 11:44:14 +0200
Subject: [PATCH 11/31] refactor(client): share browser and native transfer
 policy

---
 Cargo.lock                             |   4 +
 ant-core/src/browser/wasm_transport.rs | 297 ++++++++++++++++++-------
 ant-core/src/client_engine.rs          | 146 ++++++++++++
 ant-core/src/data/client/batch.rs      |  11 +-
 ant-core/src/data/client/data.rs       |  13 +-
 ant-core/src/data/client/file.rs       |  11 +-
 ant-core/src/data/client/merkle.rs     |  40 ++--
 ant-core/src/lib.rs                    |   3 +
 8 files changed, 411 insertions(+), 114 deletions(-)
 create mode 100644 ant-core/src/client_engine.rs

diff --git a/Cargo.lock b/Cargo.lock
index 6624000a..2518c88a 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -5264,6 +5264,10 @@ dependencies = [
 [[package]]
 name = "saorsa-dht-lookup"
 version = "0.1.0"
+dependencies = [
+ "futures-core",
+ "futures-util",
+]
 
 [[package]]
 name = "saorsa-pqc"
diff --git a/ant-core/src/browser/wasm_transport.rs b/ant-core/src/browser/wasm_transport.rs
index 0b9861e0..ffba0c19 100644
--- a/ant-core/src/browser/wasm_transport.rs
+++ b/ant-core/src/browser/wasm_transport.rs
@@ -17,13 +17,13 @@ use futures_channel::{mpsc, oneshot};
 use futures_util::{
     future::{join_all, select, Either},
     lock::Mutex,
-    stream::{self, StreamExt as _},
+    stream::{self, FuturesUnordered, StreamExt as _},
 };
 use gloo_timers::future::TimeoutFuture;
 use js_sys::{Array, ArrayBuffer, Promise, Uint8Array};
 use saorsa_dht_lookup::{
-    run_iterative_lookup, IterativeLookup, LookupConfig, LookupKey, LookupNode, LookupQuery,
-    LookupQueryOutcome,
+    collect_after_first_with_grace, run_iterative_lookup, xor_distance, IterativeLookup,
+    LookupConfig, LookupKey, LookupNode, LookupQuery, LookupQueryOutcome,
 };
 use serde::{Deserialize, Serialize};
 use serde_json::{Map, Value};
@@ -32,6 +32,7 @@ use std::collections::{HashMap, HashSet};
 use std::future::Future;
 use std::ops::Deref;
 use std::rc::Rc;
+use std::time::Duration;
 use wasm_bindgen::prelude::*;
 use wasm_bindgen_futures::JsFuture;
 use web_sys::{
@@ -43,10 +44,16 @@ const REQUEST_TIMEOUT_MS: u32 = 10_000;
 const MAX_BUFFERED_AMOUNT: u32 = 2 * 1024 * 1024;
 const ICE_CREDENTIAL_PREFIX: &str = "saorsa+webrtc+v1/";
 const ICE_ALPHABET: &[u8] = b"abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789";
-const DEFAULT_MAX_POOLED_CLIENTS: usize = 10;
+const DEFAULT_MAX_POOLED_CLIENTS: usize = 32;
 const DEFAULT_LOOKUP_K: usize = 20;
 const DEFAULT_LOOKUP_ALPHA: usize = 3;
 const DEFAULT_MAX_LOOKUP_ITERATIONS: usize = 20;
+const LOOKUP_GRACE_TIMEOUT_MS: u32 = 5_000;
+const ENDPOINT_FAILURE_COOLDOWN: Duration = Duration::from_secs(30 * 60);
+const MAX_BROWSER_ROUTING_ENTRIES: usize = 256;
+const MAX_BROWSER_ENDPOINT_FAILURES: usize = 256;
+const DEFAULT_BROWSER_QUOTE_CONCURRENCY: usize = 4;
+const DEFAULT_BROWSER_STORE_CONCURRENCY: usize = 4;
 const MAX_STORE_TARGETS: usize = 7;
 const MAX_DOWNLOAD_CONCURRENCY: usize = 6;
 const MAX_BROWSER_RANGE_BYTES: usize = 4 * 1024 * 1024;
@@ -473,10 +480,12 @@ impl BrowserNodeClientCore {
             }
         };
         if response.header.get("request_id").and_then(Value::as_u64) != Some(request_id) {
-            return Err(format!(
+            let error = format!(
                 "response ID {} does not match request {request_id}",
                 response.header.get("request_id").unwrap_or(&Value::Null)
-            ));
+            );
+            self.close();
+            return Err(error);
         }
         if response.header.get("status").and_then(Value::as_str) == Some("error") {
             return Err(response
@@ -634,6 +643,8 @@ impl BrowserNodeClientCore {
 struct BrowserNetworkCore {
     seeds: Vec,
     pool: Rc,
+    routing: Rc>>,
+    failed_endpoints: Rc>>,
 }
 
 impl BrowserNetworkCore {
@@ -654,6 +665,13 @@ impl BrowserNetworkCore {
         Ok(Self {
             seeds,
             pool: Rc::new(BrowserClientPool::new(DEFAULT_MAX_POOLED_CLIENTS)?),
+            routing: Rc::new(RefCell::new(HashMap::new())),
+            failed_endpoints: Rc::new(RefCell::new(
+                crate::client_engine::EndpointFailureCache::new(
+                    ENDPOINT_FAILURE_COOLDOWN,
+                    MAX_BROWSER_ENDPOINT_FAILURES,
+                ),
+            )),
         })
     }
 
@@ -695,12 +713,15 @@ impl BrowserNetworkCore {
                 }
             }
         });
-        let seed_candidates = join_all(seed_futures)
-            .await
-            .into_iter()
-            .flatten()
-            .collect::>();
-        if seed_candidates.is_empty() {
+        let mut initial_candidates = self.routing.borrow().values().cloned().collect::>();
+        if initial_candidates.is_empty() {
+            initial_candidates = join_all(seed_futures)
+                .await
+                .into_iter()
+                .flatten()
+                .collect::>();
+        }
+        if initial_candidates.is_empty() {
             let detail = failures
                 .borrow()
                 .iter()
@@ -720,10 +741,24 @@ impl BrowserNetworkCore {
         };
         let mut lookup =
             IterativeLookup::new(target_key, config).map_err(|error| error.to_string())?;
-        let mut known_endpoints = HashMap::new();
-        for candidate in seed_candidates {
+        let mut known_endpoints = self
+            .routing
+            .borrow()
+            .iter()
+            .filter_map(|(peer, candidate)| {
+                candidate
+                    .wire
+                    .webrtc_direct
+                    .clone()
+                    .map(|endpoint| (*peer, endpoint))
+            })
+            .collect::>();
+        for candidate in initial_candidates {
             if let Some(endpoint) = candidate.wire.webrtc_direct.clone() {
                 known_endpoints.insert(candidate.peer_id, endpoint);
+                self.routing
+                    .borrow_mut()
+                    .insert(candidate.peer_id, candidate.clone());
                 let _ = lookup.add_candidate(candidate);
             }
         }
@@ -732,10 +767,21 @@ impl BrowserNetworkCore {
             progress: progress.clone(),
             failures: Rc::clone(&failures),
             known_endpoints,
+            routing: Rc::clone(&self.routing),
+            failed_endpoints: Rc::clone(&self.failed_endpoints),
         };
         run_iterative_lookup(&mut lookup, &mut query)
             .await
             .map_err(|error| error.to_string())?;
+        let mut routes = self.routing.borrow_mut();
+        if routes.len() > MAX_BROWSER_ROUTING_ENTRIES {
+            let mut peers = routes.keys().copied().collect::>();
+            peers.sort_by_key(|peer| xor_distance(peer, &target_key));
+            for peer in peers.into_iter().skip(MAX_BROWSER_ROUTING_ENTRIES) {
+                routes.remove(&peer);
+            }
+        }
+        drop(routes);
         let nodes = lookup
             .results()
             .into_iter()
@@ -765,9 +811,7 @@ impl BrowserNetworkCore {
             progress.report(&format!("Requesting {address} from {}", node.peer_id));
             let result = async {
                 let client = self.pool.client(endpoint).await?;
-                if client.peer_id().is_none() {
-                    client.hello().await?;
-                }
+                client.hello().await?;
                 client.get_chunk(&address).await
             }
             .await;
@@ -798,11 +842,26 @@ struct BrowserNetworkLookupQuery {
     progress: ProgressReporter,
     failures: Rc>>,
     known_endpoints: HashMap,
+    routing: Rc>>,
+    failed_endpoints: Rc>>,
 }
 
 impl LookupQuery for BrowserNetworkLookupQuery {
     type Error = String;
 
+    async fn is_candidate_eligible(
+        &mut self,
+        candidate: &BrowserLookupCandidate,
+    ) -> Result {
+        let Some(endpoint) = candidate.wire.webrtc_direct.as_ref() else {
+            return Ok(false);
+        };
+        Ok(!self
+            .failed_endpoints
+            .borrow_mut()
+            .is_suppressed(&candidate.peer_id, &endpoint.multiaddr))
+    }
+
     async fn query_batch(
         &mut self,
         target: LookupKey,
@@ -811,60 +870,103 @@ impl LookupQuery for BrowserNetworkLookupQuery {
         batch: Vec,
     ) -> Result>, Self::Error> {
         let target = hex::encode(target);
-        let futures = batch.into_iter().map(|candidate| {
-            let pool = Rc::clone(&self.pool);
-            let progress = self.progress.clone();
-            let failures = Rc::clone(&self.failures);
-            let target = target.clone();
-            async move {
-                let responder = candidate.peer_id;
-                let peer_id = candidate.wire.peer_id.clone();
-                let result = async {
-                    let endpoint = candidate.wire.webrtc_direct.as_ref().ok_or_else(|| {
-                        "lookup candidate has no WebRTC Direct endpoint".to_string()
-                    })?;
-                    let client = pool.client(endpoint).await?;
-                    if client.peer_id().is_none() {
+        let attempted = batch
+            .iter()
+            .filter_map(|candidate| {
+                candidate
+                    .wire
+                    .webrtc_direct
+                    .as_ref()
+                    .map(|endpoint| (candidate.peer_id, endpoint.multiaddr.clone()))
+            })
+            .collect::>();
+        let futures: FuturesUnordered<_> = batch
+            .into_iter()
+            .map(|candidate| {
+                let pool = Rc::clone(&self.pool);
+                let progress = self.progress.clone();
+                let failures = Rc::clone(&self.failures);
+                let failed_endpoints = Rc::clone(&self.failed_endpoints);
+                let target = target.clone();
+                async move {
+                    let responder = candidate.peer_id;
+                    let peer_id = candidate.wire.peer_id.clone();
+                    let failed_endpoint = candidate
+                        .wire
+                        .webrtc_direct
+                        .as_ref()
+                        .map(|endpoint| endpoint.multiaddr.clone());
+                    let result = async {
+                        let endpoint = candidate.wire.webrtc_direct.as_ref().ok_or_else(|| {
+                            "lookup candidate has no WebRTC Direct endpoint".to_string()
+                        })?;
+                        let client = pool.client(endpoint).await?;
                         client.hello().await?;
+                        client.find_node(&target, count).await
                     }
-                    client.find_node(&target, count).await
-                }
-                .await;
-                match result {
-                    Ok(nodes) => {
-                        progress.report(&format!(
-                            "Iteration {iteration}: {peer_id} returned {} nodes",
-                            nodes.len()
-                        ));
-                        let candidates = nodes
-                            .into_iter()
-                            .filter_map(|wire| match BrowserLookupCandidate::parse(wire) {
-                                Ok(candidate) => Some(candidate),
-                                Err(error) => {
-                                    progress.report(&format!(
-                                        "Ignoring invalid candidate from {peer_id}: {error}"
-                                    ));
-                                    None
-                                }
-                            })
-                            .collect();
-                        LookupQueryOutcome::Succeeded {
-                            responder,
-                            candidates,
+                    .await;
+                    match result {
+                        Ok(nodes) => {
+                            failed_endpoints.borrow_mut().record_success(&responder);
+                            progress.report(&format!(
+                                "Iteration {iteration}: {peer_id} returned {} nodes",
+                                nodes.len()
+                            ));
+                            let candidates = nodes
+                                .into_iter()
+                                .filter_map(|wire| match BrowserLookupCandidate::parse(wire) {
+                                    Ok(candidate) => Some(candidate),
+                                    Err(error) => {
+                                        progress.report(&format!(
+                                            "Ignoring invalid candidate from {peer_id}: {error}"
+                                        ));
+                                        None
+                                    }
+                                })
+                                .collect();
+                            LookupQueryOutcome::Succeeded {
+                                responder,
+                                candidates,
+                            }
+                        }
+                        Err(error) => {
+                            if let Some(endpoint) = failed_endpoint {
+                                failed_endpoints
+                                    .borrow_mut()
+                                    .record_failure(responder, endpoint);
+                            }
+                            progress.report(&format!("Query {peer_id} failed: {error}"));
+                            failures.borrow_mut().push(BrowserLookupFailure {
+                                peer_id,
+                                message: error,
+                            });
+                            LookupQueryOutcome::Failed { responder }
                         }
-                    }
-                    Err(error) => {
-                        progress.report(&format!("Query {peer_id} failed: {error}"));
-                        failures.borrow_mut().push(BrowserLookupFailure {
-                            peer_id,
-                            message: error,
-                        });
-                        LookupQueryOutcome::Failed { responder }
                     }
                 }
+            })
+            .collect();
+        let mut outcomes =
+            collect_after_first_with_grace(futures, || TimeoutFuture::new(LOOKUP_GRACE_TIMEOUT_MS))
+                .await;
+        let responded = outcomes
+            .iter()
+            .map(|outcome| *outcome.responder())
+            .collect::>();
+        for (peer, endpoint) in attempted {
+            if !responded.contains(&peer) {
+                self.failed_endpoints
+                    .borrow_mut()
+                    .record_failure(peer, endpoint);
+                let peer_id = hex::encode(peer);
+                let message = "did not respond before the lookup grace period".to_string();
+                self.progress
+                    .report(&format!("Query {peer_id} failed: {message}"));
+                self.failures
+                    .borrow_mut()
+                    .push(BrowserLookupFailure { peer_id, message });
             }
-        });
-        let mut outcomes = join_all(futures).await;
+        }
         for outcome in &mut outcomes {
             if let LookupQueryOutcome::Succeeded { candidates, .. } = outcome {
                 candidates.retain_mut(|candidate| {
@@ -873,7 +975,14 @@ impl LookupQuery for BrowserNetworkLookupQuery {
                     } else if let Some(endpoint) = self.known_endpoints.get(&candidate.peer_id) {
                         candidate.wire.webrtc_direct = Some(endpoint.clone());
                     }
-                    candidate.wire.webrtc_direct.is_some()
+                    if candidate.wire.webrtc_direct.is_some() {
+                        self.routing
+                            .borrow_mut()
+                            .insert(candidate.peer_id, candidate.clone());
+                        true
+                    } else {
+                        false
+                    }
                 });
             }
         }
@@ -1515,18 +1624,40 @@ impl BrowserNetworkClient {
             content.len()
         ));
         let encrypted = super::encrypt_public_file(content).map_err(|error| error.to_string())?;
+        let mut records = encrypted.records.iter().cloned().enumerate();
         let mut prepared = Vec::with_capacity(encrypted.records.len());
-        for (index, record) in encrypted.records.iter().cloned().enumerate() {
+        if let Some((index, record)) = records.next() {
             progress.report(&format!(
                 "Preparing record {}/{}",
                 index + 1,
                 encrypted.records.len()
             ));
-            prepared.push(
+            prepared.push((
+                index,
                 self.prepare_record(record, &payment_network, progress)
                     .await?,
-            );
+            ));
         }
+        let record_count = encrypted.records.len();
+        let payment_network_ref = &payment_network;
+        let remaining = records.map(|(index, record)| async move {
+            progress.report(&format!("Preparing record {}/{}", index + 1, record_count));
+            self.prepare_record(record, payment_network_ref, progress)
+                .await
+                .map(|prepared| (index, prepared))
+        });
+        let remaining =
+            crate::client_engine::bounded_unordered(remaining, DEFAULT_BROWSER_QUOTE_CONCURRENCY)
+                .collect::>()
+                .await;
+        for result in remaining {
+            prepared.push(result?);
+        }
+        prepared.sort_by_key(|(index, _)| *index);
+        let prepared = prepared
+            .into_iter()
+            .map(|(_, record)| record)
+            .collect::>();
         let verified_quotes = prepared
             .iter()
             .filter_map(|record| record.verified.clone())
@@ -1554,18 +1685,24 @@ impl BrowserNetworkClient {
             *transaction_hash = super::protocol::normalize_hex(transaction_hash, 32)?;
         }
 
+        let record_count = prepared.len();
+        let payment_network_ref = &payment_network;
+        let transaction_hash = payment.transaction_hash.as_deref();
+        let stores = prepared
+            .iter()
+            .enumerate()
+            .map(|(index, record)| async move {
+                progress.report(&format!("Storing record {}/{}", index + 1, record_count));
+                self.store_prepared(record, payment_network_ref, transaction_hash, progress)
+                    .await
+            });
+        let stores =
+            crate::client_engine::bounded_unordered(stores, DEFAULT_BROWSER_STORE_CONCURRENCY)
+                .collect::>()
+                .await;
         let mut replicas = usize::MAX;
-        for (index, record) in prepared.iter().enumerate() {
-            progress.report(&format!("Storing record {}/{}", index + 1, prepared.len()));
-            let stored = self
-                .store_prepared(
-                    record,
-                    &payment_network,
-                    payment.transaction_hash.as_deref(),
-                    progress,
-                )
-                .await?;
-            replicas = replicas.min(stored);
+        for stored in stores {
+            replicas = replicas.min(stored?);
         }
         let descriptor = PublicFileDescriptor {
             name: name.to_string(),
diff --git a/ant-core/src/client_engine.rs b/ant-core/src/client_engine.rs
new file mode 100644
index 00000000..6ffbfdad
--- /dev/null
+++ b/ant-core/src/client_engine.rs
@@ -0,0 +1,146 @@
+//! Runtime-neutral scheduling and session state shared by native and browser clients.
+
+use futures_util::{stream, Stream, StreamExt as _};
+#[cfg(any(feature = "browser-wasm", test))]
+use std::collections::HashMap;
+use std::future::Future;
+#[cfg(any(feature = "browser-wasm", test))]
+use std::hash::Hash;
+#[cfg(any(feature = "browser-wasm", test))]
+use std::time::{Duration, Instant};
+
+/// Run futures with a bounded rolling concurrency window.
+///
+/// This is the common scheduling primitive behind native upload waves and the
+/// browser upload pipeline. Callers retain responsibility for classifying
+/// results and adapting the next window's limit.
+pub(crate) fn bounded_unordered(
+    futures: I,
+    concurrency: usize,
+) -> impl Stream
+where
+    I: IntoIterator,
+    F: Future,
+{
+    stream::iter(futures).buffer_unordered(concurrency.max(1))
+}
+
+#[cfg(any(feature = "browser-wasm", test))]
+#[derive(Debug, Clone)]
+struct FailureRecord {
+    endpoint: String,
+    failed_at: Instant,
+}
+
+/// Runtime-neutral negative cache for transport endpoints.
+///
+/// Entries are keyed by authenticated peer identity and also retain the exact
+/// endpoint that failed. A peer is immediately eligible again when it
+/// republishes a different endpoint, while repeated use of the same dead
+/// address is suppressed for the configured cooldown.
+#[cfg(any(feature = "browser-wasm", test))]
+#[derive(Debug)]
+pub(crate) struct EndpointFailureCache {
+    cooldown: Duration,
+    max_entries: usize,
+    entries: HashMap,
+}
+
+#[cfg(any(feature = "browser-wasm", test))]
+impl EndpointFailureCache
+where
+    K: Clone + Eq + Hash,
+{
+    pub(crate) fn new(cooldown: Duration, max_entries: usize) -> Self {
+        Self {
+            cooldown,
+            max_entries: max_entries.max(1),
+            entries: HashMap::new(),
+        }
+    }
+
+    pub(crate) fn is_suppressed(&mut self, peer: &K, endpoint: &str) -> bool {
+        let Some(record) = self.entries.get(peer) else {
+            return false;
+        };
+        if record.endpoint != endpoint || record.failed_at.elapsed() >= self.cooldown {
+            self.entries.remove(peer);
+            return false;
+        }
+        true
+    }
+
+    pub(crate) fn record_failure(&mut self, peer: K, endpoint: String) {
+        if !self.entries.contains_key(&peer) && self.entries.len() >= self.max_entries {
+            let oldest = self
+                .entries
+                .iter()
+                .max_by_key(|(_, record)| record.failed_at.elapsed())
+                .map(|(peer, _)| peer.clone());
+            if let Some(oldest) = oldest {
+                self.entries.remove(&oldest);
+            }
+        }
+        self.entries.insert(
+            peer,
+            FailureRecord {
+                endpoint,
+                failed_at: Instant::now(),
+            },
+        );
+    }
+
+    pub(crate) fn record_success(&mut self, peer: &K) {
+        self.entries.remove(peer);
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn changed_endpoint_bypasses_failure_cooldown() {
+        let mut cache = EndpointFailureCache::new(Duration::from_secs(60), 8);
+        cache.record_failure(7_u8, "old".to_string());
+        assert!(cache.is_suppressed(&7, "old"));
+        assert!(!cache.is_suppressed(&7, "new"));
+        assert!(!cache.is_suppressed(&7, "old"));
+    }
+
+    #[test]
+    fn success_clears_failure() {
+        let mut cache = EndpointFailureCache::new(Duration::from_secs(60), 8);
+        cache.record_failure(7_u8, "endpoint".to_string());
+        cache.record_success(&7);
+        assert!(!cache.is_suppressed(&7, "endpoint"));
+    }
+
+    #[test]
+    fn failure_cache_evicts_oldest_entry_at_capacity() {
+        let mut cache = EndpointFailureCache::new(Duration::from_secs(60), 1);
+        cache.record_failure(7_u8, "first".to_string());
+        cache.record_failure(8_u8, "second".to_string());
+
+        assert!(!cache.is_suppressed(&7, "first"));
+        assert!(cache.is_suppressed(&8, "second"));
+    }
+
+    #[test]
+    fn bounded_scheduler_keeps_all_outputs() {
+        let outputs = futures::executor::block_on(async {
+            bounded_unordered(
+                [
+                    futures_util::future::ready(1_u8),
+                    futures_util::future::ready(2_u8),
+                ],
+                0,
+            )
+            .collect::>()
+            .await
+        });
+        assert_eq!(outputs.len(), 2);
+        assert!(outputs.contains(&1));
+        assert!(outputs.contains(&2));
+    }
+}
diff --git a/ant-core/src/data/client/batch.rs b/ant-core/src/data/client/batch.rs
index 9a62b4c4..a739fb4c 100644
--- a/ant-core/src/data/client/batch.rs
+++ b/ant-core/src/data/client/batch.rs
@@ -20,7 +20,7 @@ use ant_protocol::payment::{
 use ant_protocol::transport::{MultiAddr, PeerId};
 use ant_protocol::{compute_address, XorName, CLOSE_GROUP_SIZE, DATA_TYPE_CHUNK};
 use bytes::Bytes;
-use futures::stream::{self, FuturesUnordered, StreamExt};
+use futures::stream::{FuturesUnordered, StreamExt};
 use std::collections::{HashMap, HashSet};
 use std::time::{Duration, Instant};
 use tokio::sync::mpsc;
@@ -785,8 +785,8 @@ impl Client {
         // See PERF-RESULTS.md — measured ~30% slowdown when
         // cap > batch size on quoting workloads (live mainnet).
         let quote_concurrency = quote_limiter.current().min(chunk_count.max(1));
-        let mut quote_stream = stream::iter(chunks_with_addr)
-            .map(|(content, address)| {
+        let mut quote_stream = crate::client_engine::bounded_unordered(
+            chunks_with_addr.into_iter().map(|(content, address)| {
                 let limiter = quote_limiter.clone();
                 async move {
                     let result = observe_op(
@@ -797,8 +797,9 @@ impl Client {
                     .await;
                     (address, result)
                 }
-            })
-            .buffer_unordered(quote_concurrency);
+            }),
+            quote_concurrency,
+        );
 
         let mut prepared = Vec::with_capacity(chunk_count);
         let mut already_stored = Vec::new();
diff --git a/ant-core/src/data/client/data.rs b/ant-core/src/data/client/data.rs
index ba5fbf9a..5cd38d60 100644
--- a/ant-core/src/data/client/data.rs
+++ b/ant-core/src/data/client/data.rs
@@ -310,8 +310,8 @@ impl Client {
         let quote_limiter = self.controller().quote.clone();
         let quote_concurrency = quote_limiter.current().min(chunk_count.max(1));
         let results: Vec<([u8; 32], Result>)> =
-            futures::stream::iter(chunks_with_addr)
-                .map(|(content, address)| {
+            crate::client_engine::bounded_unordered(
+                chunks_with_addr.into_iter().map(|(content, address)| {
                     let limiter = quote_limiter.clone();
                     async move {
                         let result = observe_op(
@@ -322,10 +322,11 @@ impl Client {
                         .await;
                         (address, result)
                     }
-                })
-                .buffer_unordered(quote_concurrency)
-                .collect()
-                .await;
+                }),
+                quote_concurrency,
+            )
+            .collect()
+            .await;
 
         let mut prepared_chunks = Vec::with_capacity(results.len());
         let mut already_stored_addresses = Vec::new();
diff --git a/ant-core/src/data/client/file.rs b/ant-core/src/data/client/file.rs
index 42bc60dd..ac0c127b 100644
--- a/ant-core/src/data/client/file.rs
+++ b/ant-core/src/data/client/file.rs
@@ -29,7 +29,7 @@ use ant_protocol::transport::{MultiAddr, PeerId};
 use ant_protocol::{compute_address, XorName as ChunkAddress, DATA_TYPE_CHUNK};
 use bytes::Bytes;
 use fs2::FileExt;
-use futures::stream::{self, StreamExt};
+use futures::stream::StreamExt;
 use self_encryption::{
     get_root_data_map_parallel, stream_decrypt_batch_size, stream_encrypt,
     streaming_decrypt_with_batch_size, DataMap,
@@ -1687,8 +1687,8 @@ impl Client {
         // a progress bar through the slow quote phase.
         let quote_limiter = self.controller().quote.clone();
         let quote_concurrency = quote_limiter.current().min(chunk_count.max(1));
-        let mut quote_stream = stream::iter(chunks_with_addr)
-            .map(|(content, address)| {
+        let mut quote_stream = crate::client_engine::bounded_unordered(
+            chunks_with_addr.into_iter().map(|(content, address)| {
                 let limiter = quote_limiter.clone();
                 async move {
                     let result = observe_op(
@@ -1699,8 +1699,9 @@ impl Client {
                     .await;
                     (address, result)
                 }
-            })
-            .buffer_unordered(quote_concurrency);
+            }),
+            quote_concurrency,
+        );
 
         let mut prepared_chunks = Vec::with_capacity(chunk_count);
         let mut already_stored = Vec::new();
diff --git a/ant-core/src/data/client/merkle.rs b/ant-core/src/data/client/merkle.rs
index 2973c7ea..2ded76b5 100644
--- a/ant-core/src/data/client/merkle.rs
+++ b/ant-core/src/data/client/merkle.rs
@@ -26,7 +26,7 @@ use ant_protocol::{
     MerkleCandidateQuoteRequest, MerkleCandidateQuoteResponse,
 };
 use bytes::Bytes;
-use futures::stream::{self, FuturesUnordered, StreamExt};
+use futures::stream::{FuturesUnordered, StreamExt};
 use rand::Rng;
 use std::collections::{HashMap, VecDeque};
 use std::time::Duration;
@@ -545,23 +545,27 @@ impl Client {
 
         let quote_limiter = self.controller().quote.clone();
         let quote_concurrency = quote_limiter.current().min(total_chunks.max(1));
-        let mut check_stream = stream::iter(chunks.into_iter().enumerate())
-            .map(|(index, (address, data_size))| {
-                let limiter = quote_limiter.clone();
-                async move {
-                    let result = observe_op(
-                        &limiter,
-                        || async move {
-                            self.chunk_already_stored_for_merkle(&address, data_type, data_size)
-                                .await
-                        },
-                        classify_error,
-                    )
-                    .await;
-                    (index, address, data_size, result)
-                }
-            })
-            .buffer_unordered(quote_concurrency);
+        let mut check_stream = crate::client_engine::bounded_unordered(
+            chunks
+                .into_iter()
+                .enumerate()
+                .map(|(index, (address, data_size))| {
+                    let limiter = quote_limiter.clone();
+                    async move {
+                        let result = observe_op(
+                            &limiter,
+                            || async move {
+                                self.chunk_already_stored_for_merkle(&address, data_type, data_size)
+                                    .await
+                            },
+                            classify_error,
+                        )
+                        .await;
+                        (index, address, data_size, result)
+                    }
+                }),
+            quote_concurrency,
+        );
 
         let mut already_stored: Vec<(usize, [u8; 32])> = Vec::new();
         let mut to_upload: Vec<(usize, [u8; 32], u64)> = Vec::new();
diff --git a/ant-core/src/lib.rs b/ant-core/src/lib.rs
index 9404d66b..e028a561 100644
--- a/ant-core/src/lib.rs
+++ b/ant-core/src/lib.rs
@@ -1,6 +1,9 @@
 /// Cross-platform Autonomi client logic and browser WASM bindings.
 pub mod browser;
 
+#[cfg(any(feature = "native", feature = "browser-wasm"))]
+mod client_engine;
+
 #[cfg(feature = "native")]
 pub mod config;
 #[cfg(feature = "native")]

From e7db6b89fb1ed4daa8380c66d77214d14dc35cce Mon Sep 17 00:00:00 2001
From: Mick van Dijke <12992260+mickvandijke@users.noreply.github.com>
Date: Mon, 31 Aug 2026 12:53:46 +0200
Subject: [PATCH 12/31] feat(browser): share native upload scheduling

---
 Cargo.lock                             |   1 +
 ant-core/Cargo.toml                    |   2 +
 ant-core/src/browser/wasm_transport.rs | 240 ++++++++++++++++++-------
 ant-core/src/client_engine.rs          | 215 +++++++++++++++++++++-
 ant-core/src/data/client/adaptive.rs   |  20 ++-
 ant-core/src/data/client/batch.rs      |  45 ++---
 ant-core/src/data/client/chunk.rs      | 102 ++++-------
 ant-core/src/data/client/mod.rs        |   4 +-
 web/README.md                          |   7 +
 9 files changed, 472 insertions(+), 164 deletions(-)

diff --git a/Cargo.lock b/Cargo.lock
index 2518c88a..d71fe1bd 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -888,6 +888,7 @@ dependencies = [
  "wasm-bindgen",
  "wasm-bindgen-futures",
  "web-sys",
+ "web-time",
  "windows-sys 0.61.2",
  "xor_name",
  "zip",
diff --git a/ant-core/Cargo.toml b/ant-core/Cargo.toml
index 79480fcc..6afd8082 100644
--- a/ant-core/Cargo.toml
+++ b/ant-core/Cargo.toml
@@ -26,6 +26,7 @@ self_encryption = "0.36"
 saorsa-dht-lookup = { version = "0.1.0", path = "../../saorsa-core-web-support/crates/saorsa-dht-lookup" }
 thiserror = "2"
 url = "2"
+web-time = "1.1"
 
 # Node management
 async-stream = { version = "0.3", optional = true }
@@ -156,6 +157,7 @@ browser-wasm = [
     "dep:gloo-timers",
     "dep:js-sys",
     "dep:serde-wasm-bindgen",
+    "dep:tracing",
     "dep:wasm-bindgen",
     "dep:wasm-bindgen-futures",
     "dep:web-sys",
diff --git a/ant-core/src/browser/wasm_transport.rs b/ant-core/src/browser/wasm_transport.rs
index ffba0c19..bb79e308 100644
--- a/ant-core/src/browser/wasm_transport.rs
+++ b/ant-core/src/browser/wasm_transport.rs
@@ -13,6 +13,10 @@ use super::protocol::{
     BrowserResponseFrame, WebRtcDirectEndpoint, MAX_BROWSER_RESPONSE_BYTES,
     WEBRTC_DIRECT_DATA_CHANNEL, WEBRTC_WRITE_CHUNK_BYTES,
 };
+use crate::client_engine::adaptive::{
+    observe_op, AdaptiveConfig, AdaptiveController, ChannelStart, Outcome,
+};
+use ant_protocol::{CLOSE_GROUP_MAJORITY, CLOSE_GROUP_SIZE};
 use futures_channel::{mpsc, oneshot};
 use futures_util::{
     future::{join_all, select, Either},
@@ -53,8 +57,6 @@ const ENDPOINT_FAILURE_COOLDOWN: Duration = Duration::from_secs(30 * 60);
 const MAX_BROWSER_ROUTING_ENTRIES: usize = 256;
 const MAX_BROWSER_ENDPOINT_FAILURES: usize = 256;
 const DEFAULT_BROWSER_QUOTE_CONCURRENCY: usize = 4;
-const DEFAULT_BROWSER_STORE_CONCURRENCY: usize = 4;
-const MAX_STORE_TARGETS: usize = 7;
 const MAX_DOWNLOAD_CONCURRENCY: usize = 6;
 const MAX_BROWSER_RANGE_BYTES: usize = 4 * 1024 * 1024;
 const MAX_RANGE_CACHE_BYTES: usize = 32 * 1024 * 1024;
@@ -1293,6 +1295,7 @@ fn required_range_records(
 #[wasm_bindgen(js_name = BrowserNetworkClient)]
 pub struct BrowserNetworkClient {
     inner: Rc,
+    controller: AdaptiveController,
 }
 
 #[wasm_bindgen(js_class = BrowserNetworkClient)]
@@ -1312,6 +1315,7 @@ impl BrowserNetworkClient {
             BrowserNetworkCore::new(endpoints).map_err(|error| JsValue::from_str(&error))?;
         Ok(Self {
             inner: Rc::new(inner),
+            controller: AdaptiveController::new(ChannelStart::default(), AdaptiveConfig::default()),
         })
     }
 
@@ -1685,25 +1689,14 @@ impl BrowserNetworkClient {
             *transaction_hash = super::protocol::normalize_hex(transaction_hash, 32)?;
         }
 
-        let record_count = prepared.len();
-        let payment_network_ref = &payment_network;
-        let transaction_hash = payment.transaction_hash.as_deref();
-        let stores = prepared
-            .iter()
-            .enumerate()
-            .map(|(index, record)| async move {
-                progress.report(&format!("Storing record {}/{}", index + 1, record_count));
-                self.store_prepared(record, payment_network_ref, transaction_hash, progress)
-                    .await
-            });
-        let stores =
-            crate::client_engine::bounded_unordered(stores, DEFAULT_BROWSER_STORE_CONCURRENCY)
-                .collect::>()
-                .await;
-        let mut replicas = usize::MAX;
-        for stored in stores {
-            replicas = replicas.min(stored?);
-        }
+        let replicas = self
+            .store_prepared_records(
+                &prepared,
+                &payment_network,
+                payment.transaction_hash.as_deref(),
+                progress,
+            )
+            .await?;
         let descriptor = PublicFileDescriptor {
             name: name.to_string(),
             address: encrypted.address,
@@ -1716,7 +1709,7 @@ impl BrowserNetworkClient {
             blake3: encrypted.blake3,
             data_map_size: encrypted.data_map_size,
             chunks: encrypted.chunks,
-            replicas: if replicas == usize::MAX { 0 } else { replicas },
+            replicas,
         };
         Ok(BrowserUploadResult {
             file: descriptor,
@@ -1743,7 +1736,7 @@ impl BrowserNetworkClient {
                     endpoint,
                 })
             })
-            .take(MAX_STORE_TARGETS)
+            .take(CLOSE_GROUP_SIZE)
             .collect::>();
         if targets.is_empty() {
             return Err(
@@ -1806,7 +1799,107 @@ impl BrowserNetworkClient {
         ))
     }
 
-    async fn store_prepared(
+    /// Store every paid record with the same adaptive, byte-bounded retry
+    /// rounds used by the native client.
+    async fn store_prepared_records(
+        &self,
+        prepared: &[PreparedRecord],
+        payment_network: &BrowserPaymentNetwork,
+        transaction_hash: Option<&str>,
+        progress: &ProgressReporter,
+    ) -> Result {
+        let record_count = prepared.len();
+        let max_record_bytes = prepared
+            .iter()
+            .map(|record| record.record.content.len())
+            .max()
+            .unwrap_or(0);
+        let byte_bound = crate::client_engine::store_byte_bound(max_record_bytes);
+        let mut to_retry = prepared.iter().enumerate().collect::>();
+        let mut replicas = usize::MAX;
+
+        for attempt in 0..=crate::client_engine::STORE_MAX_RETRIES {
+            if attempt > 0 {
+                let delay = crate::client_engine::store_retry_delay(attempt);
+                progress.report(&format!(
+                    "Retrying {} record(s), attempt {attempt}/{}",
+                    to_retry.len(),
+                    crate::client_engine::STORE_MAX_RETRIES
+                ));
+                TimeoutFuture::new(u32::try_from(delay.as_millis()).unwrap_or(u32::MAX)).await;
+            }
+
+            let op_limiter = self.controller.store.clone();
+            let cap_limiter = op_limiter.clone();
+            let results = crate::client_engine::rolling_unordered(
+                to_retry,
+                |(index, record)| {
+                    let limiter = op_limiter.clone();
+                    async move {
+                        progress.report(&format!(
+                            "Storing record {}/{} (attempt {}/{})",
+                            index + 1,
+                            record_count,
+                            attempt + 1,
+                            crate::client_engine::STORE_MAX_RETRIES + 1
+                        ));
+                        let result = observe_op(
+                            &limiter,
+                            || {
+                                self.store_prepared_once(
+                                    record,
+                                    payment_network,
+                                    transaction_hash,
+                                    progress,
+                                )
+                            },
+                            |error| classify_browser_store_error(error),
+                        )
+                        .await;
+                        ((index, record), result)
+                    }
+                },
+                || cap_limiter.current().min(byte_bound),
+            )
+            .await;
+
+            let mut failed = Vec::new();
+            for ((index, record), result) in results {
+                match result {
+                    Ok(stored) => replicas = replicas.min(stored),
+                    Err(error) => failed.push((index, record, error)),
+                }
+            }
+            if failed.is_empty() {
+                return Ok(if replicas == usize::MAX { 0 } else { replicas });
+            }
+            if attempt == crate::client_engine::STORE_MAX_RETRIES {
+                let failed_count = failed.len();
+                let details = failed
+                    .into_iter()
+                    .map(|(index, _, error)| {
+                        format!("record {}/{}: {error}", index + 1, record_count)
+                    })
+                    .collect::>()
+                    .join("; ");
+                return Err(format!(
+                    "{} paid record(s) failed after {} attempts: {details}",
+                    failed_count,
+                    crate::client_engine::STORE_MAX_RETRIES + 1
+                ));
+            }
+            to_retry = failed
+                .into_iter()
+                .map(|(index, record, _)| (index, record))
+                .collect();
+        }
+
+        Err("record store retry loop ended unexpectedly".to_string())
+    }
+
+    /// Store one record to a close-group majority, advancing through the rest
+    /// of the ordered K=7 target set only when an initial target fails.
+    async fn store_prepared_once(
         &self,
         prepared: &PreparedRecord,
         payment_network: &BrowserPaymentNetwork,
@@ -1823,46 +1916,71 @@ impl BrowserNetworkClient {
             .verified
             .as_ref()
             .ok_or_else(|| "paid record has no verified quote".to_string())?;
-        let attempts = prepared.targets.iter().cloned().map(|target| {
-            let pool = Rc::clone(&self.inner.pool);
-            let record = prepared.record.clone();
-            let quote = verified.quote.clone();
-            let payment_network = payment_network.clone();
-            let transaction_hash = transaction_hash.clone();
-            let progress = progress.clone();
-            async move {
-                let client = pool.client(&target.endpoint).await?;
-                let hello = client.hello().await?;
-                assert_upload_node(&hello, &payment_network)?;
-                let (_, already_stored) = client
-                    .put_chunk(&record.address, &record.content, quote, &transaction_hash)
-                    .await?;
-                progress.report(&format!(
-                    "{} {} on {}",
-                    if already_stored {
-                        "Confirmed"
-                    } else {
-                        "Stored"
-                    },
-                    record.address,
-                    target.peer_id
-                ));
-                Ok::<(), String>(())
-            }
-        });
-        let attempts = join_all(attempts).await;
-        let stored = attempts.iter().filter(|attempt| attempt.is_ok()).count();
-        if stored == 0 {
-            let failures = attempts
-                .into_iter()
-                .filter_map(Result::err)
-                .collect::>()
-                .join("; ");
+        let outcome = crate::client_engine::quorum_with_fallback(
+            prepared.targets.iter().cloned(),
+            CLOSE_GROUP_MAJORITY,
+            |target| {
+                let pool = Rc::clone(&self.inner.pool);
+                let record = prepared.record.clone();
+                let quote = verified.quote.clone();
+                let payment_network = payment_network.clone();
+                let transaction_hash = transaction_hash.clone();
+                let progress = progress.clone();
+                async move {
+                    let client = pool.client(&target.endpoint).await?;
+                    let hello = client.hello().await?;
+                    assert_upload_node(&hello, &payment_network)?;
+                    let (_, already_stored) = client
+                        .put_chunk(&record.address, &record.content, quote, &transaction_hash)
+                        .await?;
+                    progress.report(&format!(
+                        "{} {} on {}",
+                        if already_stored {
+                            "Confirmed"
+                        } else {
+                            "Stored"
+                        },
+                        record.address,
+                        target.peer_id
+                    ));
+                    Ok::<(), String>(())
+                }
+            },
+        )
+        .await;
+        let failures = outcome
+            .failures
+            .into_iter()
+            .map(|(target, error)| {
+                progress.report(&format!("Store target {} failed: {error}", target.peer_id));
+                format!("{}: {error}", target.peer_id)
+            })
+            .collect::>();
+        if !outcome.reached {
             return Err(format!(
-                "paid chunk was rejected by every closest node: {failures}"
+                "stored on {} peers, need {CLOSE_GROUP_MAJORITY}; failures: {}",
+                outcome.successes,
+                failures.join("; ")
             ));
         }
-        Ok(stored)
+        Ok(outcome.successes)
+    }
+}
+
+fn classify_browser_store_error(error: &str) -> Outcome {
+    let error = error.to_ascii_lowercase();
+    if error.contains("timed out") || error.contains("timeout") {
+        Outcome::Timeout
+    } else if error.contains("webrtc")
+        || error.contains("datachannel")
+        || error.contains("ice")
+        || error.contains("connect")
+        || error.contains("closed")
+        || error.contains("invalid state")
+    {
+        Outcome::NetworkError
+    } else {
+        Outcome::ApplicationError
     }
 }
 
diff --git a/ant-core/src/client_engine.rs b/ant-core/src/client_engine.rs
index 6ffbfdad..9df23a98 100644
--- a/ant-core/src/client_engine.rs
+++ b/ant-core/src/client_engine.rs
@@ -1,13 +1,159 @@
 //! Runtime-neutral scheduling and session state shared by native and browser clients.
 
-use futures_util::{stream, Stream, StreamExt as _};
+use futures_util::{stream, stream::FuturesUnordered, Stream, StreamExt as _};
 #[cfg(any(feature = "browser-wasm", test))]
 use std::collections::HashMap;
 use std::future::Future;
 #[cfg(any(feature = "browser-wasm", test))]
 use std::hash::Hash;
+use std::time::Duration;
 #[cfg(any(feature = "browser-wasm", test))]
-use std::time::{Duration, Instant};
+use std::time::Instant;
+
+#[cfg_attr(
+    all(feature = "browser-wasm", not(feature = "native")),
+    allow(dead_code)
+)]
+#[path = "data/client/adaptive.rs"]
+pub(crate) mod adaptive;
+
+/// Maximum combined source-record bytes scheduled for concurrent storage.
+///
+/// A record is sent to several close-group peers, so its actual wire footprint
+/// is larger than its source body. Keeping the shared budget expressed in
+/// source bytes lets native QUIC and browser WebRTC use the same conservative
+/// scheduling policy without coupling it to either transport.
+pub(crate) const STORE_INFLIGHT_BYTE_BUDGET: usize = 64 * 1024 * 1024;
+
+/// Number of whole-record store retries after the first attempt.
+pub(crate) const STORE_MAX_RETRIES: u32 = 3;
+
+/// Initial delay for exponential whole-record store retries.
+pub(crate) const STORE_RETRY_BASE_DELAY_MS: u64 = 500;
+
+/// Outcome of a quorum operation over an ordered target set.
+#[derive(Debug)]
+pub(crate) struct QuorumOutcome {
+    pub(crate) successes: usize,
+    pub(crate) failures: Vec<(T, E)>,
+    pub(crate) reached: bool,
+}
+
+/// Run an operation against the first `required` targets concurrently, using
+/// later targets one-for-one as fallbacks when an attempt fails.
+///
+/// The function returns as soon as quorum is reached and drops any remaining
+/// in-flight work. This is the transport-neutral close-group delivery policy
+/// shared by native QUIC and browser WebRTC uploads.
+pub(crate) async fn quorum_with_fallback(
+    targets: impl IntoIterator,
+    required: usize,
+    operation: F,
+) -> QuorumOutcome
+where
+    T: Clone,
+    F: Fn(T) -> Fut,
+    Fut: Future>,
+{
+    if required == 0 {
+        return QuorumOutcome {
+            successes: 0,
+            failures: Vec::new(),
+            reached: true,
+        };
+    }
+
+    let mut targets = targets.into_iter();
+    let launch = |target: T| {
+        let future = operation(target.clone());
+        async move { (target, future.await) }
+    };
+    let mut in_flight = FuturesUnordered::new();
+    for target in targets.by_ref().take(required) {
+        in_flight.push(launch(target));
+    }
+
+    let mut successes = 0usize;
+    let mut failures = Vec::new();
+    while let Some((target, result)) = in_flight.next().await {
+        match result {
+            Ok(_) => {
+                successes += 1;
+                if successes >= required {
+                    return QuorumOutcome {
+                        successes,
+                        failures,
+                        reached: true,
+                    };
+                }
+            }
+            Err(error) => {
+                failures.push((target, error));
+                if let Some(fallback) = targets.next() {
+                    in_flight.push(launch(fallback));
+                }
+            }
+        }
+    }
+
+    QuorumOutcome {
+        successes,
+        failures,
+        reached: false,
+    }
+}
+
+/// Run all items with a rolling concurrency window whose cap is re-read after
+/// every completion.
+///
+/// Unlike [`bounded_unordered`], this collects every result and therefore fits
+/// retry rounds: one failed item does not prevent untouched siblings from
+/// being attempted. The cap callback lets callers combine the shared adaptive
+/// limiter with a payload-byte ceiling.
+pub(crate) async fn rolling_unordered(
+    items: I,
+    mut operation: F,
+    current_cap: C,
+) -> Vec
+where
+    I: IntoIterator,
+    F: FnMut(I::Item) -> Fut,
+    Fut: Future,
+    C: Fn() -> usize,
+{
+    let mut items = items.into_iter();
+    let mut in_flight = FuturesUnordered::new();
+    let mut results = Vec::new();
+    loop {
+        let cap = current_cap().max(1);
+        while in_flight.len() < cap {
+            match items.next() {
+                Some(item) => in_flight.push(operation(item)),
+                None => break,
+            }
+        }
+        let Some(result) = in_flight.next().await else {
+            break;
+        };
+        results.push(result);
+    }
+    results
+}
+
+/// Limit concurrent record stores by the shared source-body byte budget.
+#[must_use]
+pub(crate) fn store_byte_bound(max_record_bytes: usize) -> usize {
+    STORE_INFLIGHT_BYTE_BUDGET
+        .checked_div(max_record_bytes)
+        .map_or(usize::MAX, |bound| bound.max(1))
+}
+
+/// Exponential delay for retry round `attempt`, where attempt 1 is the first
+/// retry after the initial operation.
+#[must_use]
+pub(crate) fn store_retry_delay(attempt: u32) -> Duration {
+    Duration::from_millis(STORE_RETRY_BASE_DELAY_MS * 2u64.pow(attempt.saturating_sub(1)))
+}
 
 /// Run futures with a bounded rolling concurrency window.
 ///
@@ -98,6 +244,8 @@ where
 #[cfg(test)]
 mod tests {
     use super::*;
+    use std::cell::Cell;
+    use std::rc::Rc;
 
     #[test]
     fn changed_endpoint_bypasses_failure_cooldown() {
@@ -143,4 +291,67 @@ mod tests {
         assert!(outputs.contains(&1));
         assert!(outputs.contains(&2));
     }
+
+    #[test]
+    fn quorum_starts_only_the_required_targets() {
+        let launched = Rc::new(Cell::new(0usize));
+        let outcome = futures::executor::block_on({
+            let launched = Rc::clone(&launched);
+            async move {
+                quorum_with_fallback(0_u8..7, 4, move |_| {
+                    launched.set(launched.get() + 1);
+                    futures_util::future::ready(Ok::<(), ()>(()))
+                })
+                .await
+            }
+        });
+
+        assert!(outcome.reached);
+        assert_eq!(outcome.successes, 4);
+        assert!(outcome.failures.is_empty());
+        assert_eq!(launched.get(), 4);
+    }
+
+    #[test]
+    fn quorum_advances_through_fallbacks_after_failures() {
+        let launched = Rc::new(Cell::new(0usize));
+        let outcome = futures::executor::block_on({
+            let launched = Rc::clone(&launched);
+            async move {
+                quorum_with_fallback(0_u8..7, 4, move |target| {
+                    launched.set(launched.get() + 1);
+                    futures_util::future::ready(if target < 2 { Err(target) } else { Ok(()) })
+                })
+                .await
+            }
+        });
+
+        assert!(outcome.reached);
+        assert_eq!(outcome.successes, 4);
+        assert_eq!(outcome.failures.len(), 2);
+        assert_eq!(launched.get(), 6);
+    }
+
+    #[test]
+    fn quorum_reports_exhausted_target_set() {
+        let outcome = futures::executor::block_on(async {
+            quorum_with_fallback(0_u8..7, 4, |target| {
+                futures_util::future::ready(Err::<(), _>(target))
+            })
+            .await
+        });
+
+        assert!(!outcome.reached);
+        assert_eq!(outcome.successes, 0);
+        assert_eq!(outcome.failures.len(), 7);
+    }
+
+    #[test]
+    fn store_byte_bound_and_retry_schedule_match_native_policy() {
+        assert_eq!(store_byte_bound(4 * 1024 * 1024), 16);
+        assert_eq!(store_byte_bound(0), usize::MAX);
+        assert_eq!(store_retry_delay(1), Duration::from_millis(500));
+        assert_eq!(store_retry_delay(2), Duration::from_secs(1));
+        assert_eq!(store_retry_delay(3), Duration::from_secs(2));
+    }
 }
diff --git a/ant-core/src/data/client/adaptive.rs b/ant-core/src/data/client/adaptive.rs
index cc4e5153..a586f33b 100644
--- a/ant-core/src/data/client/adaptive.rs
+++ b/ant-core/src/data/client/adaptive.rs
@@ -47,18 +47,24 @@
 //!   removed from saorsa-core; this controller only tunes client
 //!   concurrency.
 
-use futures::stream::{self, FuturesUnordered, StreamExt};
+use futures_util::stream::{self, FuturesUnordered, StreamExt};
 use serde::{Deserialize, Serialize};
 use std::collections::VecDeque;
+#[cfg(feature = "native")]
 use std::path::{Path, PathBuf};
+#[cfg(feature = "native")]
 use std::sync::atomic::{AtomicU64, Ordering};
 use std::sync::{Arc, Mutex, PoisonError};
-use std::time::{Duration, Instant};
-use tracing::{debug, warn};
+use std::time::Duration;
+use tracing::debug;
+#[cfg(feature = "native")]
+use tracing::warn;
+use web_time::Instant;
 
 /// Process-monotonic counter for unique snapshot temp filenames.
 /// Combined with PID + nanosecond timestamp, makes collision
 /// effectively impossible across concurrent save_snapshot calls.
+#[cfg(feature = "native")]
 static SAVE_COUNTER: AtomicU64 = AtomicU64::new(0);
 
 /// Fetch starts at the residential-saturation floor validated in
@@ -1480,20 +1486,25 @@ where
 /// can evolve the controller without crashing on stale files — an
 /// unknown future schema version simply causes a silent fallback to
 /// cold defaults.
+#[cfg(feature = "native")]
 #[derive(Debug, Clone, Serialize, Deserialize)]
 struct PersistedState {
     schema: u32,
     channels: ChannelStart,
 }
 
+#[cfg(feature = "native")]
 const PERSIST_SCHEMA: u32 = 2;
+#[cfg(feature = "native")]
 const PERSIST_SCHEMA_AIMD_FETCH: u32 = 1;
+#[cfg(feature = "native")]
 const PERSIST_FILENAME: &str = "client_adaptive.json";
 
 /// Default persistence path: `/client_adaptive.json`. Falls
 /// back to `None` if the platform data dir is not resolvable; in that
 /// case the controller still works, it just won't persist.
 #[must_use]
+#[cfg(feature = "native")]
 pub fn default_persist_path() -> Option {
     crate::config::data_dir()
         .ok()
@@ -1506,6 +1517,7 @@ pub fn default_persist_path() -> Option {
 /// effort — never propagate errors that would block the user's
 /// operation.
 #[must_use]
+#[cfg(feature = "native")]
 pub fn load_snapshot(path: &Path) -> Option {
     let bytes = std::fs::read(path).ok()?;
     let state: PersistedState = match serde_json::from_slice(&bytes) {
@@ -1541,6 +1553,7 @@ pub fn load_snapshot(path: &Path) -> Option {
 
 /// Save a snapshot to disk atomically (write to `.tmp`, then
 /// rename). Best effort — failures are logged at warn and discarded.
+#[cfg(feature = "native")]
 pub fn save_snapshot(path: &Path, channels: ChannelStart) {
     let state = PersistedState {
         schema: PERSIST_SCHEMA,
@@ -1602,6 +1615,7 @@ pub fn save_snapshot(path: &Path, channels: ChannelStart) {
 ///
 /// Used by `Client::drop` so a stalled filesystem cannot block
 /// process shutdown indefinitely.
+#[cfg(feature = "native")]
 pub fn save_snapshot_with_timeout(path: PathBuf, channels: ChannelStart, timeout: Duration) {
     let handle = std::thread::spawn(move || {
         save_snapshot(&path, channels);
diff --git a/ant-core/src/data/client/batch.rs b/ant-core/src/data/client/batch.rs
index a739fb4c..b1b73d43 100644
--- a/ant-core/src/data/client/batch.rs
+++ b/ant-core/src/data/client/batch.rs
@@ -20,7 +20,7 @@ use ant_protocol::payment::{
 use ant_protocol::transport::{MultiAddr, PeerId};
 use ant_protocol::{compute_address, XorName, CLOSE_GROUP_SIZE, DATA_TYPE_CHUNK};
 use bytes::Bytes;
-use futures::stream::{FuturesUnordered, StreamExt};
+use futures::stream::StreamExt;
 use std::collections::{HashMap, HashSet};
 use std::time::{Duration, Instant};
 use tokio::sync::mpsc;
@@ -29,13 +29,6 @@ use tracing::{debug, info, warn};
 /// Number of chunks per payment wave.
 const PAYMENT_WAVE_SIZE: usize = 64;
 
-/// Soft ceiling on the combined body size of chunks stored concurrently in a
-/// single wave. Caps store concurrency for large chunks so the send path's
-/// per-peer body buffers can't pin multiple GB at once (see V2-461). At ~4 MB
-/// chunks this permits ~16 concurrent stores; small chunks hit the chunk-count
-/// / adaptive limits instead and are unaffected.
-const STORE_INFLIGHT_BYTE_BUDGET: usize = 64 * 1024 * 1024;
-
 /// Variable-size single-node payment plan for a chunk.
 ///
 /// The shared `ant-protocol::payment::SingleNodePayment` helper still models
@@ -849,9 +842,6 @@ impl Client {
         stored_before: usize,
         total_chunks: usize,
     ) -> WaveResult {
-        const MAX_RETRIES: u32 = 3;
-        const BASE_DELAY_MS: u64 = 500;
-
         let mut stored = Vec::new();
         let mut to_retry = paid_chunks;
 
@@ -875,20 +865,18 @@ impl Client {
         let max_chunk_bytes = to_retry.iter().map(|c| c.content.len()).max().unwrap_or(0);
         // `checked_div` yields `None` only when `max_chunk_bytes == 0` (an
         // empty/zero-length wave), in which case there is no byte limit.
-        let byte_bound = STORE_INFLIGHT_BYTE_BUDGET
-            .checked_div(max_chunk_bytes)
-            .map_or(usize::MAX, |n| n.max(1));
+        let byte_bound = crate::client_engine::store_byte_bound(max_chunk_bytes);
 
         let mut chunk_attempts_total: usize = 0;
         let mut store_durations_ms: Vec = Vec::new();
         let mut retries_per_chunk: Vec = Vec::new();
 
-        for attempt in 0..=MAX_RETRIES {
+        for attempt in 0..=crate::client_engine::STORE_MAX_RETRIES {
             if attempt > 0 {
-                let delay = Duration::from_millis(BASE_DELAY_MS * 2u64.pow(attempt - 1));
-                tokio::time::sleep(delay).await;
+                tokio::time::sleep(crate::client_engine::store_retry_delay(attempt)).await;
                 info!(
-                    "Retry attempt {attempt}/{MAX_RETRIES} for {} chunks",
+                    "Retry attempt {attempt}/{} for {} chunks",
+                    crate::client_engine::STORE_MAX_RETRIES,
                     to_retry.len()
                 );
             }
@@ -924,21 +912,12 @@ impl Client {
                     (chunk_clone, result)
                 }
             };
-            let mut chunk_iter = to_retry.into_iter();
-            let mut in_flight = FuturesUnordered::new();
-
             let mut failed_this_round = Vec::new();
-            loop {
-                let slots = store_limiter.current().min(byte_bound).max(1);
-                while in_flight.len() < slots {
-                    match chunk_iter.next() {
-                        Some(chunk) => in_flight.push(make_store(chunk)),
-                        None => break,
-                    }
-                }
-                let Some((chunk, result)) = in_flight.next().await else {
-                    break;
-                };
+            let results = crate::client_engine::rolling_unordered(to_retry, make_store, || {
+                store_limiter.current().min(byte_bound)
+            })
+            .await;
+            for (chunk, result) in results {
                 match result {
                     Ok(name) => {
                         let duration_ms = first_seen
@@ -975,7 +954,7 @@ impl Client {
                 return result;
             }
 
-            if attempt == MAX_RETRIES {
+            if attempt == crate::client_engine::STORE_MAX_RETRIES {
                 let failed = failed_this_round
                     .into_iter()
                     .map(|(c, e)| (c.address, e))
diff --git a/ant-core/src/data/client/chunk.rs b/ant-core/src/data/client/chunk.rs
index 4f9fd338..e6b7f0c5 100644
--- a/ant-core/src/data/client/chunk.rs
+++ b/ant-core/src/data/client/chunk.rs
@@ -16,7 +16,7 @@ use ant_protocol::{
     ProofType, ProtocolError, XorName, CLOSE_GROUP_MAJORITY,
 };
 use bytes::Bytes;
-use futures::stream::{self, FuturesUnordered, StreamExt};
+use futures::stream::{self, StreamExt};
 use std::collections::HashMap;
 use std::time::{Duration, Instant};
 use tracing::{debug, info, warn};
@@ -409,21 +409,17 @@ impl Client {
     ) -> Result {
         let address = compute_address(&content);
 
-        let initial_count = peers.len().min(CLOSE_GROUP_MAJORITY);
-        let (initial_peers, fallback_peers) = peers.split_at(initial_count);
-        let mut fallback_iter = fallback_peers.iter();
-
-        let mut put_futures = FuturesUnordered::new();
-        for (peer_id, addrs) in initial_peers {
-            put_futures.push(self.spawn_chunk_put(
-                content.clone(),
-                proof.clone(),
-                *peer_id,
-                addrs.clone(),
-            ));
-        }
-
-        let mut success_count = 0usize;
+        let outcome = crate::client_engine::quorum_with_fallback(
+            peers.iter().cloned(),
+            CLOSE_GROUP_MAJORITY,
+            |(peer_id, addrs)| {
+                let content = content.clone();
+                let proof = proof.clone();
+                async move { self.spawn_chunk_put(content, proof, peer_id, addrs).await.1 }
+            },
+        )
+        .await;
+        let success_count = outcome.successes;
         let mut failures: Vec = Vec::new();
         // Tally the *cause* of each failure. The store AIMD limiter must only be
         // pushed down by a transport shortfall (V2-468): a node that responds —
@@ -439,55 +435,33 @@ impl Client {
         let mut dial = 0usize;
         let mut first_app_rejection: Option = None;
 
-        while let Some((peer_id, result)) = put_futures.next().await {
-            match result {
-                Ok(_) => {
-                    success_count += 1;
-                    if success_count >= CLOSE_GROUP_MAJORITY {
-                        debug!(
-                            "Chunk {} stored on {success_count} peers (majority reached)",
-                            hex::encode(address)
-                        );
-                        return Ok(address);
-                    }
-                }
-                Err(e) => {
-                    warn!("Failed to store chunk on {peer_id}: {e}");
-                    failures.push(format!("{peer_id}: {e}"));
-                    match classify_put_failure(&e) {
-                        PutRejection::Full => full += 1,
-                        PutRejection::PriceFloor => price_floor += 1,
-                        PutRejection::OtherRemote => other_remote += 1,
-                        PutRejection::Timeout => timeout += 1,
-                        PutRejection::Dial => dial += 1,
-                    }
-                    // An application-level decline is `RemotePut` (a structured
-                    // node rejection) or `Error::Payment` (`PaymentRequired`):
-                    // capture the first so an all-application shortfall surfaces
-                    // as `ApplicationError`, not `InsufficientPeers`
-                    // (`NetworkError`), and never suppresses the limiter.
-                    if matches!(e, Error::RemotePut { .. } | Error::Payment(_))
-                        && first_app_rejection.is_none()
-                    {
-                        first_app_rejection = Some(e);
-                    }
-
-                    // Advance to the next peer in the put-target set, reusing
-                    // the same proof.
-                    if let Some((fb_peer, fb_addrs)) = fallback_iter.next() {
-                        debug!(
-                            "Falling back to peer {fb_peer} for chunk {}",
-                            hex::encode(address)
-                        );
-                        put_futures.push(self.spawn_chunk_put(
-                            content.clone(),
-                            proof.clone(),
-                            *fb_peer,
-                            fb_addrs.clone(),
-                        ));
-                    }
-                }
+        for ((peer_id, _), error) in outcome.failures {
+            warn!("Failed to store chunk on {peer_id}: {error}");
+            failures.push(format!("{peer_id}: {error}"));
+            match classify_put_failure(&error) {
+                PutRejection::Full => full += 1,
+                PutRejection::PriceFloor => price_floor += 1,
+                PutRejection::OtherRemote => other_remote += 1,
+                PutRejection::Timeout => timeout += 1,
+                PutRejection::Dial => dial += 1,
             }
+            // An application-level decline is `RemotePut` (a structured node
+            // rejection) or `Error::Payment` (`PaymentRequired`): capture the
+            // first so an all-application shortfall surfaces as
+            // `ApplicationError`, not `InsufficientPeers` (`NetworkError`).
+            if matches!(error, Error::RemotePut { .. } | Error::Payment(_))
+                && first_app_rejection.is_none()
+            {
+                first_app_rejection = Some(error);
+            }
+        }
+
+        if outcome.reached {
+            debug!(
+                "Chunk {} stored on {success_count} peers (majority reached)",
+                hex::encode(address)
+            );
+            return Ok(address);
         }
 
         // Quorum not reached. A timeout-bearing shortfall is genuine local
diff --git a/ant-core/src/data/client/mod.rs b/ant-core/src/data/client/mod.rs
index 7a56edff..d627f81c 100644
--- a/ant-core/src/data/client/mod.rs
+++ b/ant-core/src/data/client/mod.rs
@@ -3,7 +3,9 @@
 //! Provides high-level APIs for storing and retrieving data
 //! on the Autonomi decentralized network.
 
-pub mod adaptive;
+pub mod adaptive {
+    pub use crate::client_engine::adaptive::*;
+}
 pub mod batch;
 pub mod cache;
 pub(crate) mod cached_merkle;
diff --git a/web/README.md b/web/README.md
index 26cdfed3..e12cf458 100644
--- a/web/README.md
+++ b/web/README.md
@@ -28,6 +28,13 @@ Nodes serialize these addresses from the native `saorsa_core::MultiAddr`
 representation; the shared Rust client parser consumes that canonical string
 form in WASM.
 
+Native QUIC and browser WebRTC uploads use the same `ant-core` scheduling
+policy: the adaptive store limiter, a 64 MiB in-flight source-record budget,
+four-of-seven close-group quorum with one-for-one fallback targets, and three
+whole-record retries with 500 ms, 1 s, and 2 s backoff. The browser transport
+adapts WebRTC requests to that shared engine rather than maintaining a separate
+upload algorithm in JavaScript.
+
 Install the WASM build tools once if needed:
 
 ```bash

From 39417b7daa63d29eb1000513577b3a6b6c87c84d Mon Sep 17 00:00:00 2001
From: Mick van Dijke <12992260+mickvandijke@users.noreply.github.com>
Date: Mon, 31 Aug 2026 15:01:36 +0200
Subject: [PATCH 13/31] fix(browser): harden WebRTC upload retries

---
 ant-core/src/browser/wasm_transport.rs | 259 +++++++++++++++++++------
 ant-core/src/client_engine.rs          |  30 +++
 2 files changed, 232 insertions(+), 57 deletions(-)

diff --git a/ant-core/src/browser/wasm_transport.rs b/ant-core/src/browser/wasm_transport.rs
index bb79e308..0c100ac0 100644
--- a/ant-core/src/browser/wasm_transport.rs
+++ b/ant-core/src/browser/wasm_transport.rs
@@ -16,6 +16,7 @@ use super::protocol::{
 use crate::client_engine::adaptive::{
     observe_op, AdaptiveConfig, AdaptiveController, ChannelStart, Outcome,
 };
+use ant_protocol::web_rtc::transfer_timeout;
 use ant_protocol::{CLOSE_GROUP_MAJORITY, CLOSE_GROUP_SIZE};
 use futures_channel::{mpsc, oneshot};
 use futures_util::{
@@ -455,26 +456,43 @@ impl BrowserNodeClientCore {
         self.next_request_id.set(request_id.wrapping_add(1).max(1));
         let frame = encode_request_frame(request_id, request_type, fields, content)
             .map_err(|error| error.to_string())?;
-        let channel = self
-            .connection
-            .borrow()
-            .as_ref()
-            .map(|connection| connection.data_channel.clone())
-            .ok_or_else(|| "WebRTC DataChannel is not connected".to_string())?;
-        for message in frame.chunks(WEBRTC_WRITE_CHUNK_BYTES) {
-            wait_for_capacity(&channel).await?;
-            channel
-                .send_with_u8_array(message)
-                .map_err(js_error_message)?;
-        }
-        let receiver = self
-            .connection
-            .borrow()
-            .as_ref()
-            .map(|connection| Rc::clone(&connection.inbox))
-            .ok_or_else(|| "WebRTC response inbox is unavailable".to_string())?;
-        let response = timeout(read_response(receiver), "WebRTC request timed out").await;
-        let response = match response {
+        let transfer_timeout_ms = transfer_timeout_ms(frame.len());
+        let channel = {
+            let connection = self.connection.borrow();
+            connection
+                .as_ref()
+                .map(|connection| connection.data_channel.clone())
+        };
+        let Some(channel) = channel else {
+            self.close();
+            return Err("WebRTC DataChannel is not connected".to_string());
+        };
+        let send_deadline_ms = js_sys::Date::now() + f64::from(transfer_timeout_ms);
+        let send_result = async {
+            for message in frame.chunks(WEBRTC_WRITE_CHUNK_BYTES) {
+                wait_for_capacity(&channel, remaining_timeout_ms(send_deadline_ms)).await?;
+                channel
+                    .send_with_u8_array(message)
+                    .map_err(js_error_message)?;
+            }
+            Ok::<(), String>(())
+        }
+        .await;
+        if let Err(error) = send_result {
+            self.close();
+            return Err(error);
+        }
+        let receiver = {
+            let connection = self.connection.borrow();
+            connection
+                .as_ref()
+                .map(|connection| Rc::clone(&connection.inbox))
+        };
+        let Some(receiver) = receiver else {
+            self.close();
+            return Err("WebRTC response inbox is unavailable".to_string());
+        };
+        let response = match read_response(receiver, transfer_timeout_ms).await {
             Ok(response) => response,
             Err(error) => {
                 self.close();
@@ -490,12 +508,18 @@ impl BrowserNodeClientCore {
             return Err(error);
         }
         if response.header.get("status").and_then(Value::as_str) == Some("error") {
-            return Err(response
+            let authentication_required = response.header.get("code").and_then(Value::as_str)
+                == Some("authentication_required");
+            let error = response
                 .header
                 .get("message")
                 .and_then(Value::as_str)
                 .unwrap_or("node returned an error")
-                .to_string());
+                .to_string();
+            if authentication_required {
+                self.close();
+            }
+            return Err(error);
         }
         Ok(response)
     }
@@ -514,10 +538,20 @@ impl BrowserNodeClientCore {
         let mut fields = Map::new();
         fields.insert("challenge".to_string(), Value::from(hex::encode(challenge)));
         let response = self.request("hello", fields, &[]).await?;
-        let hello: BrowserHello = serde_json::from_value(response.header)
-            .map_err(|error| format!("invalid HELLO response: {error}"))?;
-        let peer_id = verify_hello_identity(&hello, &self.endpoint, &challenge)
-            .map_err(|error| error.to_string())?;
+        let hello: BrowserHello = match serde_json::from_value(response.header) {
+            Ok(hello) => hello,
+            Err(error) => {
+                self.close();
+                return Err(format!("invalid HELLO response: {error}"));
+            }
+        };
+        let peer_id = match verify_hello_identity(&hello, &self.endpoint, &challenge) {
+            Ok(peer_id) => peer_id,
+            Err(error) => {
+                self.close();
+                return Err(error.to_string());
+            }
+        };
         self.peer_id.replace(Some(peer_id));
         self.hello.replace(Some(hello.clone()));
         Ok(hello)
@@ -1040,6 +1074,26 @@ struct PreparedRecord {
     verified: Option,
 }
 
+struct PendingStoreRecord<'a> {
+    index: usize,
+    record: &'a PreparedRecord,
+    successful_peers: HashSet,
+}
+
+struct StoreAttemptError {
+    successful_peers: HashSet,
+    message: String,
+}
+
+impl StoreAttemptError {
+    fn new(successful_peers: HashSet, message: impl Into) -> Self {
+        Self {
+            successful_peers,
+            message: message.into(),
+        }
+    }
+}
+
 #[derive(Debug, Deserialize)]
 struct BrowserPaymentSubmission {
     #[serde(rename = "transactionHash")]
@@ -1815,7 +1869,15 @@ impl BrowserNetworkClient {
             .max()
             .unwrap_or(0);
         let byte_bound = crate::client_engine::store_byte_bound(max_record_bytes);
-        let mut to_retry = prepared.iter().enumerate().collect::>();
+        let mut to_retry = prepared
+            .iter()
+            .enumerate()
+            .map(|(index, record)| PendingStoreRecord {
+                index,
+                record,
+                successful_peers: HashSet::new(),
+            })
+            .collect::>();
         let mut replicas = usize::MAX;
 
         for attempt in 0..=crate::client_engine::STORE_MAX_RETRIES {
@@ -1833,7 +1895,12 @@ impl BrowserNetworkClient {
             let cap_limiter = op_limiter.clone();
             let results = crate::client_engine::rolling_unordered(
                 to_retry,
-                |(index, record)| {
+                |pending| {
+                    let PendingStoreRecord {
+                        index,
+                        record,
+                        successful_peers,
+                    } = pending;
                     let limiter = op_limiter.clone();
                     async move {
                         progress.report(&format!(
@@ -1851,9 +1918,10 @@ impl BrowserNetworkClient {
                                     payment_network,
                                     transaction_hash,
                                     progress,
+                                    successful_peers,
                                 )
                             },
-                            |error| classify_browser_store_error(error),
+                            |error| classify_browser_store_error(&error.message),
                         )
                         .await;
                         ((index, record), result)
@@ -1867,7 +1935,14 @@ impl BrowserNetworkClient {
             for ((index, record), result) in results {
                 match result {
                     Ok(stored) => replicas = replicas.min(stored),
-                    Err(error) => failed.push((index, record, error)),
+                    Err(error) => failed.push((
+                        PendingStoreRecord {
+                            index,
+                            record,
+                            successful_peers: error.successful_peers,
+                        },
+                        error.message,
+                    )),
                 }
             }
             if failed.is_empty() {
@@ -1877,8 +1952,8 @@ impl BrowserNetworkClient {
                 let failed_count = failed.len();
                 let details = failed
                     .into_iter()
-                    .map(|(index, _, error)| {
-                        format!("record {}/{}: {error}", index + 1, record_count)
+                    .map(|(pending, error)| {
+                        format!("record {}/{}: {error}", pending.index + 1, record_count)
                     })
                     .collect::>()
                     .join("; ");
@@ -1888,10 +1963,7 @@ impl BrowserNetworkClient {
                     crate::client_engine::STORE_MAX_RETRIES + 1
                 ));
             }
-            to_retry = failed
-                .into_iter()
-                .map(|(index, record, _)| (index, record))
-                .collect();
+            to_retry = failed.into_iter().map(|(pending, _)| pending).collect();
         }
 
         Err("record store retry loop ended unexpectedly".to_string())
@@ -1905,20 +1977,32 @@ impl BrowserNetworkClient {
         payment_network: &BrowserPaymentNetwork,
         transaction_hash: Option<&str>,
         progress: &ProgressReporter,
-    ) -> Result {
+        mut successful_peers: HashSet,
+    ) -> Result {
         if prepared.already_stored {
             return Ok(1);
         }
-        let transaction_hash = transaction_hash
-            .ok_or_else(|| "paid record has no transaction hash".to_string())?
-            .to_string();
-        let verified = prepared
-            .verified
-            .as_ref()
-            .ok_or_else(|| "paid record has no verified quote".to_string())?;
+        let Some(transaction_hash) = transaction_hash else {
+            return Err(StoreAttemptError::new(
+                successful_peers,
+                "paid record has no transaction hash",
+            ));
+        };
+        let transaction_hash = transaction_hash.to_string();
+        let Some(verified) = prepared.verified.as_ref() else {
+            return Err(StoreAttemptError::new(
+                successful_peers,
+                "paid record has no verified quote",
+            ));
+        };
+        let required = CLOSE_GROUP_MAJORITY.saturating_sub(successful_peers.len());
         let outcome = crate::client_engine::quorum_with_fallback(
-            prepared.targets.iter().cloned(),
-            CLOSE_GROUP_MAJORITY,
+            prepared
+                .targets
+                .iter()
+                .filter(|target| !successful_peers.contains(&target.peer_id))
+                .cloned(),
+            required,
             |target| {
                 let pool = Rc::clone(&self.inner.pool);
                 let record = prepared.record.clone();
@@ -1948,6 +2032,10 @@ impl BrowserNetworkClient {
             },
         )
         .await;
+        debug_assert_eq!(outcome.successes, outcome.successful_targets.len());
+        for target in outcome.successful_targets {
+            successful_peers.insert(target.peer_id);
+        }
         let failures = outcome
             .failures
             .into_iter()
@@ -1956,14 +2044,18 @@ impl BrowserNetworkClient {
                 format!("{}: {error}", target.peer_id)
             })
             .collect::>();
-        if !outcome.reached {
-            return Err(format!(
-                "stored on {} peers, need {CLOSE_GROUP_MAJORITY}; failures: {}",
-                outcome.successes,
-                failures.join("; ")
+        if !outcome.reached || successful_peers.len() < CLOSE_GROUP_MAJORITY {
+            let replicas = successful_peers.len();
+            return Err(StoreAttemptError::new(
+                successful_peers,
+                format!(
+                    "stored on {} peers, need {CLOSE_GROUP_MAJORITY}; failures: {}",
+                    replicas,
+                    failures.join("; ")
+                ),
             ));
         }
-        Ok(outcome.successes)
+        Ok(successful_peers.len())
     }
 }
 
@@ -2147,11 +2239,22 @@ impl BrowserNodeClient {
     }
 }
 
-async fn read_response(receiver: ResponseInbox) -> Result {
+async fn read_response(
+    receiver: ResponseInbox,
+    initial_timeout_ms: u32,
+) -> Result {
     let mut frame = Vec::with_capacity(8 * 1024);
     let mut expected_length = None;
+    let response_started_ms = js_sys::Date::now();
+    let mut response_deadline_ms = response_started_ms + f64::from(initial_timeout_ms);
     loop {
-        let next = receiver.lock().await.next().await;
+        let remaining_ms = remaining_timeout_ms(response_deadline_ms);
+        let next = timeout_with_ms(
+            async { Ok(receiver.lock().await.next().await) },
+            "WebRTC request timed out",
+            remaining_ms,
+        )
+        .await?;
         let message = next
             .ok_or_else(|| "response ended before its declared frame was complete".to_string())??;
         let next_length = frame
@@ -2166,6 +2269,10 @@ async fn read_response(receiver: ResponseInbox) -> Result expected {
@@ -2178,7 +2285,10 @@ async fn read_response(receiver: ResponseInbox) -> Result Result<(), String> {
+async fn wait_for_capacity(channel: &RtcDataChannel, timeout_ms: u32) -> Result<(), String> {
+    if channel.ready_state() != RtcDataChannelState::Open {
+        return Err("WebRTC DataChannel closed while draining".to_string());
+    }
     if channel.buffered_amount() <= MAX_BUFFERED_AMOUNT {
         return Ok(());
     }
@@ -2192,13 +2302,22 @@ async fn wait_for_capacity(channel: &RtcDataChannel) -> Result<(), String> {
         }
     });
     channel.set_onbufferedamountlow(Some(on_ready.as_ref().unchecked_ref()));
-    let result = timeout(
+    // The buffer can cross the threshold between the first check and callback
+    // installation. Re-check after installing it so that race cannot turn a
+    // completed drain into a full transfer-timeout wait.
+    if channel.buffered_amount() <= MAX_BUFFERED_AMOUNT {
+        if let Some(sender) = sender.borrow_mut().take() {
+            let _ = sender.send(());
+        }
+    }
+    let result = timeout_with_ms(
         async move {
             receiver
                 .await
                 .map_err(|_| "WebRTC DataChannel closed while draining".to_string())
         },
         "WebRTC DataChannel drain timed out",
+        timeout_ms,
     )
     .await;
     channel.set_onbufferedamountlow(None);
@@ -2207,17 +2326,43 @@ async fn wait_for_capacity(channel: &RtcDataChannel) -> Result<(), String> {
 }
 
 async fn timeout(future: F, message: &'static str) -> Result
+where
+    F: Future>,
+{
+    timeout_with_ms(future, message, REQUEST_TIMEOUT_MS).await
+}
+
+async fn timeout_with_ms(
+    future: F,
+    message: &'static str,
+    timeout_ms: u32,
+) -> Result
 where
     F: Future>,
 {
     let operation = Box::pin(future);
-    let timer = Box::pin(TimeoutFuture::new(REQUEST_TIMEOUT_MS));
+    let timer = Box::pin(TimeoutFuture::new(timeout_ms));
     match select(operation, timer).await {
         Either::Left((result, _)) => result,
         Either::Right(((), _)) => Err(message.to_string()),
     }
 }
 
+fn transfer_timeout_ms(content_bytes: usize) -> u32 {
+    u32::try_from(transfer_timeout(content_bytes).as_millis()).unwrap_or(u32::MAX)
+}
+
+fn remaining_timeout_ms(deadline_ms: f64) -> u32 {
+    let remaining_ms = (deadline_ms - js_sys::Date::now()).ceil();
+    if !remaining_ms.is_finite() || remaining_ms <= 0.0 {
+        0
+    } else if remaining_ms >= f64::from(u32::MAX) {
+        u32::MAX
+    } else {
+        remaining_ms as u32
+    }
+}
+
 fn random_ice_credential() -> Result {
     let mut random = [0u8; 32];
     getrandom::getrandom(&mut random)
diff --git a/ant-core/src/client_engine.rs b/ant-core/src/client_engine.rs
index 9df23a98..c17fa47f 100644
--- a/ant-core/src/client_engine.rs
+++ b/ant-core/src/client_engine.rs
@@ -35,6 +35,7 @@ pub(crate) const STORE_RETRY_BASE_DELAY_MS: u64 = 500;
 #[derive(Debug)]
 pub(crate) struct QuorumOutcome {
     pub(crate) successes: usize,
+    pub(crate) successful_targets: Vec,
     pub(crate) failures: Vec<(T, E)>,
     pub(crate) reached: bool,
 }
@@ -58,6 +59,7 @@ where
     if required == 0 {
         return QuorumOutcome {
             successes: 0,
+            successful_targets: Vec::new(),
             failures: Vec::new(),
             reached: true,
         };
@@ -74,14 +76,17 @@ where
     }
 
     let mut successes = 0usize;
+    let mut successful_targets = Vec::with_capacity(required);
     let mut failures = Vec::new();
     while let Some((target, result)) = in_flight.next().await {
         match result {
             Ok(_) => {
                 successes += 1;
+                successful_targets.push(target);
                 if successes >= required {
                     return QuorumOutcome {
                         successes,
+                        successful_targets,
                         failures,
                         reached: true,
                     };
@@ -98,6 +103,7 @@ where
 
     QuorumOutcome {
         successes,
+        successful_targets,
         failures,
         reached: false,
     }
@@ -308,6 +314,9 @@ mod tests {
 
         assert!(outcome.reached);
         assert_eq!(outcome.successes, 4);
+        let mut successful_targets = outcome.successful_targets;
+        successful_targets.sort_unstable();
+        assert_eq!(successful_targets, vec![0, 1, 2, 3]);
         assert!(outcome.failures.is_empty());
         assert_eq!(launched.get(), 4);
     }
@@ -328,6 +337,9 @@ mod tests {
 
         assert!(outcome.reached);
         assert_eq!(outcome.successes, 4);
+        let mut successful_targets = outcome.successful_targets;
+        successful_targets.sort_unstable();
+        assert_eq!(successful_targets, vec![2, 3, 4, 5]);
         assert_eq!(outcome.failures.len(), 2);
         assert_eq!(launched.get(), 6);
     }
@@ -343,9 +355,27 @@ mod tests {
 
         assert!(!outcome.reached);
         assert_eq!(outcome.successes, 0);
+        assert!(outcome.successful_targets.is_empty());
         assert_eq!(outcome.failures.len(), 7);
     }
 
+    #[test]
+    fn quorum_reports_partial_success_targets_when_exhausted() {
+        let outcome = futures::executor::block_on(async {
+            quorum_with_fallback(0_u8..7, 4, |target| {
+                futures_util::future::ready(if target < 3 { Ok(()) } else { Err(target) })
+            })
+            .await
+        });
+
+        assert!(!outcome.reached);
+        assert_eq!(outcome.successes, 3);
+        let mut successful_targets = outcome.successful_targets;
+        successful_targets.sort_unstable();
+        assert_eq!(successful_targets, vec![0, 1, 2]);
+        assert_eq!(outcome.failures.len(), 4);
+    }
+
     #[test]
     fn store_byte_bound_and_retry_schedule_match_native_policy() {
         assert_eq!(store_byte_bound(4 * 1024 * 1024), 16);

From 18b08863025d8b8d95e09faaa67b7d43d0247595 Mon Sep 17 00:00:00 2001
From: Mick van Dijke <12992260+mickvandijke@users.noreply.github.com>
Date: Mon, 31 Aug 2026 15:49:46 +0200
Subject: [PATCH 14/31] fix(browser): use wasm-compatible endpoint clock

---
 ant-core/src/client_engine.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ant-core/src/client_engine.rs b/ant-core/src/client_engine.rs
index c17fa47f..d9a5ff96 100644
--- a/ant-core/src/client_engine.rs
+++ b/ant-core/src/client_engine.rs
@@ -8,7 +8,7 @@ use std::future::Future;
 use std::hash::Hash;
 use std::time::Duration;
 #[cfg(any(feature = "browser-wasm", test))]
-use std::time::Instant;
+use web_time::Instant;
 
 #[cfg_attr(
     all(feature = "browser-wasm", not(feature = "native")),

From b3ac2fbd8282b0cee89a9b6ca8815e1e7118f61a Mon Sep 17 00:00:00 2001
From: Mick van Dijke <12992260+mickvandijke@users.noreply.github.com>
Date: Mon, 31 Aug 2026 15:51:14 +0200
Subject: [PATCH 15/31] fix(web): retain wallet key after uploads

---
 web/index.html  | 5 +++--
 web/src/main.js | 1 -
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/web/index.html b/web/index.html
index b849e5d4..00e99b74 100644
--- a/web/index.html
+++ b/web/index.html
@@ -92,8 +92,9 @@ 

Paid public file upload

The wallet key is used only by this page to sign the EVM transactions. It - is cleared from the form immediately and is never sent to a node or stored - in the manifest. Use the disposable funded key printed by ant-devnet. + is never sent to a node or stored in the manifest. This demo keeps it in + the form for repeat uploads, so use the disposable funded key printed by + ant-devnet.