feat(lws): type index and type search

.github/workflows/lws-conformance.yml

@@ -0,0 +1,110 @@
+# Linked Web Storage conformance for sparq-lws-core in LWS mode (SOLID_SERVER_PROTOCOL=lws).
+#
+# - touchstone: the Touchstone suite (ebremer/touchstone) against a secured sparq: the harness holds
+#   the authorization server key, serves alice's and bob's identity documents and webhook inboxes,
+#   and alice owns the storage. A ratchet: the run fails when any module passes fewer tests than
+#   crates/sparq-lws-core/conformance/lws/floor.json records.
+# - lws-net: the LWS.net harness (langsamu/LWS.net) with the elf-pavlik/lws-test-suite YAML-LD
+#   definition, as lws-contrib/dagger-workspace runs it, against sparq in LWS open mode.
+#
+# Both harnesses are pinned to a commit; bump the pins deliberately and re-baseline floor.json.
+name: lws-conformance
+
+on:
+  pull_request:
+    paths:
+      - "crates/sparq-lws-core/**"
+      - ".github/workflows/lws-conformance.yml"
+  push:
+    branches: [main]
+    paths:
+      - "crates/sparq-lws-core/**"
+      - ".github/workflows/lws-conformance.yml"
+  workflow_dispatch:
+
+concurrency:
+  group: lws-conformance-${{ github.ref }}
+  cancel-in-progress: true
+
+permissions:
+  contents: read
+
+env:
+  TOUCHSTONE_REF: 971dc8def56200c3eb0f08348295ca545cf92a00
+  LWS_NET_REF: eca99a4a5beb949561a9989a183dd3216712bb79
+  LWS_TESTS_REF: 72cfc0a80c718dddbcf43ea6bea98a13a3c14e99
+
+jobs:
+  touchstone:
+    name: lws touchstone
+    runs-on: ubuntu-latest
+    timeout-minutes: 45
+    steps:
+      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
+      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
+        with:
+          repository: ebremer/touchstone
+          ref: ${{ env.TOUCHSTONE_REF }}
+          path: touchstone
+          persist-credentials: false
+      - uses: actions/setup-java@03ad4de0992f5dab5e18fcb136590ce7c4a0ac95 # v5.6.0
+        with:
+          distribution: temurin
+          java-version: "21"
+          cache: maven
+      - uses: Swatinem/rust-cache@e18b497796c12c097a38f9edb9d0641fb99eee32 # v2
+        with:
+          key: lws-conformance
+          save-if: ${{ github.ref == 'refs/heads/main' }}
+      - name: build touchstone
+        working-directory: touchstone
+        run: ./mvnw -q -B -ntp -pl harness-cli -am -Dmaven.test.skip=true package
+      - name: build sparq-lws-core
+        run: cargo build -p sparq-lws-core
+      - name: run touchstone
+        env:
+          TOUCHSTONE: ${{ github.workspace }}/touchstone
+        run: crates/sparq-lws-core/conformance/lws/touchstone.sh all || true
+      - name: check the floor
+        run: python3 crates/sparq-lws-core/conformance/lws/check-floor.py target/lws-touchstone/runs
+      - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+        if: always()
+        with:
+          name: lws-touchstone-report
+          path: |
+            target/lws-touchstone/runs
+            target/lws-touchstone/server.log
+          retention-days: 14
+
+  lws-net:
+    name: lws lws-net
+    runs-on: ubuntu-latest
+    timeout-minutes: 45
+    steps:
+      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
+      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
+        with:
+          repository: langsamu/LWS.net
+          ref: ${{ env.LWS_NET_REF }}
+          path: LWS.net
+          persist-credentials: false
+      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
+        with:
+          repository: elf-pavlik/lws-test-suite
+          ref: ${{ env.LWS_TESTS_REF }}
+          path: lws-test-suite
+          persist-credentials: false
+      - uses: actions/setup-dotnet@a98b56852c35b8e3190ac28c8c2271da59106c68 # v6.0.0
+        with:
+          dotnet-version: "10.0.x"
+      - uses: Swatinem/rust-cache@e18b497796c12c097a38f9edb9d0641fb99eee32 # v2
+        with:
+          key: lws-conformance
+          save-if: ${{ github.ref == 'refs/heads/main' }}
+      - name: build sparq-lws-core
+        run: cargo build -p sparq-lws-core
+      - name: run lws-net
+        env:
+          LWS_NET: ${{ github.workspace }}/LWS.net
+          LWS_TESTS: ${{ github.workspace }}/lws-test-suite/lws10/tests.yaml
+        run: crates/sparq-lws-core/conformance/lws/lws-net.sh

crates/sparq-lws-core/Cargo.toml

@@ -219,6 +219,13 @@ itoa = "1"
 # Declared for native targets below.
 
 [target.'cfg(not(target_arch = "wasm32"))'.dependencies]
+# Linked Web Storage (`src/lws/`): ES256 access tokens, did:key / CID / OpenID Connect subject
+# tokens and RFC 9421 notification signatures. p256 and reqwest are already in the resolved graph
+# (p256 as a dev-dependency, reqwest via solid-oidc-verifier), so these entries add no new crate.
+p256 = { version = "0.13", features = ["ecdsa", "jwk", "std"] }
+reqwest = { version = "0.12", default-features = false, features = ["rustls-tls", "json"] }
+bs58 = "0.5"
+httpdate = "1"
 # Preserve the native dependency graph and feature set exactly. The target split keeps listener,
 # runtime, verifier, filesystem, TLS, PoP, notification, and non-memory backend code out of wasm.
 axum = { version = "0.8", features = ["ws"] }
@@ -265,6 +272,9 @@ http-body-util = "0.1"
 sha2 = "0.10"
 base64 = "0.22"
 url = "2"
+# RFC 3987 IRI validation: every IRI a client sends (filters, grants, inboxes) is checked by
+# one parser. Already in the tree through oxttl.
+oxiri = "0.2"
 aws-lc-rs = "1"
 subtle = "2.6"
 redis = { version = "1", default-features = false, features = ["r2d2"], optional = true }

crates/sparq-lws-core/README.md

@@ -49,8 +49,7 @@ docker run --rm --name sparq-lws-core -p 127.0.0.1:3000:3000 \
 
 **Required production configuration:**
 
-- **Base URL:** `SOLID_SERVER_BASE_URL` = the public `https://` origin (set
-  `SOLID_SERVER_AUDIENCE` only if the token audience differs).
+- **Base URL:** `SOLID_SERVER_BASE_URL` = the public `https://` origin (set `SOLID_SERVER_AUDIENCE` only if the token audience differs).
 - **TLS:** terminate at a trusted proxy (keep the container port private), or mount PEMs
   and set both `SOLID_SERVER_TLS_CERT` + `SOLID_SERVER_TLS_KEY` (setting one fails boot;
   uid 65532 must read them).
@@ -86,6 +85,7 @@ docker run --rm --name sparq-lws-core -p 127.0.0.1:3000:3000 \
 - **Transport hardening** — HTTP/2 rapid-reset and HTTP/1 slowloris guards (explicit
   header-count, aggregate-byte, and slow-header timeout bounds); request timeouts, body
   limits, per-connection max-requests, rate limiting, and overload shedding.
+- **LWS 1.0 mode** — `SOLID_SERVER_PROTOCOL=lws` serves W3C Linked Web Storage (`src/lws/`, skill § *Run the LWS 1.0 protocol*, [`conformance/lws/`](./conformance/lws)).
 - Cargo features:
   - `embedded-sparq` (**default-on**, sq-gg0qq.3) — the first-class in-process
     SPARQ engine backend (in-workspace path deps on `sparq-core`/`sparq-engine`);

crates/sparq-lws-core/conformance/lws/check-floor.py

@@ -0,0 +1,50 @@
+#!/usr/bin/env python3
+"""Fail when the latest Touchstone run passes fewer tests in any module than floor.json records.
+
+    check-floor.py <runs-dir> [floor.json]
+
+Modules are the first path segment of a test id (core, auth, index), except notifications/webhook.
+Raise the floor in the same change that makes more tests pass; never lower it to go green.
+"""
+import glob
+import json
+import os
+import sys
+from collections import Counter
+
+
+def module(test_id: str) -> str:
+    path = test_id.split("#")[0]
+    return "notifications/webhook" if path.startswith("notifications/") else path.split("/")[0]
+
+
+def main() -> int:
+    runs = sys.argv[1]
+    floor_path = sys.argv[2] if len(sys.argv) > 2 else os.path.join(os.path.dirname(__file__), "floor.json")
+    reports = sorted(glob.glob(os.path.join(runs, "*", "report.json")))
+    if not reports:
+        print(f"no Touchstone report under {runs}", file=sys.stderr)
+        return 2
+    report = json.load(open(reports[-1]))
+    passed, total = Counter(), Counter()
+    for t in report["tests"]:
+        m = module(t["id"])
+        total[m] += 1
+        passed[m] += t["outcome"] == "passed"
+    floor = json.load(open(floor_path))["passed"]
+    ok = True
+    for m in sorted(set(total) | set(floor)):
+        want = floor.get(m, 0)
+        line = f"{m:24} {passed[m]:4} passed of {total[m]:4} (floor {want})"
+        if passed[m] < want:
+            ok = False
+            line += "  BELOW FLOOR"
+        print(line)
+    for t in report["tests"]:
+        if t["outcome"] in ("failed", "cantTell"):
+            print(f"  {t['outcome']:9} {t['level']:6} {t['id']}")
+    return 0 if ok else 1
+
+
+if __name__ == "__main__":
+    sys.exit(main())

crates/sparq-lws-core/conformance/lws/floor.json

@@ -0,0 +1,9 @@
+{
+  "_doc": "Touchstone passes per module that CI must keep (check-floor.py). Raise with the change that earns it.",
+  "passed": {
+    "core": 121,
+    "auth": 22,
+    "index": 39,
+    "notifications/webhook": 11
+  }
+}

crates/sparq-lws-core/conformance/lws/lws-net.sh

@@ -0,0 +1,57 @@
+#!/usr/bin/env bash
+# Run the LWS.net conformance harness (https://github.com/langsamu/LWS.net) with the YAML-LD
+# suite definition from elf-pavlik/lws-test-suite (branch yaml), as lws-contrib/dagger-workspace
+# does, against sparq-lws-core in LWS open mode (the harness sends no credentials).
+#
+#   LWS_NET=../LWS.net LWS_TESTS=../lws-test-suite/lws10/tests.yaml \
+#     crates/sparq-lws-core/conformance/lws/lws-net.sh
+#
+# Needs the .NET 10 SDK and Node.js (to convert the YAML-LD definition to N-Triples).
+set -euo pipefail
+here=$(cd "$(dirname "$0")" && pwd)
+repo=$(cd "$here/../../../.." && pwd)
+LWS_NET=${LWS_NET:-$repo/../LWS.net}
+LWS_TESTS=${LWS_TESTS:-$repo/../lws-test-suite/lws10/tests.yaml}
+PORT=${PORT:-3919}
+OUT=${OUT:-$repo/target/lws-net}
+BIN=${BIN:-$repo/target/debug/sparq-lws-core}
+mkdir -p "$OUT"
+[ -x "$BIN" ] || cargo build -q -p sparq-lws-core --manifest-path "$repo/Cargo.toml"
+
+# YAML-LD -> N-Triples, mounted as the harness's embedded suite definition (Resources/new.ttl).
+conv="$OUT/convert"
+mkdir -p "$conv"
+if [ ! -d "$conv/node_modules/jsonld" ]; then
+  (cd "$conv" && npm init -y >/dev/null && npm install --silent jsonld@8 yaml@2 >/dev/null)
+fi
+cat > "$conv/convert.mjs" <<'JS'
+import { readFile, writeFile } from "node:fs/promises";
+import jsonld from "jsonld";
+import { parse } from "yaml";
+const [, , src, dst] = process.argv;
+const doc = parse(await readFile(src, "utf8"));
+await writeFile(dst, await jsonld.toRDF(doc, { format: "application/n-quads" }));
+JS
+(cd "$conv" && node convert.mjs "$LWS_TESTS" "$LWS_NET/Suite/Model/Resources/new.ttl")
+
+SOLID_SERVER_PROTOCOL=lws \
+SOLID_SERVER_LWS_OPEN=1 \
+SOLID_SERVER_BIND=127.0.0.1:$PORT \
+SOLID_SERVER_BASE_URL=http://localhost:$PORT \
+  "$BIN" > "$OUT/server.log" 2>&1 &
+pid=$!
+trap 'kill $pid 2>/dev/null || true' EXIT
+for _ in $(seq 1 100); do
+  curl -s -o /dev/null "http://localhost:$PORT/" && break
+  sleep 0.2
+done
+cd "$LWS_NET"
+status=0
+# LWS.net's global.json selects Microsoft.Testing.Platform, which takes --report-trx rather than
+# VSTest's --logger (it exits 5, invalid arguments, on --logger).
+Suite__BaseUri="http://localhost:$PORT/" dotnet test Suite/Test \
+  --report-trx --report-trx-filename lws-net.trx --results-directory "$OUT" || status=$?
+# One line with the counts, as a CI annotation when running in GitHub Actions.
+counts=$(grep -o '<Counters [^>]*>' "$OUT/lws-net.trx" 2>/dev/null | head -1 || true)
+echo "${GITHUB_ACTIONS:+::notice title=lws-net::}lws-net ${counts:-no results}"
+exit $status

crates/sparq-lws-core/conformance/lws/touchstone.sh

@@ -0,0 +1,91 @@
+#!/usr/bin/env bash
+# Run the Touchstone LWS conformance suite (https://github.com/ebremer/touchstone) against
+# sparq-lws-core in LWS mode, as a secured target: the harness holds the authorization server's
+# signing key (HarnessIssuedTokens), serves alice's and bob's identity documents and webhook
+# inboxes from its fixture host (ReachableFixtures), its SAML identity provider is trusted
+# (SamlTrust, when openssl is available), and alice owns the storage.
+#
+#   TOUCHSTONE=../touchstone crates/sparq-lws-core/conformance/lws/touchstone.sh [module]
+#
+# module: all (default), core, auth, notifications/webhook, index, or one manifest or test.
+# Needs JDK 21 and a built touchstone.jar (cd $TOUCHSTONE && ./mvnw -q -pl harness-cli -am
+# -Dmaven.test.skip=true package). Reports land in $OUT (default target/lws-touchstone).
+set -euo pipefail
+here=$(cd "$(dirname "$0")" && pwd)
+repo=$(cd "$here/../../../.." && pwd)
+TOUCHSTONE=${TOUCHSTONE:-$repo/../touchstone}
+MODULE=${1:-all}
+PORT=${PORT:-3917}
+FIXTURES_PORT=${FIXTURES_PORT:-3918}
+OUT=${OUT:-$repo/target/lws-touchstone}
+BIN=${BIN:-$repo/target/debug/sparq-lws-core}
+mkdir -p "$OUT"
+key="$OUT/as-key.json"
+rm -f "$key"
+[ -x "$BIN" ] || cargo build -q -p sparq-lws-core --manifest-path "$repo/Cargo.toml"
+
+fixtures="http://localhost:$FIXTURES_PORT/"
+
+# The harness SAML identity provider's key (SamlTrust): an RSA key pair whose private JWK the
+# harness signs assertions with, and whose public key the authorization server trusts for the
+# entity ${fixtures}idp. Needs openssl; without it the SAML tests are inapplicable.
+saml_idps="$OUT/saml-idps.json"
+saml_jwk=""
+capabilities="Authentication, HarnessIssuedTokens, ReachableFixtures"
+rm -f "$saml_idps"
+if command -v openssl > /dev/null && command -v python3 > /dev/null; then
+  openssl genrsa -out "$OUT/saml-idp.pem" 2048 2> /dev/null
+  openssl rsa -in "$OUT/saml-idp.pem" -pubout -out "$OUT/saml-idp.pub.pem" 2> /dev/null
+  saml_jwk=$(openssl rsa -in "$OUT/saml-idp.pem" -text -noout 2> /dev/null | python3 -c '
+import base64, json, re, sys
+text = sys.stdin.read()
+def field(name):
+    m = re.search(r"^" + name + r":\s*((?:\s+[0-9a-f:]+\n?)+)", text, re.M)
+    return int(m.group(1).replace(":", "").replace("\n", "").replace(" ", ""), 16)
+def b64(n):
+    return base64.urlsafe_b64encode(n.to_bytes((n.bit_length() + 7) // 8, "big")).rstrip(b"=").decode()
+e = int(re.search(r"publicExponent: (\d+)", text).group(1))
+names = {"n": "modulus", "d": "privateExponent", "p": "prime1", "q": "prime2", "dp": "exponent1", "dq": "exponent2", "qi": "coefficient"}
+jwk = {"kty": "RSA", "kid": "saml-idp", "e": b64(e)}
+jwk.update({k: b64(field(v)) for k, v in names.items()})
+print(json.dumps(jwk))
+')
+  python3 -c 'import json, sys; print(json.dumps({sys.argv[1]: open(sys.argv[2]).read()}))' \
+    "${fixtures}idp" "$OUT/saml-idp.pub.pem" > "$saml_idps"
+  capabilities="$capabilities, SamlTrust"
+fi
+SOLID_SERVER_PROTOCOL=lws \
+SOLID_SERVER_BIND=127.0.0.1:$PORT \
+SOLID_SERVER_BASE_URL=http://localhost:$PORT \
+SOLID_SERVER_LWS_OWNER="${fixtures}agents/alice" \
+SOLID_SERVER_LWS_AS_KEY_FILE="$key" \
+SOLID_SERVER_LWS_PAGE_SIZE=${PAGE_SIZE:-4} \
+SOLID_SERVER_LWS_ALLOW_INSECURE_FETCH=1 \
+SOLID_SERVER_LWS_SAML_IDPS_FILE="$([ -s "$saml_idps" ] && echo "$saml_idps")" \
+  "$BIN" > "$OUT/server.log" 2>&1 &
+pid=$!
+trap 'kill $pid 2>/dev/null || true' EXIT
+for _ in $(seq 1 100); do
+  [ -s "$key" ] && curl -s -o /dev/null "http://localhost:$PORT/" && break
+  sleep 0.2
+done
+signing_key=$(cat "$key")
+cat > "$OUT/targets.yaml" <<YAML
+targets:
+  sparq:
+    baseUrl: http://localhost:$PORT/
+    adapter: env
+    capabilities: [$capabilities]
+    properties:
+      saml.idpKey: '$saml_jwk'
+      as.signingKey: '$signing_key'
+      fixtures.baseUrl: '$fixtures'
+      webid.alice: '${fixtures}agents/alice'
+      webid.bob: '${fixtures}agents/bob'
+YAML
+cd "$TOUCHSTONE"
+set +e
+java -jar harness-cli/target/touchstone.jar run --target sparq --targets "$OUT/targets.yaml" \
+  --module "$MODULE" --report-dir "$OUT/runs" 2>&1 | grep -v '^Picked up JAVA_TOOL_OPTIONS'
+status=${PIPESTATUS[0]}
+exit "$status"