From dcb858ef3c36dbe8a73d513a43be9dc090563ae2 Mon Sep 17 00:00:00 2001 From: Nathan Herald Date: Sat, 5 Sep 2026 05:54:03 +0200 Subject: [PATCH] Publish the Release A transition archive --- .github/workflows/release.yml | 10 +++++----- docs/fabric-sync-process-plan.md | 30 +++++++++++++++++++++++++++++- now.md | 30 +++++++++++++++++++++++++++--- src/update.rs | 18 ++++++++++-------- 4 files changed, 71 insertions(+), 17 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 0941159..afd457e 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -48,13 +48,13 @@ jobs: set -euo pipefail mkdir -p dist/package cp "target/${{ matrix.target }}/release/fabric" dist/package/fabric - cp "target/${{ matrix.target }}/release/fabric-sync" dist/package/fabric-sync - chmod 755 dist/package/fabric dist/package/fabric-sync + chmod 755 dist/package/fabric archive="dist/fabric-${{ matrix.target }}.tar.gz" - tar -czf "$archive" -C dist/package fabric fabric-sync + # Release A is the reader-only transition. Step 7 restores the pair. + tar -czf "$archive" -C dist/package fabric members="$(tar -tzf "$archive")" - if [[ "$members" != $'fabric\nfabric-sync' ]]; then - echo "release archive must contain exactly fabric and fabric-sync" >&2 + if [[ "$members" != "fabric" ]]; then + echo "transition archive must contain exactly fabric" >&2 printf 'archive members:\n%s\n' "$members" >&2 exit 1 fi diff --git a/docs/fabric-sync-process-plan.md b/docs/fabric-sync-process-plan.md index c11eb81..f69e8cf 100644 --- a/docs/fabric-sync-process-plan.md +++ b/docs/fabric-sync-process-plan.md @@ -418,6 +418,31 @@ Before the transition release, run one rollback through the actual production service on Linux and macOS. Use hetz first and Silber second. Record every observed state and any manual cleanup. Do not use Bluey for this exercise. +Test the recovery route from the actor who will use it before each deliberate +outage. A reachable host and an open SSH port do not prove that the actor can +log in. Fabric is the only working route to hetz for every agent and for +Silber.cos. If Fabric fails there and cannot recover itself, Nathan must recover +hetz personally. Silber.cos has local shell access on Silber and can repair that +machine by hand. + +Both production rollback exercises passed on 2026-09-05 at exact main +`ebfd0ec`. On hetz, the broken pair installed at 03:33:02Z. The systemd reader +restored exact main at 03:34:00Z, and Fabric answered externally at 03:34:14Z. +The observer depended on Fabric because no agent or Silber.cos could complete +an SSH login. The exercise then restored fleet build `0.2.1+48208e4`. + +On Silber, the broken pair installed at 03:46:14Z. The launchd reader restored +exact main on disk at 03:47:18Z. The control socket answered at 03:47:27Z. The +transient job and plist disappeared, and the reader removed the new companion. +Local shell observation stayed available. The exercise then restored fleet +build `0.2.1+48208e4`. Doctor passed on both hosts after the exercises. + +The deployed `48208e4` reader also received the exact paired archive in an +isolated home. It accepted the checksum, then refused the archive because it +expected exactly one `fabric` member. It changed no executable or staging file. +This clean refusal enforces the Release A order for that deployed reader. It is +not a property of the Release A pair-aware reader. + Add each future real-machine test to this named list when the test is added. The measurement-only ignored tests are not release gates unless this list names them. @@ -432,7 +457,10 @@ The process-boundary release needs all of these results: - The bridge keeps at least 90 percent of embedded content throughput during the fixed 10-minute comparison. A larger loss stops activation and reopens the transport decision with Nathan. -- The release archive and updater verify a matched binary pair. +- The Release A archive contains exactly `fabric`. The deployed `48208e4` + reader refuses a paired archive without changing the machine. +- Step 7 restores an archive with exactly `fabric` and `fabric-sync`. The + updater verifies that matched pair. This plan changes where sync executes. It does not claim that the connection cache caused Nathan's earlier pauses, and it does not optimize sync memory. diff --git a/now.md b/now.md index a78f784..baf4bbf 100644 --- a/now.md +++ b/now.md @@ -42,9 +42,9 @@ status, `fabric status`, `fabric sync ls`, and doctor show that runtime. A configured entry remains visible when the daemon is down. The embedded engine remains the production owner. -The current change adds a detached macOS update supervisor. Launchd owns it -before the updater renames the first binary. It survives terminal loss and -sleep, then removes its job after success or rollback. +PR #164 added a detached macOS update supervisor. PR #165 added pair-aware, +fail-closed rollback. PR #166 made real launchd and systemd tests exercise that +reader. All three changes are merged. The fleet build `0.2.1+48208e4` has only single-binary supervisor logic. The rollback runs this old binary on purpose because it is the machine's known-good @@ -61,6 +61,22 @@ The Release A reader must handle OS service state as well as binary paths. It must restart both old services when a companion rollback exists. It must remove the companion service when the old release had no companion. +Both production rollback exercises passed on 2026-09-05 at exact main +`ebfd0ec`. Hetz restored exact main 58 seconds after the broken pair installed, +and Fabric answered externally after 72 seconds. Silber restored exact main on +disk after 64 seconds, and its control socket answered after 73 seconds. Both +hosts returned to fleet build `0.2.1+48208e4`, and both doctors passed. + +Fabric is the only working route to hetz for every agent and for Silber.cos. +An open SSH port did not provide a usable login. Nathan must personally recover +hetz if Fabric cannot recover itself. Test the recovery actor's route before +every deliberate outage. Silber.cos has local shell access on Silber. + +The release workflow must publish a fabric-only Release A archive. The deployed +`48208e4` reader cleanly refused a paired archive in isolation and changed no +file. Step 7 restores paired archives after Release A reaches every machine. +Do not tag current main until the workflow has this transition shape. + The local library suite passed 499 tests with five measurements ignored. The binary suite passed 19 tests, and the update contract passed eight tests. The five-second latency property measured 502 records, seven scans, a 10.001166 ms @@ -124,6 +140,14 @@ The Silber-to-hetz direct path has p50 50 ms, p90 100 ms, and p99 500 ms. It has 73 of 127,753 samples above one second, or 0.057 percent. The relay path has no sample above one second. These are not Bluey-to-hetz path measurements. +A three-sample direct window after both rollback exercises separated connection +setup from the settled path. Silber-to-hetz measured 1,772.560, 36.751, and +37.637 milliseconds. Hetz-to-Silber measured 116.158, 37.388, and 35.640 +milliseconds. Current path telemetry records a probe that opens a connection in +the same latency series as a probe on an established path. Split those cases so +the network tail and reconnect cost have separate meanings. Release A does not +wait for this low-priority telemetry change. + The `droppy` status row is retained telemetry for a removed peer. Its 10,885 reconnect attempts did not change during a 30-second window. The latest droppy validation event is from 2026-09-02T09:37:30Z. No current churn was measured. diff --git a/src/update.rs b/src/update.rs index 6f743da..ca7fabb 100644 --- a/src/update.rs +++ b/src/update.rs @@ -638,10 +638,11 @@ fn generation_is_current(home: &crate::config::FabricHome, generation: &str) -> /// Check that the daemon came back, and put the old binary back if it did not. /// -/// THIS EXISTS BECAUSE NOTHING OUTSIDE THE MACHINE CAN FIX IT. If a bad binary -/// takes the daemon down, `fabric exec` stops working, and the tool that would -/// repair the machine is the tool that just broke it. On hetz and droppy there -/// is still ssh. On a travelling laptop there may be nothing for days. +/// THIS EXISTS BECAUSE THE RECOVERY ACTOR MAY HAVE NO WORKING ROUTE. If a bad +/// binary takes the daemon down, `fabric exec` stops working, and the tool that +/// would repair the machine is the tool that just broke it. The route must be +/// tested before a deliberate outage. Do not infer that an open SSH port means +/// the recovery actor can log in. /// /// It cannot run in the updating process either: on Linux that process lives /// inside the service's own cgroup and dies with the restart. So it is scheduled @@ -1844,11 +1845,12 @@ mod tests { } #[test] - fn the_release_workflow_packages_exactly_the_binary_pair() { + fn the_release_a_workflow_packages_only_the_reader() { let workflow = include_str!("../.github/workflows/release.yml"); - assert!(workflow.contains("release/fabric-sync\" dist/package/fabric-sync")); - assert!(workflow.contains("-C dist/package fabric fabric-sync")); - assert!(workflow.contains("$'fabric\\nfabric-sync'")); + assert!(workflow.contains("tar -czf \"$archive\" -C dist/package fabric\n")); + assert!(workflow.contains("[[ \"$members\" != \"fabric\" ]]")); + assert!(!workflow.contains("release/fabric-sync\" dist/package/fabric-sync")); + assert!(!workflow.contains("-C dist/package fabric fabric-sync")); } }