From 022dbf1eb6b1ed6023012a2c26fdefec60a3f7f7 Mon Sep 17 00:00:00 2001 From: Nathan Herald Date: Tue, 25 Aug 2026 09:18:23 +0200 Subject: [PATCH] test: bound the amplification, not the machine `three_node_2000_file_continuous_mutation_stays_bounded` capped reconciles per SECOND, and derived the scan and persist caps from that same per-second figure. Every bound was therefore a function of how fast the machine happened to be, so the test failed on hardware that was merely quick. IT FAILED THAT WAY TWICE AND THE SECOND ONE WAS MINE. The cap was 4/s, derived when a persist wrote 72 MB of pretty-printed JSON. Compact bookkeeping made a pass cheaper, more legitimate passes fit the same fixed burst, and I re-derived the cap as 5/s from macOS runs. Linux CI then produced 5.04/s and turned main red on a run whose behaviour was fine. Raising it again from one platform's numbers would just move the trap. MEASURED ON BOTH, AND THIS IS THE POINT. The rate moves a lot. The ratios barely move. macOS 3.99 to 4.27 reconciles/s 2.50-2.61 scans 1.86-1.95 persists Linux CI 5.04 reconciles/s 2.53 scans 1.81 persists Scans and persists PER RECONCILE are what amplification means, and they do not depend on clock speed. So they are bounded directly, at exactly the strictness the old per-second forms implied: 112/36 scans and 76/36 persists per reconcile. NOTHING IS LOOSENED. The failing CI run sits inside both, at 2.53 and 1.81. The reconcile rate keeps a ceiling, but only as a runaway backstop, and it sits far above anything either platform has produced. A feedback loop is orders of magnitude, not a fast laptop. I PROVED THE NEW BOUNDS CAN FAIL. Tightening the scan allowance makes it fail and name the real figure. AND THE FIRST FAILURE MESSAGE LIED, so the allowance now has ONE source. With a tightened numerator the message still printed the OLD limit, because the assertion used the constant and the message hardcoded the number. That would send the next reader looking in the wrong place. Agent: Silber.fabric --- src/sync/engine.rs | 73 +++++++++++++++++++++++++++++++--------------- 1 file changed, 50 insertions(+), 23 deletions(-) diff --git a/src/sync/engine.rs b/src/sync/engine.rs index 3514c20..caaa96f 100644 --- a/src/sync/engine.rs +++ b/src/sync/engine.rs @@ -5783,37 +5783,64 @@ mod tests { // caps expressed against that same rate, 112/36 and 76/36 of it, which is // the derivation the previous comment described. No new number is invented. let elapsed_millis = elapsed.as_millis().max(1); - // The ceiling was 4 per second, derived when a persist wrote 72 MB of - // pretty-printed JSON. Compact bookkeeping made a pass cheaper, so more - // legitimate passes fit the same fixed burst, and the old ceiling stopped - // bounding amplification and started bounding speed. + + // BOUND THE AMPLIFICATION, NOT THE MACHINE. + // + // This test used to cap reconciles per SECOND, and the scan and persist + // caps were derived from that same per-second figure. That made every + // bound a function of how fast the machine happened to be, so the test + // failed on hardware that was merely quick. + // + // It failed exactly that way twice. The cap was 4/s, derived when a + // persist wrote 72 MB of pretty-printed JSON. Compact bookkeeping made a + // pass cheaper, more legitimate passes fit the same fixed burst, and I + // re-derived the cap as 5/s from macOS runs. Linux CI then produced + // 5.04/s and went red on a run whose behaviour was fine. // - // MEASURED ON ONE MACHINE, FIVE RUNS EACH, THE SAME BURST: - // pretty 30 to 34 reconciles, 2.86 to 3.15 per second - // compact 38 reconciles every run, 3.52 to 3.60 per second + // MEASURED, AND THIS IS THE POINT. Across both platforms the RATE moves + // a lot and the RATIOS barely move: // - // That rise is the intended effect and not amplification. The writers - // still perform a fixed number of revisions, and the run still has to - // converge below. The ceiling is re-derived to keep roughly the headroom - // it used to have above the observed rate, so a genuine feedback loop - // still trips it. It is fractionally more headroom than before, which is - // deliberate: this test has a history of failing on a loaded machine at - // rates that were legitimate. + // macOS 3.99 to 4.27 reconciles/s 2.50-2.61 scans 1.86-1.95 persists + // Linux CI 5.04 reconciles/s 2.53 scans 1.81 persists // - // The scan and persist ceilings stay expressed against this same rate, - // 112/36 and 76/36 of it, so the original relationships are unchanged. - const MAX_RECONCILES_PER_SEC: u128 = 5; + // Scans and persists PER RECONCILE are what "amplification" means, and + // they are independent of clock speed. So they are bounded directly, at + // the same strictness the old per-second forms implied: 112/36 scans and + // 76/36 persists per reconcile. Nothing is loosened. + // + // The reconcile rate keeps a ceiling, but only as a runaway backstop. A + // genuine feedback loop is orders of magnitude, not a fast laptop, so it + // sits far above anything either platform has produced. + const RUNAWAY_RECONCILES_PER_SEC: u128 = 10; + // One source for the allowance so the assertion and the message it + // prints can never disagree. They did: a tightened numerator still + // printed the old limit, which would send the next reader looking in + // the wrong place. + const ALLOWANCE_DEN: u128 = 36; + const SCANS_ALLOWED: u128 = 112; + const PERSISTS_ALLOWED: u128 = 76; + assert!( + (reconciles as u128) * 1_000 <= elapsed_millis * RUNAWAY_RECONCILES_PER_SEC, + "runaway reconcile rate: {reconciles} reconciles in {elapsed:?}, which is \ + far beyond anything a slow or fast machine explains" + ); assert!( - (reconciles as u128) * 1_000 <= elapsed_millis * MAX_RECONCILES_PER_SEC, - "reconcile rate exceeded: {reconciles} reconciles, {scans} scans, and {persists} persists in {elapsed:?}" + reconciles > 0, + "no reconciles happened, so the ratios below would prove nothing" ); assert!( - (scans as u128) * 1_000 * 36 <= elapsed_millis * MAX_RECONCILES_PER_SEC * 112, - "full-folder scan rate exceeded: {scans} scans against {reconciles} reconciles in {elapsed:?}" + (scans as u128) * ALLOWANCE_DEN <= (reconciles as u128) * SCANS_ALLOWED, + "scan amplification: {scans} scans for {reconciles} reconciles is {:.2} \ + per reconcile, over the {:.2} this entry is allowed", + scans as f64 / reconciles as f64, + SCANS_ALLOWED as f64 / ALLOWANCE_DEN as f64 ); assert!( - (persists as u128) * 1_000 * 36 <= elapsed_millis * MAX_RECONCILES_PER_SEC * 76, - "state persist rate exceeded: {persists} persists against {reconciles} reconciles in {elapsed:?}" + (persists as u128) * ALLOWANCE_DEN <= (reconciles as u128) * PERSISTS_ALLOWED, + "persist amplification: {persists} persists for {reconciles} reconciles is \ + {:.2} per reconcile, over the {:.2} this entry is allowed", + persists as f64 / reconciles as f64, + PERSISTS_ALLOWED as f64 / ALLOWANCE_DEN as f64 ); // The bounded watcher work must still converge the latest value.