From 89233ba730be4f22d46ccf34f4bcfdd2c5147a18 Mon Sep 17 00:00:00 2001 From: Elle Mouton Date: Thu, 16 Jul 2026 10:48:27 -0700 Subject: [PATCH 1/4] unroll: Make unilateral exits reorg recoverable Unilateral exits previously treated confirmation and spend observations as one-way facts. A chain replacement could therefore leave an actor driving side effects from stale proof, checkpoint, or sweep state, especially after a restart replayed that state without first checking the canonical chain. Persist the chain anchors behind each transition and keep confirmation, sweep, and external-spend evidence reversible until policy finality. Reconcile every persisted positive observation before restart side effects, while treating a missing observation or lookup timeout as unknown rather than proof of absence. Keep the source VTXO under a durable unsafe hold whenever an admitted exit has an ambiguous footprint. A failed exit may restore the VTXO only after both the local footprint is empty and the chain boundary supplies objective canonical-absence evidence. Persist terminal exit state before forwarding the corresponding VTXO lifecycle outcome. Use versioned TLV messages and snapshots for the new durable state so rolling upgrades can replay it safely. Extend the actor, registry, reconciliation, restart, and reorg tests to cover reordered and duplicated observations. --- unroll/AGENTS.md | 22 +- unroll/CLAUDE.md | 22 +- unroll/actor.go | 352 +++++++++--- unroll/actor_test.go | 230 ++++++-- unroll/db_store.go | 23 +- unroll/fsm_logic.go | 217 +++++++- unroll/fsm_types.go | 179 +++++- unroll/messages.go | 270 ++++++++- unroll/messages_test.go | 50 ++ unroll/notification_map_test.go | 36 +- unroll/reconcile.go | 320 +++++++++++ unroll/reconcile_chainsource.go | 319 +++++++++++ unroll/reconcile_chainsource_test.go | 317 +++++++++++ unroll/reconcile_test.go | 614 +++++++++++++++++++++ unroll/registry.go | 282 +++++++--- unroll/registry_exit_test.go | 278 +++++----- unroll/registry_messages.go | 15 +- unroll/registry_test.go | 152 +++++- unroll/reorg_safety_test.go | 781 +++++++++++++++++++++++++++ unroll/snapshot.go | 193 ++++++- unroll/snapshot_test.go | 81 +++ unroll/state_snapshot.go | 68 ++- unroll/state_snapshot_test.go | 19 + vhtlcrecovery/coordinator/service.go | 8 +- waved/rpc_server.go | 13 + 25 files changed, 4410 insertions(+), 451 deletions(-) create mode 100644 unroll/reconcile.go create mode 100644 unroll/reconcile_chainsource.go create mode 100644 unroll/reconcile_chainsource_test.go create mode 100644 unroll/reconcile_test.go create mode 100644 unroll/reorg_safety_test.go diff --git a/unroll/AGENTS.md b/unroll/AGENTS.md index db008b220..6420ac977 100644 --- a/unroll/AGENTS.md +++ b/unroll/AGENTS.md @@ -66,11 +66,16 @@ For field-level detail, use `go doc github.com/lightninglabs/wavelength/unroll.< terminal outcome is forwarded to the VTXO manager as a `vtxo.ExitOutcomeNotification` so VTXO lifecycle tracks the unroll's terminal on-chain result rather than the user's intent to exit - (wavelength#602): a clean failure (`!HadOnChainFootprint`) → + (wavelength#602): a failure with neither a locally recorded footprint nor + chain-boundary uncertainty (`!HadOnChainFootprint && !ReliveUnsafe`) → `ExitOutcomeRecoverable` (roll back to live), a completed exit → - `ExitOutcomeConfirmed` (retire to spent). `UnrollTerminatedMsg` carries - `HadOnChainFootprint`, computed by `jobHadOnChainFootprint` (any - confirmed/in-flight proof node or a non-pending sweep). It also carries + `ExitOutcomeConfirmed` (retire to spent). The live actor sets + `ReliveUnsafe` before every chain-boundary attempt; only objective + canonical-absence evidence may clear it. A mere rejection, absence, or + timeout therefore leaves the VTXO held in unilateral exit. + `UnrollTerminatedMsg` carries `HadOnChainFootprint`, computed by + `jobHadOnChainFootprint` (any confirmed/in-flight proof node or a + non-pending sweep), and the durable `ReliveUnsafe` guard. It also carries the child's `ExitPolicyKind`, which the child stamps from its own durable exit policy (`exitPolicyKind`), so the terminal message is self-contained: it stays authoritative after the registry has evicted its in-memory @@ -99,8 +104,8 @@ For field-level detail, use `go doc github.com/lightninglabs/wavelength/unroll.< repeat after termination returns `Created=false` with the historical `ActorID`, never clobbering the sweep txid / failure reason. The one exception is a **recoverable** terminal failure - (`RecoverableFailure`): the prior exit failed cleanly with no on-chain - footprint and the VTXO was rolled back to live (wavelength#602), so + (`RecoverableFailure`): objective canonical-absence evidence allowed the + VTXO to roll back to live (wavelength#602), so a fresh `EnsureUnrollRequest` re-admits (spawns a new child, overwriting the stale record) instead of deduping — otherwise a recovered VTXO could never be unrolled again. Any existing unroll job @@ -264,8 +269,9 @@ For field-level detail, use `go doc github.com/lightninglabs/wavelength/unroll.< `r.active`, `r.pending`, AND `Store.GetRecord` before spawning so a repeat for an already-terminal outpoint returns the historical `ActorID` and never overwrites stored sweep txid / failure reason. - A **recoverable** terminal failure is the deliberate exception: the - VTXO was rolled back to live (wavelength#602), so `handleEnsure` + A **recoverable** terminal failure is the deliberate exception: objective + canonical-absence evidence allowed the VTXO to roll back to live + (wavelength#602), so `handleEnsure` falls through both the `r.pending` and `Store.GetRecord` arms to re-admit a fresh exit rather than strand the recovered coin. - **Fail-closed on restore gaps.** `handleEnsure` validates restorable diff --git a/unroll/CLAUDE.md b/unroll/CLAUDE.md index db008b220..6420ac977 100644 --- a/unroll/CLAUDE.md +++ b/unroll/CLAUDE.md @@ -66,11 +66,16 @@ For field-level detail, use `go doc github.com/lightninglabs/wavelength/unroll.< terminal outcome is forwarded to the VTXO manager as a `vtxo.ExitOutcomeNotification` so VTXO lifecycle tracks the unroll's terminal on-chain result rather than the user's intent to exit - (wavelength#602): a clean failure (`!HadOnChainFootprint`) → + (wavelength#602): a failure with neither a locally recorded footprint nor + chain-boundary uncertainty (`!HadOnChainFootprint && !ReliveUnsafe`) → `ExitOutcomeRecoverable` (roll back to live), a completed exit → - `ExitOutcomeConfirmed` (retire to spent). `UnrollTerminatedMsg` carries - `HadOnChainFootprint`, computed by `jobHadOnChainFootprint` (any - confirmed/in-flight proof node or a non-pending sweep). It also carries + `ExitOutcomeConfirmed` (retire to spent). The live actor sets + `ReliveUnsafe` before every chain-boundary attempt; only objective + canonical-absence evidence may clear it. A mere rejection, absence, or + timeout therefore leaves the VTXO held in unilateral exit. + `UnrollTerminatedMsg` carries `HadOnChainFootprint`, computed by + `jobHadOnChainFootprint` (any confirmed/in-flight proof node or a + non-pending sweep), and the durable `ReliveUnsafe` guard. It also carries the child's `ExitPolicyKind`, which the child stamps from its own durable exit policy (`exitPolicyKind`), so the terminal message is self-contained: it stays authoritative after the registry has evicted its in-memory @@ -99,8 +104,8 @@ For field-level detail, use `go doc github.com/lightninglabs/wavelength/unroll.< repeat after termination returns `Created=false` with the historical `ActorID`, never clobbering the sweep txid / failure reason. The one exception is a **recoverable** terminal failure - (`RecoverableFailure`): the prior exit failed cleanly with no on-chain - footprint and the VTXO was rolled back to live (wavelength#602), so + (`RecoverableFailure`): objective canonical-absence evidence allowed the + VTXO to roll back to live (wavelength#602), so a fresh `EnsureUnrollRequest` re-admits (spawns a new child, overwriting the stale record) instead of deduping — otherwise a recovered VTXO could never be unrolled again. Any existing unroll job @@ -264,8 +269,9 @@ For field-level detail, use `go doc github.com/lightninglabs/wavelength/unroll.< `r.active`, `r.pending`, AND `Store.GetRecord` before spawning so a repeat for an already-terminal outpoint returns the historical `ActorID` and never overwrites stored sweep txid / failure reason. - A **recoverable** terminal failure is the deliberate exception: the - VTXO was rolled back to live (wavelength#602), so `handleEnsure` + A **recoverable** terminal failure is the deliberate exception: objective + canonical-absence evidence allowed the VTXO to roll back to live + (wavelength#602), so `handleEnsure` falls through both the `r.pending` and `Store.GetRecord` arms to re-admit a fresh exit rather than strand the recovered coin. - **Fail-closed on restore gaps.** `handleEnsure` validates restorable diff --git a/unroll/actor.go b/unroll/actor.go index 13f6c30d4..db9c8663c 100644 --- a/unroll/actor.go +++ b/unroll/actor.go @@ -75,6 +75,21 @@ type Config struct { // LedgerSink receives the confirmed on-chain exit fee once the // final sweep has confirmed. LedgerSink fn.Option[ledger.Sink] + + // ChainReconcilerFactory, when set, is invoked after the actor + // loads its proof to construct a ChainReconciler that reconciles + // the persisted checkpoint against the canonical chain at + // startup. Anchors that are no longer live on chain (typically + // reorged out while the daemon was offline) are pruned before + // the FSM session is built so the actor does not broadcast a + // sweep on top of stale planner state. + // + // The factory shape lets the production implementation + // (NewChainSourceReconciler) bind to the actor's proof for + // pkScript lookups while still letting unit tests inject a stub + // that ignores the proof. When None the actor skips + // reconciliation entirely. + ChainReconcilerFactory fn.Option[ChainReconcilerFactory] } // VTXOUnrollActor wraps one durable per-target unroll actor. @@ -178,6 +193,22 @@ type behavior struct { // so a policy-gated sweep is not reported as imminent. None until the // sweep phase resolves the policy. requiredLockTime fn.Option[uint32] + + // reconciled records that the restored checkpoint has been + // reconciled against the canonical chain via cfg.ChainReconciler. + // Reconciliation runs exactly once per actor lifetime, on the + // first ensureLoaded call, before the FSM session is bound. + reconciled bool + + // sweepFinalized latches true after a TxFinalizedMsg arrives for + // the txid currently recorded as the sweep in PlannerState. While + // false, PhaseCompleted is treated as PROVISIONAL: the registry is + // NOT told the actor has terminated, so a reorg of the sweep + // confirmation still has a live actor to deliver the rollback to. + // Once sweepFinalized is true the next notifyRegistryOfTerminal + // call fires UnrollTerminatedMsg and the registry evicts the + // child. + sweepFinalized bool } // unrollTx is the transaction-scoped store handed to the unroll behavior inside @@ -310,9 +341,40 @@ func (b *behavior) dispatch(ctx context.Context, ax actor.Exec[unrollTx], Reason: b.failureReasonForTx(m.Txid, m.Reason), }) + case *TxReorgedMsg: + return b.handleEvent(ctx, ax, &TxReorgedEvent{ + Txid: m.Txid, + }) + + case *TxFinalizedMsg: + // Latch sweepFinalized BEFORE handleEvent so the + // notifyRegistry call inside driveEvent sees the + // post-finalization view and fires UnrollTerminatedMsg + // instead of holding the actor in provisional Completed. + if err := b.ensureLoaded(ctx); err != nil { + return fn.Err[Resp](err) + } + b.maybeLatchSweepFinalized(m.Txid) + + return b.handleEvent(ctx, ax, &TxFinalizedEvent{ + Txid: m.Txid, + }) + case *SpendObservedMsg: return b.handleSpendObserved(ctx, ax, m) + case *SpendReorgedMsg: + return b.handleEvent(ctx, ax, &SpendReorgedEvent{}) + + case *SpendFinalizedMsg: + return b.handleEvent(ctx, ax, &SpendFinalizedEvent{}) + + // GetStateRequest is intentionally NOT handled here: Receive + // short-circuits it before it ever reaches dispatch (it is a read-only + // probe that drives no FSM transition and writes nothing). A + // GetStateRequest can only arrive at this switch via a future refactor + // that breaks that invariant, in which case the default arm's loud + // error is the signal we want -- not a silently duplicated read path. default: return fn.Err[Resp]( fmt.Errorf("unknown unroll message: %T", msg), @@ -974,14 +1036,35 @@ func (b *behavior) watchDeferredCheckpoint(ctx context.Context, }, ) + // Deferred-checkpoint watches register directly against + // chainsource rather than txconfirm, but the rollback contract + // is identical: an operator-confirmed checkpoint that reorgs out + // while the actor is live must drop from ConfirmedTxids so the + // planner does not advance off a stale anchor, and a finality + // signal lets the chainsource sub-actor release its registration + // at the reorg-safety horizon. Wire both lifecycle refs through + // the same selfRef the positive event uses. + reorgRef := chainsource.MapConfReorgedEvent( + b.selfRef, func(event chainsource.ConfReorgedEvent) Msg { + return &TxReorgedMsg{Txid: event.Txid} + }, + ) + doneRef := chainsource.MapConfDoneEvent( + b.selfRef, func(event chainsource.ConfDoneEvent) Msg { + return &TxFinalizedMsg{Txid: event.Txid} + }, + ) + txidCopy := txid _, err = b.cfg.ChainSource.Ask(ctx, &chainsource.RegisterConfRequest{ - CallerID: b.deferredCheckpointCallerID(), - Txid: &txidCopy, - PkScript: append([]byte(nil), pkScript...), - TargetConfs: 1, - HeightHint: b.proofNodeConfHeightHint(ctx, txid), - NotifyActor: fn.Some(notifyRef), + CallerID: b.deferredCheckpointCallerID(), + Txid: &txidCopy, + PkScript: append([]byte(nil), pkScript...), + TargetConfs: 1, + HeightHint: b.proofNodeConfHeightHint(ctx, txid), + NotifyActor: fn.Some(notifyRef), + NotifyReorged: fn.Some(reorgRef), + NotifyDone: fn.Some(doneRef), }).Await(ctx).Unpack() if err != nil { return err @@ -1271,6 +1354,23 @@ func (b *behavior) ensureLoaded(ctx context.Context) error { b.planner = planner } + // Reconcile the restored checkpoint against the canonical chain + // before binding the FSM session. Any confirmed-tx anchor the + // reconciler reports as absent gets dropped (with its + // descendants), and a target / sweep confirmation that vanished + // while the daemon was offline downgrades the sweep state. This + // must run before stateFromCheckpoint so the FSM is built from + // the post-reconciliation snapshot, and before any side effects + // (block subscription, spend watch, ResumeEvent reissue) so a + // stale sweep is never re-broadcast on a now-unconfirmed target. + if !b.reconciled { + if err := b.reconcileOnRestart(ctx); err != nil { + return err + } + + b.reconciled = true + } + if b.session == nil { initialState := State(&Idle{}) if b.pending != nil && b.pending.Started { @@ -1308,11 +1408,9 @@ func (b *behavior) ensureLoaded(ctx context.Context) error { // // txconfirm delivers notifications in its own type space // ([txconfirm.Notification]) but our durable mailbox only accepts [Msg] -// variants so the delivery store can codec them. This helper threads a -// filtering adapter: every txconfirm notification is synchronously -// re-wrapped into the matching mailbox message and forwarded to our -// self-ref for durable enqueue, or dropped when the lifecycle event has -// no unroll-side meaning (see mapTxconfirmNotification). +// variants so the delivery store can codec them. This helper maps every +// reorg-aware lifecycle notification into the corresponding durable unroll +// message. Unknown variants still fail loudly. func (b *behavior) notificationRef() actor.TellOnlyRef[txconfirm.Notification] { return txconfirm.FilterMapNotification( b.selfRef, mapTxconfirmNotification, @@ -1321,27 +1419,10 @@ func (b *behavior) notificationRef() actor.TellOnlyRef[txconfirm.Notification] { // mapTxconfirmNotification re-wraps one txconfirm lifecycle event into // the unroll actor's durable mailbox message space. Subscribers receive -// the FULL reorg-aware lifecycle, so every variant needs an explicit -// disposition: -// -// - TxConfirmed marks the proof node (or sweep) confirmed. -// -// - TxFinalized re-plays the authoritative confirmation numbers once -// the tx is past the backend's reorg-safety depth. The FSM's -// confirmation handling is idempotent (appendUniqueSorted et al), -// so mapping it to a second TxConfirmedMsg is a safe no-op when the -// original TxConfirmed already landed and a recovery when it was -// dropped. Treating it as anything else is a live bug: it used to -// fall into the unknown-notification arm below and terminally -// failed the whole unroll job the moment a backend synthesized -// finality. -// -// - TxReorged is best-effort and superseded by the next reliable -// event (re-TxConfirmed / TxFinalized / TxFailed), and the planner -// has no un-confirm transition; drop it and let the follow-up event -// re-establish state. -// -// - TxFailed, and any unknown future variant, terminate loudly. +// the full reorg-aware lifecycle. Unlike the foundation version of the +// unroll planner, this FSM has explicit reversible reorg and finality +// transitions, so neither notification may be dropped or collapsed into a +// repeated confirmation. Unknown future variants still terminate loudly. func mapTxconfirmNotification(msg txconfirm.Notification) (Msg, bool) { switch m := msg.(type) { case *txconfirm.TxConfirmed: @@ -1352,14 +1433,14 @@ func mapTxconfirmNotification(msg txconfirm.Notification) (Msg, bool) { }, true case *txconfirm.TxFinalized: - return &TxConfirmedMsg{ - Txid: m.Txid, - Height: m.BlockHeight, - NumConfs: m.NumConfs, + return &TxFinalizedMsg{ + Txid: m.Txid, }, true case *txconfirm.TxReorged: - return nil, false + return &TxReorgedMsg{ + Txid: m.Txid, + }, true case *txconfirm.TxFailed: return &TxFailedMsg{ @@ -1405,10 +1486,40 @@ func (b *behavior) restoreCheckpoint(ctx context.Context) error { b.pending = decoded b.sweepTx = copyTx(decoded.SweepTx) + b.sweepFinalized = decoded.SweepFinalized return nil } +// reconcileOnRestart asks the configured ChainReconciler whether each +// anchor recorded in the restored checkpoint is still live on the +// canonical chain, and prunes the in-memory checkpoint accordingly. +// Called from ensureLoaded exactly once per actor lifetime, BEFORE the +// FSM session is bound. +// +// A missing reconciler is a no-op: backends that cannot answer +// historical confirmation queries fall back to the conservative +// behavior of treating the restored checkpoint as authoritative. +// Transport errors from the reconciler are surfaced upward; the actor +// must not proceed to broadcast off a checkpoint we could not verify. +func (b *behavior) reconcileOnRestart(ctx context.Context) error { + if b.cfg.ChainReconcilerFactory.IsNone() { + return nil + } + + if b.pending == nil || !b.pending.Started { + return nil + } + + factory := b.cfg.ChainReconcilerFactory.UnsafeFromSome() + reconciler := factory(b.cfg.TargetOutpoint, b.proof) + if reconciler == nil { + return nil + } + + return reconcileCheckpoint(ctx, reconciler, b.proof, b.pending) +} + // ensureBlockSubscription starts the actor's shared block epoch subscription on // first use so CSV waits advance in the live daemon. func (b *behavior) ensureBlockSubscription(ctx context.Context) error { @@ -1494,14 +1605,28 @@ func (b *behavior) ensureSpendWatch(ctx context.Context) error { } }, ) + reorgRef := chainsource.MapSpendReorgedEvent( + b.selfRef, + func(_ chainsource.SpendReorgedEvent) Msg { + return &SpendReorgedMsg{} + }, + ) + doneRef := chainsource.MapSpendDoneEvent( + b.selfRef, + func(_ chainsource.SpendDoneEvent) Msg { + return &SpendFinalizedMsg{} + }, + ) _, err = b.cfg.ChainSource.Ask( ctx, &chainsource.RegisterSpendRequest{ - CallerID: b.spendCallerID(), - Outpoint: &targetOutpoint, - PkScript: pkScript, - HeightHint: uint32(b.desc.CreatedHeight), - NotifyActor: fn.Some(notifyRef), + CallerID: b.spendCallerID(), + Outpoint: &targetOutpoint, + PkScript: pkScript, + HeightHint: uint32(b.desc.CreatedHeight), + NotifyActor: fn.Some(notifyRef), + NotifyReorged: fn.Some(reorgRef), + NotifyDone: fn.Some(doneRef), }, ).Await(ctx).Unpack() if err != nil { @@ -1757,18 +1882,36 @@ func (b *behavior) handleSpendObserved(ctx context.Context, } } - // Case 4: neither of the above. Someone else spent the watched output. - // This happens if the operator cooperatively claimed the VTXO, - // if a reorg replaced history, or in fraud scenarios. There is - // no way for this unroll to proceed, so terminate with a - // reason string that identifies the spender for operator - // triage. - spentOutpoint := b.cfg.TargetOutpoint - if msg.Outpoint != (wire.OutPoint{}) { - spentOutpoint = msg.Outpoint + // Case 4: neither of the above. Someone else spent the watched + // output. This can happen for two structurally different + // outputs, with different reversibility properties: + // + // a. The target outpoint was spent by an unknown party. This + // is the cooperative-claim / fraud scenario the external- + // spend reorg-safety work is built for: a reorg of the + // spending block can resurrect the recovery job, so the + // actor parks in AwaitingExternalSpendFinality. A + // subsequent SpendFinalizedMsg promotes the spend to a + // permanent FailReason; a SpendReorgedMsg clears the + // observation and resumes planning. + // + // b. A proof-node output was spent by a transaction that is + // not itself part of the proof graph (ackProofOutputSpend + // acked the parent confirmation already, but returned + // false for the spender lookup). The proof graph cannot + // complete through this fork, so the unroll job is + // terminally dead. Fail with a reason that identifies the + // spender for operator triage. + if msg.Outpoint == (wire.OutPoint{}) || + msg.Outpoint == b.cfg.TargetOutpoint { + return b.handleEvent(ctx, ax, &ExternalSpendObservedEvent{ + SpendingTxid: msg.SpendingTxid, + SpendingHeight: msg.SpendingHeight, + }) } + reason := fmt.Sprintf("watched outpoint %s spent externally by tx %s "+ - "at height %d", spentOutpoint, msg.SpendingTxid, + "at height %d", msg.Outpoint, msg.SpendingTxid, msg.SpendingHeight) return b.handleEvent(ctx, ax, &FailEvent{Reason: reason}) @@ -1841,6 +1984,14 @@ func (b *behavior) checkpointWrite() (*actorCheckpoint, } checkpoint := checkpointFromState(state, b.sweepTx) + + // Persist the sweep-finalized latch so it survives restart: commitAck + // re-persists this checkpoint atomically with the message ack, so a + // finalized sweep whose terminal handoff was deferred/failed is not + // lost — the actor rehydrates as terminal-eligible rather than stuck + // "provisional completed". + checkpoint.SweepFinalized = b.sweepFinalized + raw, err := encodeCheckpoint(checkpoint) if err != nil { return nil, nil, err @@ -2145,19 +2296,60 @@ func (b *behavior) failureReasonForTx(txid chainhash.Hash, return fmt.Sprintf("proof tx %s failed: %s", txid, reason) } +// maybeLatchSweepFinalized sets b.sweepFinalized when an incoming +// TxFinalizedMsg refers to the txid currently recorded as the sweep in +// PlannerState. The flag gates notifyRegistryIfTerminal so that a +// PhaseCompleted entry without a matching finalization stays +// provisional — the actor is kept alive by the registry until the +// sweep is past the backend's reorg-safety depth, so a reorg of the +// sweep confirmation has a live actor to receive the rollback. +// +// Late finalizations (e.g. for a proof tx, or for a sweep txid that has +// since been replaced by a re-broadcast) are ignored. +func (b *behavior) maybeLatchSweepFinalized(txid chainhash.Hash) { + state, err := b.currentState() + if err != nil { + return + } + + job := stateJob(state) + if job.PlannerState.Sweep.Txid.IsNone() { + return + } + if job.PlannerState.Sweep.Txid.UnsafeFromSome() != txid { + return + } + + b.sweepFinalized = true +} + // notifyRegistryIfTerminal forwards one UnrollTerminatedMsg to the -// registry when the FSM reaches Completed or Failed, at most once per -// actor lifetime. +// registry when the FSM reaches a TRULY terminal state, at most once +// per actor lifetime. +// +// PhaseFailed always means FailReason has been set (proof-tx terminal failure +// or sweep retry budget exhausted), so the actor is terminal immediately. // -// The registry uses this to move the outpoint out of its active map and -// mark the durable store terminal. If the FSM receives additional events -// after reaching terminal (e.g. a late TxConfirmed for a proof node that -// materialized before we failed for other reasons) terminalNotified -// keeps us from spamming the registry with repeats. +// PhaseCompleted is treated as PROVISIONAL until b.sweepFinalized is +// latched by a matching TxFinalizedMsg for the recorded sweep txid. +// While provisional, the registry keeps the child in its active map so +// a reorg of the sweep confirmation has a live actor to deliver the +// rollback to. The sweep's finality signal arrives via txconfirm's +// TxFinalized, which txconfirm emits off chainsource's height-based Done +// synthesis -- enabled for every backend (including lndclient over gRPC, +// whose native Done channel is allocated-but-never-written) by wiring +// FinalityDepth = configured policy horizon + 1 at waved/server.go. So the +// child IS evicted once the sweep confirmation buries past the +// reorg-safety depth, not retained indefinitely. // -// Failure to Tell is warned but not fatal — the registry will rediscover -// the terminal phase the next time it queries child state, and it holds -// its own persistence retry loop for the control-plane record. +// PhaseExternalSpendObserved is reversible by design and never reaches this +// function as terminal; SpendFinalizedMsg moves the actor to PhaseCompleted +// with ExternalSpendFinalized set so the consumed VTXO retires on chain. +// +// Failure to Tell is warned but not fatal — the registry will +// rediscover the terminal phase the next time it queries child state, +// and it holds its own persistence retry loop for the control-plane +// record. func (b *behavior) notifyRegistryIfTerminal(ctx context.Context) { state, err := b.currentState() if err != nil { @@ -2167,11 +2359,14 @@ func (b *behavior) notifyRegistryIfTerminal(ctx context.Context) { } phase := phaseFromState(state) - if phase != PhaseCompleted && phase != PhaseFailed { + job := stateJob(state) + trulyTerminal := phase == PhaseFailed || + (phase == PhaseCompleted && + (b.sweepFinalized || job.ExternalSpendFinalized)) + if !trulyTerminal { return } - job := stateJob(state) if !b.emitExitCostIfCompleted(ctx, phase, job) { // The exit-cost leg has not yet been durably handed to the @@ -2182,13 +2377,24 @@ func (b *behavior) notifyRegistryIfTerminal(ctx context.Context) { return } - // On a terminal failure, ask the wallet backend to drop the job's - // broadcast-but-unconfirmed transactions so a full-node wallet stops - // perpetually rebroadcasting a proof (or sweep) tx that can never - // confirm now that the exit has failed (wavelength#609). Gated by its - // own once-flag so it is independent of the registry handoff below and - // does not re-issue removals on every subsequent terminal tick. - if phase == PhaseFailed && !b.abandonedBroadcastsRemoved { + // On a terminal failure — or once an external spend that finalized the + // target out from under an in-flight exit is itself final — ask the + // wallet backend to drop the job's broadcast-but-unconfirmed + // transactions so a full-node wallet stops perpetually rebroadcasting a + // proof (or sweep) tx that can never confirm now that the exit is over + // (wavelength#609). Under reorg-safe exits an external target spend is + // a reversible completion rather than a failure, so we defer the + // cleanup to ExternalSpendFinalized: the still-in-flight proof tree is + // just as abandoned as on the failure path, but only once the spend can + // no longer reorg back out. A normal sweep completion is excluded — its + // transactions confirmed on chain and are never rebroadcast. Gated by + // its own once-flag so it is independent of the registry handoff below + // and does not re-issue removals on every subsequent terminal tick. + externalSpendFinal := phase == PhaseCompleted && + job.ExternalSpendFinalized + if (phase == PhaseFailed || externalSpendFinal) && + !b.abandonedBroadcastsRemoved { + b.removeAbandonedBroadcasts(ctx, job) b.abandonedBroadcastsRemoved = true } @@ -2203,6 +2409,7 @@ func (b *behavior) notifyRegistryIfTerminal(ctx context.Context) { Phase: phase, FailReason: job.FailReason, HadOnChainFootprint: jobHadOnChainFootprint(job), + ReliveUnsafe: job.ReliveUnsafe, ExitPolicyKind: b.exitPolicyKind(), } @@ -2372,7 +2579,8 @@ func (b *behavior) removeAbandonedBroadcasts(ctx context.Context, func (b *behavior) emitExitCostIfCompleted(ctx context.Context, phase Phase, job *JobState) bool { - if phase != PhaseCompleted || b.exitCostNotified || + if phase != PhaseCompleted || job.ExternalSpendFinalized || + b.exitCostNotified || b.cfg.LedgerSink.IsNone() { return true } diff --git a/unroll/actor_test.go b/unroll/actor_test.go index cd143c4d2..59922ac3d 100644 --- a/unroll/actor_test.go +++ b/unroll/actor_test.go @@ -406,6 +406,42 @@ func (f *fakeTxConfirmRef) emitFailed(t *testing.T, index int, require.NoError(t, err) } +// emitReorged delivers a txconfirm reorg notification to the subscriber +// behind the request at index. +func (f *fakeTxConfirmRef) emitReorged(t *testing.T, index int, + txid chainhash.Hash) { + + t.Helper() + + f.mu.Lock() + require.Less(t, index, len(f.requests)) + subscriber := f.requests[index].Subscriber + f.mu.Unlock() + + err := subscriber.Tell(t.Context(), &txconfirm.TxReorged{ + Txid: txid, + }) + require.NoError(t, err) +} + +// emitFinalized delivers a txconfirm finalized notification to the +// subscriber behind the request at index. +func (f *fakeTxConfirmRef) emitFinalized(t *testing.T, index int, + txid chainhash.Hash) { + + t.Helper() + + f.mu.Lock() + require.Less(t, index, len(f.requests)) + subscriber := f.requests[index].Subscriber + f.mu.Unlock() + + err := subscriber.Tell(t.Context(), &txconfirm.TxFinalized{ + Txid: txid, + }) + require.NoError(t, err) +} + // confRef aliases the chainsource confirmation notification target. type confRef = actor.TellOnlyRef[chainsource.ConfirmationEvent] @@ -415,16 +451,18 @@ type confReq = chainsource.RegisterConfRequest // fakeChainSourceRef is a minimal chainsource actor ref for sweep fee // estimation tests. type fakeChainSourceRef struct { - mu sync.Mutex - bestHeight int32 - feeRate int64 - feeErr error - blockRef actor.TellOnlyRef[chainsource.BlockEpoch] - spendRefs map[wire.OutPoint]spendEventRef - spendRegs []wire.OutPoint - removedTxes []chainhash.Hash - confRefs map[chainhash.Hash]confRef - confReqs map[chainhash.Hash]*confReq + mu sync.Mutex + bestHeight int32 + feeRate int64 + feeErr error + blockRef actor.TellOnlyRef[chainsource.BlockEpoch] + spendRefs map[wire.OutPoint]spendEventRef + spendRegs []wire.OutPoint + spendReorgedRef actor.TellOnlyRef[chainsource.SpendReorgedEvent] + spendFinalizedRef actor.TellOnlyRef[chainsource.SpendDoneEvent] + removedTxes []chainhash.Hash + confRefs map[chainhash.Hash]confRef + confReqs map[chainhash.Hash]*confReq } // spendEventRef is the fake chain-source spend notification actor reference. @@ -542,6 +580,18 @@ func (f *fakeChainSourceRef) Ask(_ context.Context, } f.spendRefs[outpoint] = msg.NotifyActor.UnwrapOr(nil) f.spendRegs = append(f.spendRegs, outpoint) + + // Reorg/finalized refs are only wired by ensureSpendWatch + // (target outpoint). Proof-node spend watches from + // ensureProofSpendWatches leave these unset; capture them + // only when the caller actually provided them so a later + // proof-node registration cannot wipe the target's refs. + if msg.NotifyReorged.IsSome() { + f.spendReorgedRef = msg.NotifyReorged.UnwrapOr(nil) + } + if msg.NotifyDone.IsSome() { + f.spendFinalizedRef = msg.NotifyDone.UnwrapOr(nil) + } f.mu.Unlock() promise.Complete( fn.Ok[chainsource.ChainSourceResp]( @@ -681,6 +731,40 @@ func (f *fakeChainSourceRef) removedTxSnapshot() []chainhash.Hash { return append([]chainhash.Hash(nil), f.removedTxes...) } +// emitSpendReorged delivers a SpendReorgedEvent to the subscribed actor. +func (f *fakeChainSourceRef) emitSpendReorged(t *testing.T) { + t.Helper() + + f.mu.Lock() + ref := f.spendReorgedRef + f.mu.Unlock() + + require.NotNil(t, ref) + require.NoError( + t, + ref.Tell( + t.Context(), chainsource.SpendReorgedEvent{}, + ), + ) +} + +// emitSpendFinalized delivers a SpendDoneEvent to the subscribed actor. +func (f *fakeChainSourceRef) emitSpendFinalized(t *testing.T) { + t.Helper() + + f.mu.Lock() + ref := f.spendFinalizedRef + f.mu.Unlock() + + require.NotNil(t, ref) + require.NoError( + t, + ref.Tell( + t.Context(), chainsource.SpendDoneEvent{}, + ), + ) +} + // fakeSweepWallet is a minimal signer plus wallet-destination test double. type fakeSweepWallet struct{} @@ -2182,6 +2266,11 @@ func TestResumeReissuesDeferredCheckpointWatch(t *testing.T) { t.Cleanup(resumedActor.Stop) mustAsk(t, resumedActor.Ref(), &ResumeUnrollRequest{Height: 111}) + checkpoint := mustDecodeCheckpoint(t, store, "resume-deferred-test") + require.True( + t, checkpoint.ReliveUnsafe, "restart must persist the "+ + "fail-closed relive guard before reissue", + ) require.Eventually(t, func() bool { return chainRef.confWatchCount() == 1 }, testTimeout, 10*time.Millisecond) @@ -2509,19 +2598,20 @@ func TestTerminalFailureRemovesAbandonedBroadcasts(t *testing.T) { }) // The root proof tx is submitted (in-flight) on start. Wait for the - // target spend watch, then fail the job with an external spend of the - // target outpoint (case 4). A target spend, unlike a root-output spend, - // does not confirm the still-in-flight root. + // reversible spend watches to arm, then spend the target externally + // (case 4). Under reorg-safe exits this is a reversible completion — + // the VTXO left on chain via someone else's tx — not an immediate + // failure, so the job parks in PhaseExternalSpendObserved until the + // spend finalizes. A target spend, unlike a root-output spend, does not + // confirm the still-in-flight root. target := proof.TargetOutpoint() rootTxid := proof.RootTxids()[0] require.Eventually(t, func() bool { - for _, outpoint := range chainSource.spendRegistrations() { - if outpoint == target { - return true - } - } + chainSource.mu.Lock() + defer chainSource.mu.Unlock() - return false + return chainSource.spendReorgedRef != nil && + chainSource.spendFinalizedRef != nil }, testTimeout, 10*time.Millisecond) chainSource.emitSpendForOutpoint(t, target, chainhash.Hash{0xee}, 101) @@ -2532,11 +2622,24 @@ func TestTerminalFailureRemovesAbandonedBroadcasts(t *testing.T) { ).(*GetStateResp) require.True(t, ok) - return stateResp.Phase == PhaseFailed + return stateResp.Phase == PhaseExternalSpendObserved + }, testTimeout, 10*time.Millisecond) + + // Finalize the external spend past the reorg-safety depth. The job + // completes and the abandoned in-flight root — which can never confirm + // now that the target is spent — must be removed so a full-node wallet + // stops rebroadcasting it (wavelength#609). + chainSource.emitSpendFinalized(t) + + require.Eventually(t, func() bool { + stateResp, ok := mustAsk( + t, unrollActor.Ref(), &GetStateRequest{}, + ).(*GetStateResp) + require.True(t, ok) + + return stateResp.Phase == PhaseCompleted }, testTimeout, 10*time.Millisecond) - // The in-flight root tx must have been removed from the wallet so a - // full-node wallet stops rebroadcasting it. require.Eventually(t, func() bool { for _, txid := range chainSource.removedTxSnapshot() { if txid == rootTxid { @@ -2546,7 +2649,8 @@ func TestTerminalFailureRemovesAbandonedBroadcasts(t *testing.T) { return false }, testTimeout, 10*time.Millisecond, - "terminal failure must remove the in-flight proof tx (#609)") + "terminal external spend must remove the in-flight proof tx "+ + "(#609)") } // TestRejectedProofTxIsRemovedOnTerminalFailure covers wavelength#609's own @@ -3127,6 +3231,12 @@ func TestSweepConfirmationCompletesActor(t *testing.T) { ) require.True(t, checkpoint.State.Sweep.ConfirmHeight.IsSome()) + // Reorg-safe completion treats PhaseCompleted as provisional on + // confirmation and defers the terminal handoff (and the ExitCostMsg + // emission) until the sweep finalizes past reorg-safety depth. Drive + // the finalize so the exit cost lands. + txconfirmRef.emitFinalized(t, 2, sweepTxid) + ledgerMsg, ok := ledgerSink.AwaitMessage(testTimeout) require.True(t, ok) exitCostMsg, ok := ledgerMsg.(*ledger.ExitCostMsg) @@ -3228,6 +3338,20 @@ func TestExitCostTellFailureDefersTerminalHandoff(t *testing.T) { return stateResp.Phase == PhaseCompleted }, testTimeout, 10*time.Millisecond) + // Finalize the sweep so PhaseCompleted is no longer provisional and + // the actor becomes terminal-eligible. The terminal handoff still + // requires a successful exit-cost emission, which fails here because + // the sink has no ledger actor behind it. + txconfirmRef.emitFinalized(t, 2, sweepTxid) + + // Synchronize with the actor before mutating its config below. The + // finalize above is delivered on the actor mailbox ahead of this Ask + // (FIFO), so by the time the Ask returns the deferred-handoff attempt + // it triggered -- including its read of cfg.LedgerSink -- has completed + // and happens-before the sink swap. Without this edge the race detector + // flags the concurrent field access against that read. + mustAsk(t, unrollActor.Ref(), &GetStateRequest{}) + // The failing ledger sink must have deferred the terminal handoff: // the registry sees no UnrollTerminatedMsg. _, ok := registryRef.AwaitMessage(50 * time.Millisecond) @@ -3591,12 +3715,19 @@ func TestSweepFailureRetriesThenFails(t *testing.T) { require.Equal(t, maxSweepAttempts, checkpoint.SweepAttempts) } -// TestExternalSpendTerminatesActor verifies that an external spend of the -// target VTXO (not our proof nodes or sweep) terminates the actor. +// TestExternalSpendTerminatesActor verifies that an external spend of +// the target VTXO is treated as a provisional block, and that +// finalization resolves it as a terminal on-chain completion. Without the +// finalization signal the actor stays in AwaitingExternalSpendFinality +// so a reorg of the spending block has a live actor to resume. func TestExternalSpendTerminatesActor(t *testing.T) { proof := buildLinearProof(t) desc := testDescriptor(t, proof.TargetOutpoint(), proof.CSVDelay()) - unrollActor, beh, _, _ := newActorHarness(t, proof, desc) + unrollActor, beh, _, store := newActorHarness(t, proof, desc) + registryRef := actor.NewChannelTellOnlyRef[RegistryMsg]( + "unroll-registry", 1, + ) + beh.cfg.RegistryRef = registryRef chainSource, ok := beh.cfg.ChainSource.(*fakeChainSourceRef) require.True(t, ok) @@ -3606,15 +3737,26 @@ func TestExternalSpendTerminatesActor(t *testing.T) { Trigger: TriggerManual, }) - // Ensure spend watch is registered. + // Ensure spend watch is registered for the target outpoint with + // reorg / done callback refs wired so the actor can be driven + // through the full reversible lifecycle. require.Eventually(t, func() bool { + var targetRegistered bool for _, outpoint := range chainSource.spendRegistrations() { if outpoint == proof.TargetOutpoint() { - return true + targetRegistered = true + break } } + if !targetRegistered { + return false + } - return false + chainSource.mu.Lock() + defer chainSource.mu.Unlock() + + return chainSource.spendReorgedRef != nil && + chainSource.spendFinalizedRef != nil }, testTimeout, 10*time.Millisecond) // Simulate an external party spending the target VTXO. @@ -3623,20 +3765,46 @@ func TestExternalSpendTerminatesActor(t *testing.T) { t, proof.TargetOutpoint(), externalTxid, 101, ) + // The actor must enter the reversible + // AwaitingExternalSpendFinality phase rather than terminating. require.Eventually(t, func() bool { stateResp, ok := mustAsk( t, unrollActor.Ref(), &GetStateRequest{}, ).(*GetStateResp) require.True(t, ok) - return stateResp.Phase == PhaseFailed + return stateResp.Phase == PhaseExternalSpendObserved + }, testTimeout, 10*time.Millisecond) + + // Finalize the spend. The actor resolves the target on chain and + // transitions to PhaseCompleted without fabricating a sweep. + chainSource.emitSpendFinalized(t) + + require.Eventually(t, func() bool { + stateResp, ok := mustAsk( + t, unrollActor.Ref(), &GetStateRequest{}, + ).(*GetStateResp) + require.True(t, ok) + + return stateResp.Phase == PhaseCompleted }, testTimeout, 10*time.Millisecond) stateResp, ok := mustAsk( t, unrollActor.Ref(), &GetStateRequest{}, ).(*GetStateResp) require.True(t, ok) - require.Contains(t, stateResp.FailReason, "spent externally") + require.Empty(t, stateResp.FailReason) + + terminalMsg, ok := registryRef.AwaitMessage(testTimeout) + require.True(t, ok, "final external spend must retire the target") + terminated, ok := terminalMsg.(*UnrollTerminatedMsg) + require.True(t, ok) + require.Equal(t, PhaseCompleted, terminated.Phase) + require.True(t, terminated.HadOnChainFootprint) + require.Nil(t, terminated.SweepTxid) + + checkpoint := mustDecodeCheckpoint(t, store, "unroll-test") + require.True(t, checkpoint.ExternalSpendFinalized) } // TestStartUnrollIsIdempotent verifies that reissuing start does not duplicate diff --git a/unroll/db_store.go b/unroll/db_store.go index fc1b8ef47..0acfeec8c 100644 --- a/unroll/db_store.go +++ b/unroll/db_store.go @@ -107,8 +107,9 @@ func (s *DBRegistryStore) ListNonTerminalRecords(ctx context.Context) ( // MarkTerminal marks one target terminal in the unilateral-exit job table. // recoverable selects UnilateralExitJobStatusFailedRecoverable over the -// plain Failed status for a no-footprint failure so boot-time reconciliation -// can roll the VTXO back to live (wavelength#602). +// plain Failed status for a no-footprint failure with objective +// canonical-absence evidence, allowing boot-time reconciliation to roll the +// VTXO back to live (wavelength#602). func (s *DBRegistryStore) MarkTerminal(ctx context.Context, target wire.OutPoint, phase Phase, recoverable bool, failReason string, sweepTxid *chainhash.Hash) error { @@ -194,8 +195,8 @@ func registryExitPolicy(record RegistryRecord, return exitPolicyKind(record.ExitPolicyKind), record.ExitPolicyRef } -// statusForRecord maps a registry record into the DB status enum, routing a -// recoverable (no-footprint) failure to the distinct FailedRecoverable status +// statusForRecord maps a registry record into the DB status enum, routing an +// authoritatively recoverable failure to the distinct FailedRecoverable status // so it round-trips back to RecoverableFailure=true on the next read. func statusForRecord(record RegistryRecord) db.UnilateralExitJobStatus { if record.Phase == PhaseFailed && record.RecoverableFailure { @@ -230,6 +231,20 @@ func statusForPhase(phase Phase) db.UnilateralExitJobStatus { case PhaseFailed: return db.UnilateralExitJobStatusFailed + case PhaseMaterializing: + return db.UnilateralExitJobStatusMaterializing + + // PhaseExternalSpendObserved is deliberately collapsed onto + // Materializing: it is a transient, reorg-reversible parked phase (an + // external spend was seen but has not finalized), the DB status enum + // has no dedicated value for it, and the true phase is always + // reconstructed from the durable checkpoint on restore -- so no state + // is lost. The collapse only affects the coarse operator-facing status + // during the parked window; add a dedicated enum value if that window + // needs to be independently observable. + case PhaseExternalSpendObserved: + return db.UnilateralExitJobStatusMaterializing + default: return db.UnilateralExitJobStatusMaterializing } diff --git a/unroll/fsm_logic.go b/unroll/fsm_logic.go index e9f0df9ac..8d592facf 100644 --- a/unroll/fsm_logic.go +++ b/unroll/fsm_logic.go @@ -77,6 +77,10 @@ func processEventWithJob(ctx context.Context, job *JobState, event Event, if e.Height > nextJob.Height { nextJob.Height = e.Height } + // A positive-only notifier cannot prove that no external spend + // landed while the daemon was offline. Persist the fail-closed + // relive guard before any reissued side effect. + nextJob.ReliveUnsafe = true reissue = true case *HeightUpdatedEvent: @@ -91,6 +95,41 @@ func processEventWithJob(ctx context.Context, job *JobState, event Event, case *TxFailedEvent: applyFailedEvent(nextJob, e) + case *TxReorgedEvent: + applyReorgedEvent(nextJob, e, env) + + case *TxFinalizedEvent: + // TxFinalized is informational at the unroll layer: the + // underlying chain anchor is no longer reversible, but the + // planner has already accounted for the confirmation. Nothing + // to mutate; we still run the planner below in case the + // finality changes any derived decision. + + case *ExternalSpendObservedEvent: + nextJob.ProvisionalExternalSpend = fn.Some(ExternalSpendAnchor{ + SpendingTxid: e.SpendingTxid, + SpendingHeight: e.SpendingHeight, + }) + if e.SpendingHeight > nextJob.Height { + nextJob.Height = e.SpendingHeight + } + + case *SpendReorgedEvent: + nextJob.ProvisionalExternalSpend = + fn.None[ExternalSpendAnchor]() + + case *SpendFinalizedEvent: + // Finalizing a provisional external spend resolves the target + // on chain. If no provisional anchor was held, SpendFinalized + // is informational and ignored. + nextJob.ProvisionalExternalSpend.WhenSome( + func(ExternalSpendAnchor) { + nextJob.ExternalSpendFinalized = true + nextJob.ProvisionalExternalSpend = + fn.None[ExternalSpendAnchor]() + }, + ) + case *SweepBroadcastedEvent: nextJob.PlannerState.Sweep.Status = unrollplan.SweepStatusBroadcasted @@ -194,10 +233,9 @@ func deriveStateTransition(ctx context.Context, job *JobState, env *Environment, return nil, err } - // Terminal short-circuit. applyFailedEvent populates FailReason - // for proof-tx terminal failures, and handleSpendObserved emits - // explicit FailEvents for external-spend detection; both land - // here before any planner work. + // Failure short-circuit. applyFailedEvent populates FailReason for + // proof-tx terminal failures; finalized external spends use the + // distinct consumed-success branch immediately below. if job.FailReason != "" { return &StateTransition{ NextState: &Failed{ @@ -206,6 +244,32 @@ func deriveStateTransition(ctx context.Context, job *JobState, env *Environment, }, nil } + // A finalized external spend is an objective terminal success from the + // VTXO lifecycle's perspective: the target is consumed and must retire. + if job.ExternalSpendFinalized { + return &StateTransition{ + NextState: &Completed{ + Job: job.Copy(), + }, + }, nil + } + + // Provisional external spend short-circuit. While the spend is + // observed but not finalized, the planner must not advance toward + // a sweep: on chain the target output no longer exists, and + // broadcasting a sweep on top would fail. A SpendReorgedEvent + // clears ProvisionalExternalSpend and the next derivation falls + // through to the normal planner phase decision; a + // SpendFinalizedEvent marks the target durably consumed and the + // completed branch above handles the terminal transition. + if job.ProvisionalExternalSpend.IsSome() { + return &StateTransition{ + NextState: &AwaitingExternalSpendFinality{ + Job: job.Copy(), + }, + }, nil + } + // Consult the pure planner. Plan() is stateless; it reads // PlannerState + the proof graph and returns a snapshot with Done // / NeedSweep / CSV / Ready fields. All phase decisions below @@ -441,6 +505,151 @@ func applyFailedEvent(job *JobState, event *TxFailedEvent) { job.FailReason = event.Reason } +// applyReorgedEvent rolls back the chain anchor for a previously +// confirmed proof or sweep transaction. The semantics mirror +// applyConfirmedEvent in reverse: +// +// - If the reorged txid matches the recorded sweep txid, downgrade +// SweepStatus from Confirmed back to Broadcasted and clear the +// stored ConfirmHeight. The signed sweep bytes are still durable +// in the actor checkpoint and the txconfirm subscription is still +// live, so the actor naturally re-runs AwaitingSweepConfirmation +// until the sweep reconfirms. +// +// - Otherwise the reorged txid is a proof node: drop it from +// ConfirmedTxids so the planner stops treating it as ready. If +// the reorged tx is the target itself, clear TargetConfirmHeight +// so CSV maturity is recomputed when the target reconfirms, and +// downgrade any sweep that depended on the now-invalidated target +// confirmation. The proof node remains broadcastable; the next +// deriveStateTransition will route an EnsureReadyTransactions for +// it if the planner's frontier turns up the txid again. +// +// Height is never rolled back: chain height is a global property +// reported by the block subscription, not by the per-tx watch, so a +// reorged confirmation does not move best-height down even though it +// invalidates the per-tx anchor. +func applyReorgedEvent(job *JobState, event *TxReorgedEvent, env *Environment) { + if job == nil || event == nil { + return + } + + // Sweep reorg: downgrade SweepStatus so deriveStateTransition lands + // in AwaitingSweepConfirmation (the sweep tx is still on chain, + // pending re-confirmation) rather than Completed. + if job.PlannerState.Sweep.Txid.IsSome() && + job.PlannerState.Sweep.Txid.UnsafeFromSome() == event.Txid { + + if job.PlannerState.Sweep.Status == + unrollplan.SweepStatusConfirmed { + + job.PlannerState.Sweep.Status = + unrollplan.SweepStatusBroadcasted + } + job.PlannerState.Sweep.ConfirmHeight = fn.None[int32]() + + return + } + + // Proof-node reorg: clear the confirmed anchor for this txid AND + // every descendant that we had recorded as confirmed or in-flight. + // State.Validate enforces a topological invariant (every + // confirmed/in-flight node has confirmed parents) that would fail + // if we dropped only the immediate ancestor and left descendants + // in place. Pruning the entire reorged subtree keeps the planner + // state internally consistent; txconfirm's txid-keyed dedup + // absorbs the re-submits when the planner re-emits the same nodes + // on its ready frontier after the parent reconfirms. + reorgedSubtree := collectReorgedSubtree(env, event.Txid) + job.PlannerState.ConfirmedTxids = removeHashes( + job.PlannerState.ConfirmedTxids, reorgedSubtree, + ) + job.PlannerState.InFlightTxids = removeHashes( + job.PlannerState.InFlightTxids, reorgedSubtree, + ) + for txid := range reorgedSubtree { + job.DeferredCheckpoints = removeDeferredCheckpoint( + job.DeferredCheckpoints, txid, + ) + } + + // If this proof tx was the target, the CSV anchor is also gone. + // unrollplan's State.Validate enforces "broadcasted / confirmed + // sweep requires confirmed target" so any non-pending sweep must + // be reset to Pending when the target loses its anchor. The + // signed sweep bytes still live in the actor checkpoint + // (b.sweepTx); a re-confirmed target drives NeedSweep again and + // startSweep reuses those bytes rather than deriving a new wallet + // pkScript or producing a different sweep txid. + if env != nil && env.Proof != nil && + event.Txid == env.Proof.TargetOutpoint().Hash { + + job.PlannerState.TargetConfirmHeight = fn.None[int32]() + + if job.PlannerState.Sweep.Status != + unrollplan.SweepStatusPending { + + job.PlannerState.Sweep.Status = + unrollplan.SweepStatusPending + job.PlannerState.Sweep.Txid = fn.None[chainhash.Hash]() + job.PlannerState.Sweep.ConfirmHeight = fn.None[int32]() + } + } +} + +// collectReorgedSubtree returns the set of every txid transitively +// descended from root within the proof graph, inclusive of root itself. +// A nil environment or missing root yields a singleton set so the +// reducer can still drop the reorged txid from local bookkeeping even +// when the proof is not loaded. +func collectReorgedSubtree(env *Environment, + root chainhash.Hash) map[chainhash.Hash]struct{} { + + subtree := map[chainhash.Hash]struct{}{root: {}} + if env == nil || env.Proof == nil { + return subtree + } + + queue := []chainhash.Hash{root} + for len(queue) > 0 { + next := queue[0] + queue = queue[1:] + + children, err := env.Proof.ChildTxids(next) + if err != nil { + continue + } + for _, child := range children { + if _, ok := subtree[child]; ok { + continue + } + subtree[child] = struct{}{} + queue = append(queue, child) + } + } + + return subtree +} + +// removeHashes returns hashes with every entry in drop removed. +func removeHashes(hashes []chainhash.Hash, + drop map[chainhash.Hash]struct{}) []chainhash.Hash { + + if len(hashes) == 0 || len(drop) == 0 { + return hashes + } + + filtered := hashes[:0:0] + for _, h := range hashes { + if _, ok := drop[h]; ok { + continue + } + filtered = append(filtered, h) + } + + return filtered +} + // applySweepBuildFailed records a sweep build or broadcast failure and // decides whether to terminate or retry. // diff --git a/unroll/fsm_types.go b/unroll/fsm_types.go index bab30dedb..3eacb12f2 100644 --- a/unroll/fsm_types.go +++ b/unroll/fsm_types.go @@ -103,6 +103,40 @@ type JobState struct { // SweepAttempts counts sweep build or broadcast failures so the actor // can retry up to maxSweepAttempts before giving up. SweepAttempts int + + // ProvisionalExternalSpend records an external spend of the target + // outpoint that has been observed but has not yet been finalized + // past the backend's reorg-safety depth. While set, the actor is + // parked: deriveStateTransition refuses to advance toward a sweep, + // since the on-chain state says the target output no longer exists. + // A SpendReorgedEvent clears the anchor and lets planning resume; + // a SpendFinalizedEvent resolves the target as permanently consumed. + // + // This field is persisted so a restart cannot forget a reversible + // spend and resume planning before canonical evidence is reconciled. + ProvisionalExternalSpend fn.Option[ExternalSpendAnchor] + + // ExternalSpendFinalized records that the provisional target spend + // crossed policy finality. It is a terminal on-chain resolution, not a + // recoverable failure: the VTXO manager must retire the consumed coin. + ExternalSpendFinalized bool + + // ReliveUnsafe is set before a job issues or reissues any chain side + // effect. Until authoritative canonical absence exists, a later + // clean-looking broadcast failure cannot prove the off-chain VTXO + // remained live. + ReliveUnsafe bool +} + +// ExternalSpendAnchor captures the identity of an external spend of the +// target outpoint that the actor has observed but has not yet treated +// as final. +type ExternalSpendAnchor struct { + // SpendingTxid is the txid that consumed the target outpoint. + SpendingTxid chainhash.Hash + + // SpendingHeight is the block height the spending tx confirmed at. + SpendingHeight int32 } // Copy returns a deep copy of the job state. @@ -113,14 +147,17 @@ func (j *JobState) Copy() *JobState { deferred := copyDeferredCheckpoints(j.DeferredCheckpoints) copyState := &JobState{ - Height: j.Height, - Trigger: j.Trigger, - ExitPolicyKind: exitPolicyKind(j.ExitPolicyKind), - ExitPolicyRef: j.ExitPolicyRef, - PlannerState: copyPlannerState(j.PlannerState), - DeferredCheckpoints: deferred, - FailReason: j.FailReason, - SweepAttempts: j.SweepAttempts, + Height: j.Height, + Trigger: j.Trigger, + ExitPolicyKind: exitPolicyKind(j.ExitPolicyKind), + ExitPolicyRef: j.ExitPolicyRef, + PlannerState: copyPlannerState(j.PlannerState), + DeferredCheckpoints: deferred, + FailReason: j.FailReason, + SweepAttempts: j.SweepAttempts, + ProvisionalExternalSpend: j.ProvisionalExternalSpend, + ExternalSpendFinalized: j.ExternalSpendFinalized, + ReliveUnsafe: j.ReliveUnsafe, } return copyState @@ -203,6 +240,57 @@ type TxFailedEvent struct { // eventSealed marks TxFailedEvent as an FSM event. func (e *TxFailedEvent) eventSealed() {} +// TxReorgedEvent records that a previously confirmed proof or sweep +// transaction was reorged out of the canonical chain. +type TxReorgedEvent struct { + // Txid is the reorged transaction hash. + Txid chainhash.Hash +} + +// eventSealed marks TxReorgedEvent as an FSM event. +func (e *TxReorgedEvent) eventSealed() {} + +// TxFinalizedEvent records that a confirmation is past the backend's +// reorg-safety depth. +type TxFinalizedEvent struct { + // Txid is the finalized transaction hash. + Txid chainhash.Hash +} + +// eventSealed marks TxFinalizedEvent as an FSM event. +func (e *TxFinalizedEvent) eventSealed() {} + +// ExternalSpendObservedEvent records an external spend of the target +// outpoint that has not yet been finalized. The reducer parks the +// actor in AwaitingExternalSpendFinality. +type ExternalSpendObservedEvent struct { + // SpendingTxid is the txid that consumed the target outpoint. + SpendingTxid chainhash.Hash + + // SpendingHeight is the block height the spending tx confirmed at. + SpendingHeight int32 +} + +// eventSealed marks ExternalSpendObservedEvent as an FSM event. +func (e *ExternalSpendObservedEvent) eventSealed() {} + +// SpendReorgedEvent records that a previously observed spend of the +// target outpoint was reorged out of the canonical chain. The reducer +// clears any provisional external-spend block on JobState. +type SpendReorgedEvent struct{} + +// eventSealed marks SpendReorgedEvent as an FSM event. +func (e *SpendReorgedEvent) eventSealed() {} + +// SpendFinalizedEvent records that a previously observed external spend +// of the target outpoint is past the backend's reorg-safety depth. The +// reducer promotes any provisional external-spend block to terminal consumed +// state so the VTXO is retired rather than restored. +type SpendFinalizedEvent struct{} + +// eventSealed marks SpendFinalizedEvent as an FSM event. +func (e *SpendFinalizedEvent) eventSealed() {} + // SweepBroadcastedEvent records that the actor built the final sweep and // submitted it to txconfirm. type SweepBroadcastedEvent struct { @@ -315,6 +403,7 @@ func (s *Idle) ProcessEvent(ctx context.Context, event Event, Trigger: e.Trigger, ExitPolicyKind: exitPolicyKind(e.ExitPolicyKind), ExitPolicyRef: e.ExitPolicyRef, + ReliveUnsafe: true, } return deriveStateTransition( @@ -323,9 +412,10 @@ func (s *Idle) ProcessEvent(ctx context.Context, event Event, case *ResumeEvent: job := &JobState{ - Height: e.Height, - Trigger: TriggerRestart, - FailReason: "", + Height: e.Height, + Trigger: TriggerRestart, + FailReason: "", + ReliveUnsafe: true, } return deriveStateTransition( @@ -445,7 +535,46 @@ func (s *AwaitingSweepConfirmation) ProcessEvent(ctx context.Context, return processEventWithJob(ctx, s.Job, event, env) } -// Completed indicates the final sweep has confirmed. +// AwaitingExternalSpendFinality indicates the actor has observed an +// external spend of the target outpoint but is waiting for either a +// reorg (which clears the observation and resumes planning) or a +// finality signal (which retires the consumed target). +type AwaitingExternalSpendFinality struct { + // Job is the durable FSM state. + Job *JobState +} + +// String returns a human-readable state label. +func (s *AwaitingExternalSpendFinality) String() string { + return "AwaitingExternalSpendFinality" +} + +// IsTerminal returns false because the observation is reversible. +func (s *AwaitingExternalSpendFinality) IsTerminal() bool { + return false +} + +// stateSealed marks AwaitingExternalSpendFinality as implementing State. +func (s *AwaitingExternalSpendFinality) stateSealed() {} + +// ProcessEvent delegates to the shared reducer so every event kind is +// applied uniformly; the reducer's planner-gate keeps the actor parked +// in this state until SpendReorgedEvent or SpendFinalizedEvent clears +// or promotes the provisional anchor. +func (s *AwaitingExternalSpendFinality) ProcessEvent(ctx context.Context, + event Event, env *Environment) (*StateTransition, error) { + + return processEventWithJob(ctx, s.Job, event, env) +} + +// Completed indicates the target is durably resolved on chain, either by the +// final sweep or by a finalized external spend. +// +// Completed is reversible until finality: a TxReorgedEvent for the +// recorded sweep txid (or for the target tx that the sweep depended on) +// must be routed through the same reducer as the non-terminal states so +// the actor can roll back to AwaitingSweepConfirmation when the chain +// disagrees. Any other event is rejected as before. type Completed struct { // Job is the durable FSM state. Job *JobState @@ -456,19 +585,33 @@ func (s *Completed) String() string { return "Completed" } -// IsTerminal returns true because this state is terminal. +// IsTerminal reports whether the planner-derived state is still +// terminal. Completed remains the public phase label, but a reorg event +// can re-enter the FSM so it cannot be marked terminal at the protofsm +// level — otherwise the engine refuses to deliver the rollback event. func (s *Completed) IsTerminal() bool { - return true + return false } // stateSealed marks Completed as implementing State. func (s *Completed) stateSealed() {} -// ProcessEvent rejects further events in the terminal completed state. -func (s *Completed) ProcessEvent(context.Context, Event, *Environment) ( - *StateTransition, error) { +// ProcessEvent applies reorg / finality events in the completed state +// and absorbs every other event kind as an idempotent no-op. Late +// chain notifications (a TxConfirmedEvent or HeightUpdatedEvent that +// raced the terminal transition while the registry is draining the +// actor for cleanup) are already reflected in the terminal checkpoint +// and should not error. +func (s *Completed) ProcessEvent(ctx context.Context, event Event, + env *Environment) (*StateTransition, error) { - return nil, fmt.Errorf("completed state is terminal") + switch event.(type) { + case *TxReorgedEvent, *TxFinalizedEvent: + return processEventWithJob(ctx, s.Job, event, env) + + default: + return &StateTransition{NextState: s}, nil + } } // Failed indicates the actor reached terminal failure. diff --git a/unroll/messages.go b/unroll/messages.go index 09e3bd00d..3499716d9 100644 --- a/unroll/messages.go +++ b/unroll/messages.go @@ -42,6 +42,10 @@ const ( txFailedMsgTLVType tlv.Type = 0x7904 getStateRequestTLVType tlv.Type = 0x7905 spendObservedMsgTLVType tlv.Type = 0x7906 + txReorgedMsgTLVType tlv.Type = 0x7907 + txFinalizedMsgTLVType tlv.Type = 0x7908 + spendReorgedMsgTLVType tlv.Type = 0x7909 + spendFinalizedMsgTLVType tlv.Type = 0x790a ) // Durable mailbox priorities. Admission stays at the default priority so a @@ -77,6 +81,9 @@ const ( spendObservedHeightRecType tlv.Type = 3 spendObservedOutHashRecType tlv.Type = 5 spendObservedOutIndexRecType tlv.Type = 7 + + txReorgedTxidRecType tlv.Type = 1 + txFinalizedTxidRecType tlv.Type = 1 ) // StartTrigger identifies what caused the unroll actor to start. @@ -125,11 +132,20 @@ const ( // awaiting confirmation. PhaseSweepConfirmation Phase = "sweep_confirmation" - // PhaseCompleted indicates the sweep confirmed successfully. + // PhaseCompleted indicates the target resolved on chain through either + // the final sweep or a finalized external spend. PhaseCompleted Phase = "completed" // PhaseFailed indicates the actor reached terminal failure. PhaseFailed Phase = "failed" + + // PhaseExternalSpendObserved indicates the actor has observed an + // external spend of the target outpoint that has not yet been + // finalized past the backend's reorg-safety depth. The actor is + // parked: it does not advance toward a sweep, but it has not + // terminated either, since a reorg of the spending block can + // resurrect the recovery job. + PhaseExternalSpendObserved Phase = "external_spend_observed" ) // Msg is the durable mailbox surface accepted by the VTXO unroll actor. @@ -800,6 +816,258 @@ func newCodec() *actor.MessageCodec { spendObservedMsgTLVType, func() actor.TLVMessage { return &SpendObservedMsg{} }, ) + codec.MustRegister( + txReorgedMsgTLVType, + func() actor.TLVMessage { return &TxReorgedMsg{} }, + ) + codec.MustRegister( + txFinalizedMsgTLVType, + func() actor.TLVMessage { return &TxFinalizedMsg{} }, + ) + codec.MustRegister( + spendReorgedMsgTLVType, + func() actor.TLVMessage { return &SpendReorgedMsg{} }, + ) + codec.MustRegister( + spendFinalizedMsgTLVType, + func() actor.TLVMessage { return &SpendFinalizedMsg{} }, + ) return codec } + +// SpendReorgedMsg reports that a previously delivered SpendObservedMsg +// for the target outpoint has been reorged out of the canonical chain. +// The actor must roll back any provisional external-spend state it +// recorded for the prior observation; if a new spend on the new tip +// follows, it arrives as a fresh SpendObservedMsg on the same +// subscription. +// +// The payload is intentionally empty: each unroll actor watches +// exactly one target outpoint, and the chainsource sub-actor is +// already keyed on that outpoint, so no additional correlation +// metadata is needed. +type SpendReorgedMsg struct { + actor.BaseMessage +} + +// MessageType returns the stable message type identifier. +func (m *SpendReorgedMsg) MessageType() string { + return "SpendReorgedMsg" +} + +// TLVType returns the durable mailbox type ID. +func (m *SpendReorgedMsg) TLVType() tlv.Type { + return spendReorgedMsgTLVType +} + +// Priority returns the durable mailbox priority for spend reorg events. +func (m *SpendReorgedMsg) Priority() int { + return unrollProgressPriority +} + +// Encode serializes the message as a (currently empty) TLV stream. +func (m *SpendReorgedMsg) Encode(w io.Writer) error { + stream, err := tlv.NewStream() + if err != nil { + return fmt.Errorf("create stream: %w", err) + } + + return stream.Encode(w) +} + +// Decode deserializes the message from a TLV stream. +func (m *SpendReorgedMsg) Decode(r io.Reader) error { + stream, err := tlv.NewStream() + if err != nil { + return fmt.Errorf("create stream: %w", err) + } + + if err := stream.Decode(r); err != nil { + return fmt.Errorf("decode: %w", err) + } + + return nil +} + +// unrollMsgSealed seals SpendReorgedMsg into the message surface. +func (m *SpendReorgedMsg) unrollMsgSealed() {} + +// SpendFinalizedMsg reports that a previously observed external spend +// of the target outpoint is past the backend's reorg-safety depth. +// Receiving it converts any provisional external-spend block on the +// actor into a terminal on-chain completion. +type SpendFinalizedMsg struct { + actor.BaseMessage +} + +// MessageType returns the stable message type identifier. +func (m *SpendFinalizedMsg) MessageType() string { + return "SpendFinalizedMsg" +} + +// TLVType returns the durable mailbox type ID. +func (m *SpendFinalizedMsg) TLVType() tlv.Type { + return spendFinalizedMsgTLVType +} + +// Priority returns the durable mailbox priority for spend finality +// events. +func (m *SpendFinalizedMsg) Priority() int { + return unrollProgressPriority +} + +// Encode serializes the message as a (currently empty) TLV stream. +func (m *SpendFinalizedMsg) Encode(w io.Writer) error { + stream, err := tlv.NewStream() + if err != nil { + return fmt.Errorf("create stream: %w", err) + } + + return stream.Encode(w) +} + +// Decode deserializes the message from a TLV stream. +func (m *SpendFinalizedMsg) Decode(r io.Reader) error { + stream, err := tlv.NewStream() + if err != nil { + return fmt.Errorf("create stream: %w", err) + } + + if err := stream.Decode(r); err != nil { + return fmt.Errorf("decode: %w", err) + } + + return nil +} + +// unrollMsgSealed seals SpendFinalizedMsg into the message surface. +func (m *SpendFinalizedMsg) unrollMsgSealed() {} + +// TxReorgedMsg reports that a previously delivered TxConfirmedMsg has +// been reorged out of the canonical chain. Subscribers should treat the +// prior confirmation as no longer valid; if the transaction re-confirms, +// a fresh TxConfirmedMsg will follow on the same subscription. +type TxReorgedMsg struct { + actor.BaseMessage + + // Txid identifies the reorged transaction. + Txid chainhash.Hash +} + +// MessageType returns the stable message type identifier. +func (m *TxReorgedMsg) MessageType() string { + return "TxReorgedMsg" +} + +// TLVType returns the durable mailbox type ID. +func (m *TxReorgedMsg) TLVType() tlv.Type { + return txReorgedMsgTLVType +} + +// Priority returns the durable mailbox priority for reorg events. Reorg +// rollback must run before low-value reads and height ticks because +// downstream side effects (e.g. sweep gating) gate on its outcome. +func (m *TxReorgedMsg) Priority() int { + return unrollProgressPriority +} + +// Encode serializes the message as a TLV stream. +func (m *TxReorgedMsg) Encode(w io.Writer) error { + txid := [32]byte(m.Txid) + + stream, err := tlv.NewStream( + tlv.MakePrimitiveRecord(txReorgedTxidRecType, &txid), + ) + if err != nil { + return fmt.Errorf("create stream: %w", err) + } + + return stream.Encode(w) +} + +// Decode deserializes the message from a TLV stream. +func (m *TxReorgedMsg) Decode(r io.Reader) error { + var txid [32]byte + + stream, err := tlv.NewStream( + tlv.MakePrimitiveRecord(txReorgedTxidRecType, &txid), + ) + if err != nil { + return fmt.Errorf("create stream: %w", err) + } + + if err := stream.Decode(r); err != nil { + return fmt.Errorf("decode: %w", err) + } + + m.Txid = chainhash.Hash(txid) + + return nil +} + +// unrollMsgSealed seals TxReorgedMsg into the message surface. +func (m *TxReorgedMsg) unrollMsgSealed() {} + +// TxFinalizedMsg reports that a tracked transaction is past the +// backend's reorg-safety depth and will receive no further events. +// Consumers may use this signal to drop any reorg-recovery bookkeeping +// they were holding for the watch. +type TxFinalizedMsg struct { + actor.BaseMessage + + // Txid identifies the finalized transaction. + Txid chainhash.Hash +} + +// MessageType returns the stable message type identifier. +func (m *TxFinalizedMsg) MessageType() string { + return "TxFinalizedMsg" +} + +// TLVType returns the durable mailbox type ID. +func (m *TxFinalizedMsg) TLVType() tlv.Type { + return txFinalizedMsgTLVType +} + +// Priority returns the durable mailbox priority for finality events. +func (m *TxFinalizedMsg) Priority() int { + return unrollProgressPriority +} + +// Encode serializes the message as a TLV stream. +func (m *TxFinalizedMsg) Encode(w io.Writer) error { + txid := [32]byte(m.Txid) + + stream, err := tlv.NewStream( + tlv.MakePrimitiveRecord(txFinalizedTxidRecType, &txid), + ) + if err != nil { + return fmt.Errorf("create stream: %w", err) + } + + return stream.Encode(w) +} + +// Decode deserializes the message from a TLV stream. +func (m *TxFinalizedMsg) Decode(r io.Reader) error { + var txid [32]byte + + stream, err := tlv.NewStream( + tlv.MakePrimitiveRecord(txFinalizedTxidRecType, &txid), + ) + if err != nil { + return fmt.Errorf("create stream: %w", err) + } + + if err := stream.Decode(r); err != nil { + return fmt.Errorf("decode: %w", err) + } + + m.Txid = chainhash.Hash(txid) + + return nil +} + +// unrollMsgSealed seals TxFinalizedMsg into the message surface. +func (m *TxFinalizedMsg) unrollMsgSealed() {} diff --git a/unroll/messages_test.go b/unroll/messages_test.go index 097e39204..e4321bcb4 100644 --- a/unroll/messages_test.go +++ b/unroll/messages_test.go @@ -127,6 +127,56 @@ func TestDurableMessageTLVRoundTrip(t *testing.T) { require.Equal(t, orig.SpendingHeight, got.SpendingHeight) }) + t.Run("TxReorgedMsg", func(t *testing.T) { + t.Parallel() + + orig := &TxReorgedMsg{Txid: txid} + + var buf bytes.Buffer + require.NoError(t, orig.Encode(&buf)) + + got := &TxReorgedMsg{} + require.NoError(t, got.Decode(bytes.NewReader(buf.Bytes()))) + require.Equal(t, orig.Txid, got.Txid) + }) + + t.Run("SpendReorgedMsg", func(t *testing.T) { + t.Parallel() + + orig := &SpendReorgedMsg{} + + var buf bytes.Buffer + require.NoError(t, orig.Encode(&buf)) + + got := &SpendReorgedMsg{} + require.NoError(t, got.Decode(bytes.NewReader(buf.Bytes()))) + }) + + t.Run("SpendFinalizedMsg", func(t *testing.T) { + t.Parallel() + + orig := &SpendFinalizedMsg{} + + var buf bytes.Buffer + require.NoError(t, orig.Encode(&buf)) + + got := &SpendFinalizedMsg{} + require.NoError(t, got.Decode(bytes.NewReader(buf.Bytes()))) + }) + + t.Run("TxFinalizedMsg", func(t *testing.T) { + t.Parallel() + + orig := &TxFinalizedMsg{Txid: txid} + + var buf bytes.Buffer + require.NoError(t, orig.Encode(&buf)) + + got := &TxFinalizedMsg{} + require.NoError(t, got.Decode(bytes.NewReader(buf.Bytes()))) + require.Equal(t, orig.Txid, got.Txid) + }) + t.Run("GetStateRequest", func(t *testing.T) { t.Parallel() diff --git a/unroll/notification_map_test.go b/unroll/notification_map_test.go index fea01e23a..d93a75a0c 100644 --- a/unroll/notification_map_test.go +++ b/unroll/notification_map_test.go @@ -10,11 +10,8 @@ import ( // TestMapTxconfirmNotification pins the disposition of every txconfirm // lifecycle event at the unroll subscriber boundary. The load-bearing -// cases are TxFinalized and TxReorged: subscribers receive the FULL -// reorg-aware lifecycle, and before this mapping existed both fell into -// the unknown-notification arm and terminally failed the unroll job — -// finality (synthesized by every non-lnd backend once the confirmation -// passed the reorg-safety depth) killed otherwise-healthy exits. +// cases are TxFinalized and TxReorged: this planner has durable transitions +// for both, so the adapter must preserve rather than drop or collapse them. func TestMapTxconfirmNotification(t *testing.T) { t.Parallel() @@ -37,9 +34,7 @@ func TestMapTxconfirmNotification(t *testing.T) { require.EqualValues(t, 3, confirmed.NumConfs) }) - t.Run("finalized maps to confirmation, not failure", func( - t *testing.T) { - + t.Run("finalized maps to finality", func(t *testing.T) { t.Parallel() msg, ok := mapTxconfirmNotification(&txconfirm.TxFinalized{ @@ -49,28 +44,21 @@ func TestMapTxconfirmNotification(t *testing.T) { }) require.True(t, ok) - confirmed, isConfirmed := msg.(*TxConfirmedMsg) - require.True( - t, isConfirmed, "finality must replay a "+ - "confirmation; mapping it to a failure "+ - "terminally kills the unroll job", - ) - require.Equal(t, txid, confirmed.Txid) - require.EqualValues(t, 130, confirmed.Height) - require.EqualValues(t, 6, confirmed.NumConfs) + finalized, isFinalized := msg.(*TxFinalizedMsg) + require.True(t, isFinalized) + require.Equal(t, txid, finalized.Txid) }) - t.Run("reorged is dropped", func(t *testing.T) { + t.Run("reorged maps to reorg", func(t *testing.T) { t.Parallel() - _, ok := mapTxconfirmNotification(&txconfirm.TxReorged{ + msg, ok := mapTxconfirmNotification(&txconfirm.TxReorged{ Txid: txid, }) - require.False( - t, ok, "TxReorged is best-effort and superseded by "+ - "the next reliable event; it must not "+ - "become a failure", - ) + require.True(t, ok) + reorged, isReorged := msg.(*TxReorgedMsg) + require.True(t, isReorged) + require.Equal(t, txid, reorged.Txid) }) t.Run("failed maps to failure", func(t *testing.T) { diff --git a/unroll/reconcile.go b/unroll/reconcile.go new file mode 100644 index 000000000..1aec81387 --- /dev/null +++ b/unroll/reconcile.go @@ -0,0 +1,320 @@ +package unroll + +import ( + "context" + "fmt" + + "github.com/btcsuite/btcd/chainhash/v2" + "github.com/btcsuite/btcd/wire/v2" + "github.com/lightninglabs/wavelength/lib/recovery" + "github.com/lightninglabs/wavelength/unrollplan" + fn "github.com/lightningnetwork/lnd/fn/v2" +) + +// ConfirmedAnchor describes a tx confirmation that the backend reports +// is currently live on the canonical chain. +type ConfirmedAnchor struct { + // Txid is the confirmed transaction hash. + Txid chainhash.Hash + + // Height is the block height the transaction confirmed at. + Height int32 +} + +// SpendAnchor describes an outpoint spend that the backend reports is +// currently live on the canonical chain. +type SpendAnchor struct { + // Outpoint is the spent outpoint. + Outpoint wire.OutPoint + + // SpendingTxid is the transaction that consumed the outpoint. + SpendingTxid chainhash.Hash + + // SpendingHeight is the block height the spending tx confirmed at. + SpendingHeight int32 +} + +// ChainReconciler is the narrow interface the unroll actor uses on +// restart to verify that the chain anchors recorded in its checkpoint +// are still live on the canonical chain. +// +// The interface is intentionally small: it answers two questions, both +// keyed on data the actor already has in its checkpoint. Implementations +// query the backend (lnd's chainntnfs, neutrino, or a bitcoind RPC) and +// return fn.None for "not on chain anymore" so the unroll layer can +// stay backend-agnostic. +// +// Both methods return errors for transport-level failures the caller +// should retry; an "anchor is not on chain" answer is conveyed via +// fn.None, not via error. +type ChainReconciler interface { + // ConfirmedTx reports the current confirmation status of a tx on + // the canonical chain. fn.None means the tx is no longer + // confirmed (typically because the block was reorged out while + // the daemon was offline). + ConfirmedTx(ctx context.Context, + txid chainhash.Hash) (fn.Option[ConfirmedAnchor], error) + + // SpentOutpoint reports the current spend status of an outpoint + // on the canonical chain. fn.None means the outpoint is currently + // unspent. + SpentOutpoint(ctx context.Context, + outpoint wire.OutPoint) (fn.Option[SpendAnchor], error) +} + +// NegativeEvidenceReconciler optionally declares whether fn.None is backed by +// an authoritative completed query. Reconcilers that do not implement this +// interface retain the full ChainReconciler contract and are assumed capable +// of proving both positive and negative answers. +type NegativeEvidenceReconciler interface { + // HasAuthoritativeNegativeEvidence reports whether the implementation + // can prove absence without inferring it from silence or a timeout. + HasAuthoritativeNegativeEvidence() bool +} + +// ChainReconcilerFactory builds a ChainReconciler bound to a specific +// per-target unroll actor. Per-target actors invoke the factory after +// loading their proof; the target outpoint identifies the calling +// actor and lets the implementation namespace its chainsource probe +// caller IDs so two actors that share a proof-graph ancestor (the +// common case for sibling VTXOs) cannot collide on chainsource +// service keys when they reconcile concurrently. The proof argument +// supplies output scripts because lnd's tx index is not always +// available, so the (txid, pkScript) pair is the canonical way to +// identify a tx during a historical block scan. Test stubs that do +// not need either argument can ignore them. +type ChainReconcilerFactory func(target wire.OutPoint, + proof *recovery.Proof) ChainReconciler + +// reconcileCheckpoint walks the persisted checkpoint and prunes any +// chain anchor that the reconciler reports is no longer live on the +// canonical chain. The checkpoint is mutated in place. +// +// The reconciliation order mirrors applyReorgedEvent: +// +// - For every confirmed proof-graph txid, ask the reconciler whether +// it is still confirmed. Anchors that are not are dropped, and so +// are their descendants in the proof graph (State.Validate's +// topological invariant requires every confirmed/in-flight node to +// have confirmed parents). +// - If the target tx was confirmed but is now absent, clear +// TargetConfirmHeight and downgrade the sweep to Pending — same +// gate the live reorg reducer enforces. +// - If a Broadcasted / Confirmed sweep tx is no longer on chain, +// reset the sweep state so the actor re-broadcasts on its next +// planning step. +// - Reconcile the target outpoint spend status against any persisted +// ProvisionalExternalSpend anchor: set, refresh, or clear it so +// the restored FSM enters the same parked state a live +// SpendObservedEvent / SpendReorgedEvent sequence would have +// produced. +// +// On any reconciler error we surface it untouched: the actor must not +// proceed to broadcast off a checkpoint we could not verify. +func reconcileCheckpoint(ctx context.Context, reconciler ChainReconciler, + proof *recovery.Proof, checkpoint *actorCheckpoint) error { + + if reconciler == nil || checkpoint == nil || proof == nil { + return nil + } + + // Snapshot the input list so the loop iterates over a stable + // slice while we mutate the live ConfirmedTxids field. + confirmedIn := append( + []chainhash.Hash(nil), checkpoint.State.ConfirmedTxids..., + ) + + for _, txid := range confirmedIn { + anchor, err := reconciler.ConfirmedTx(ctx, txid) + if err != nil { + return fmt.Errorf("reconcile confirmed tx %s: %w", txid, + err) + } + if anchor.IsSome() { + // The tx is still confirmed. Refresh the + // target-derived height in case the tx was re-mined + // at a different block height while the daemon was + // offline; leaving the stale height in place would + // make CSV maturity look closer than it actually is + // and could drive a premature sweep on the next + // planning step. Also raise the checkpoint's + // best-height watermark to at least the live anchor + // height so derived state is internally consistent. + liveHeight := anchor.UnsafeFromSome().Height + if txid == proof.TargetOutpoint().Hash { + checkpoint.State.TargetConfirmHeight = fn.Some( + liveHeight, + ) + } + if liveHeight > checkpoint.Height { + checkpoint.Height = liveHeight + } + + continue + } + + pruneReorgedSubtree(proof, &checkpoint.State, txid) + + if txid == proof.TargetOutpoint().Hash { + downgradeSweepOnTargetLoss(&checkpoint.State) + } + } + + // Verify only a sweep checkpoint that already claims confirmation. + // A merely broadcast sweep may legitimately be unconfirmed, so notifier + // silence cannot distinguish its expected state from a reorg. + if checkpoint.State.Sweep.Txid.IsSome() && + checkpoint.State.Sweep.Status == + unrollplan.SweepStatusConfirmed { + + sweepTxid := checkpoint.State.Sweep.Txid.UnsafeFromSome() + anchor, err := reconciler.ConfirmedTx(ctx, sweepTxid) + if err != nil { + return fmt.Errorf("reconcile sweep tx %s: %w", + sweepTxid, err) + } + switch { + case anchor.IsNone(): + // Sweep is no longer on chain. Reset so the next + // derivation re-broadcasts using the cached bytes. + checkpoint.State.Sweep.Status = + unrollplan.SweepStatusPending + checkpoint.State.Sweep.Txid = + fn.None[chainhash.Hash]() + checkpoint.State.Sweep.ConfirmHeight = + fn.None[int32]() + + case checkpoint.State.Sweep.Status == + unrollplan.SweepStatusConfirmed: + + // Sweep is still on chain but at potentially a new + // height. Recompute ConfirmHeight from the live + // answer so a sweep that confirmed in a different + // block is not stuck reporting a stale height. + checkpoint.State.Sweep.ConfirmHeight = fn.Some( + anchor.UnsafeFromSome().Height, + ) + } + } + + if err := reconcileExternalSpend( + ctx, reconciler, proof, checkpoint, + ); err != nil { + return err + } + + return nil +} + +// reconcileExternalSpend cross-checks the target outpoint's spend +// status on the canonical chain against any ProvisionalExternalSpend +// anchor the checkpoint carries. Four cases land here: +// +// 1. Chain says target spent + checkpoint anchor matches the same +// spender txid: no-op (refresh the height in case it shifted to a +// re-organized block). +// 2. Chain says target spent + checkpoint anchor for a DIFFERENT +// spender: update the anchor to the live spender. +// 3. Chain says target spent + no checkpoint anchor: install one. +// This is the "external spend confirmed while the daemon was +// offline" path. +// 4. Chain says target unspent + checkpoint anchor present: clear +// the anchor. The spending block was reorged out during downtime. +// +// In cases 2/3, "spent by a proof-graph node" or "spent by our own +// sweep" are treated as benign: those are the same classifications +// the live spend-watch handler runs in handleSpendObserved. +func reconcileExternalSpend(ctx context.Context, reconciler ChainReconciler, + proof *recovery.Proof, checkpoint *actorCheckpoint) error { + + // With no persisted positive there is nothing a positive-only notifier + // can safely disprove. The live historical spend watch is re-armed + // before Resume routes work; later canonicality Snapshot/Ready supplies + // the authoritative negative barrier required by one-confirmation mode. + if checkpoint.ProvisionalExternalSpend.IsNone() { + capability, ok := reconciler.(NegativeEvidenceReconciler) + if ok && !capability.HasAuthoritativeNegativeEvidence() { + return nil + } + } + + targetOutpoint := proof.TargetOutpoint() + anchor, err := reconciler.SpentOutpoint(ctx, targetOutpoint) + if err != nil { + return fmt.Errorf("reconcile target spend %s: %w", + targetOutpoint, err) + } + + if anchor.IsNone() { + // Case 4: chain reports target unspent. Any persisted + // anchor was reorged out while we were offline; clear it + // so the FSM does NOT park in + // AwaitingExternalSpendFinality on restore. + checkpoint.ProvisionalExternalSpend = + fn.None[ExternalSpendAnchor]() + + return nil + } + + live := anchor.UnsafeFromSome() + + // Skip benign spenders that the live spend-watch path would also + // classify as "expected materialization traffic": a proof-graph + // node, or our own sweep. + if proof != nil { + if _, ok := proof.Node(live.SpendingTxid); ok { + checkpoint.ProvisionalExternalSpend = + fn.None[ExternalSpendAnchor]() + + return nil + } + } + if checkpoint.State.Sweep.Txid.IsSome() && + checkpoint.State.Sweep.Txid.UnsafeFromSome() == + live.SpendingTxid { + + checkpoint.ProvisionalExternalSpend = + fn.None[ExternalSpendAnchor]() + + return nil + } + + // Cases 1, 2, 3: install or refresh the anchor with the live + // spender so the restored FSM enters AwaitingExternalSpendFinality + // rather than blindly resuming planner-driven materialization. + checkpoint.ProvisionalExternalSpend = fn.Some(ExternalSpendAnchor{ + SpendingTxid: live.SpendingTxid, + SpendingHeight: live.SpendingHeight, + }) + + return nil +} + +// pruneReorgedSubtree drops a txid and every transitive descendant from +// both ConfirmedTxids and InFlightTxids, and clears any deferred +// checkpoints referencing the subtree. +func pruneReorgedSubtree(proof *recovery.Proof, state *unrollplan.State, + root chainhash.Hash) { + + subtree := collectReorgedSubtree( + &Environment{Proof: proof}, root, + ) + state.ConfirmedTxids = removeHashes(state.ConfirmedTxids, subtree) + state.InFlightTxids = removeHashes(state.InFlightTxids, subtree) +} + +// downgradeSweepOnTargetLoss resets the target-derived planner state +// when the target tx is no longer on the canonical chain. Mirrors the +// "target reorg" branch of applyReorgedEvent so restart reconciliation +// and live rollback converge on identical post-conditions. +func downgradeSweepOnTargetLoss(state *unrollplan.State) { + state.TargetConfirmHeight = fn.None[int32]() + + if state.Sweep.Status == unrollplan.SweepStatusPending { + return + } + + state.Sweep.Status = unrollplan.SweepStatusPending + state.Sweep.Txid = fn.None[chainhash.Hash]() + state.Sweep.ConfirmHeight = fn.None[int32]() +} diff --git a/unroll/reconcile_chainsource.go b/unroll/reconcile_chainsource.go new file mode 100644 index 000000000..366f48480 --- /dev/null +++ b/unroll/reconcile_chainsource.go @@ -0,0 +1,319 @@ +package unroll + +import ( + "context" + "errors" + "fmt" + "log/slog" + "time" + + "github.com/btcsuite/btcd/chainhash/v2" + "github.com/btcsuite/btcd/wire/v2" + "github.com/btcsuite/btclog/v2" + "github.com/lightninglabs/wavelength/baselib/actor" + "github.com/lightninglabs/wavelength/chainsource" + "github.com/lightninglabs/wavelength/lib/recovery" + fn "github.com/lightningnetwork/lnd/fn/v2" +) + +// defaultReconcileProbeTimeout bounds how long the chainsource-backed +// reconciler waits for a confirmation or spend event before reporting +// that canonical evidence is unavailable. lnd's chainntnfs dispatches +// historical confirmation events immediately when the tx is in chain +// and the height-hint is fresh; legitimate "still confirmed" answers +// return well under a second on a healthy node, so a 10-second +// budget leaves comfortable headroom for slow restarts while still +// failing closed when the backend cannot answer. +const defaultReconcileProbeTimeout = 10 * time.Second + +// ErrReconcileEvidenceUnavailable means restart reconciliation could not +// obtain objective chain evidence. Callers must retain the durable checkpoint +// and retry; absence or timeout is never proof of a reorg or an unspent output. +var ErrReconcileEvidenceUnavailable = errors.New("reconciliation chain " + + "evidence unavailable") + +// ChainSourceReconcilerConfig configures NewChainSourceReconciler. +type ChainSourceReconcilerConfig struct { + // ChainSource is the actor ref used to issue RegisterConf / + // RegisterSpend probes. + ChainSource actor.ActorRef[ + chainsource.ChainSourceMsg, chainsource.ChainSourceResp, + ] + + // Proof is the immutable recovery graph the reconciler queries + // pkScripts from. Required. + Proof *recovery.Proof + + // CallerID prefix used when registering probes; the reconciler + // appends per-probe suffixes so chainsource sees a unique key + // per (caller, txid / outpoint, confs) tuple. + // + // Callers that share the underlying ChainSource between multiple + // reconcilers MUST give each instance a unique prefix (typically + // by including the per-actor target identity) so concurrent + // probes of the same shared proof-graph txid do not collide on + // the chainsource service-key namespace. + CallerID string + + // ProbeTimeout bounds each individual confirmation / spend + // probe. Zero falls back to defaultReconcileProbeTimeout. + // Slow chainsource backends can legitimately need a higher + // budget here. A probe that times out leaves the durable checkpoint + // unchanged and causes restart recovery to fail closed for retry. + ProbeTimeout time.Duration + + // CleanupTimeout bounds the best-effort unregister send that + // runs when a probe is interrupted before its future fires. + // Zero falls back to 5 seconds. + CleanupTimeout time.Duration + + // Log is an optional logger. Probe timeouts are surfaced at warn level + // so operators can distinguish unavailable evidence from an actual + // negative canonicality result. Zero falls back to btclog.Disabled. + Log fn.Option[btclog.Logger] +} + +// chainSourceReconciler is a ChainReconciler that answers queries by +// probing the chainsource actor with short-timeout RegisterConf and +// RegisterSpend requests in future mode. +// +// The implementation is intentionally simple: it does NOT consume a +// dedicated "is this tx in chain right now" API (chainsource does not expose +// one today). Instead it leans on lnd's chainntnfs behavior of dispatching a +// historical positive immediately when the watched item is currently on +// chain. Silence cannot prove the negative, so a timed-out probe returns +// ErrReconcileEvidenceUnavailable and leaves the durable checkpoint intact. +// +// Probes that complete on their own self-clean (the chainsource +// sub-actor exits after delivering the single positive event in +// future mode); probes that time out enqueue a best-effort +// UnregisterConfRequest on a fresh background context so the +// long-lived chainsource sub-actor does not leak per restart. +type chainSourceReconciler struct { + chainSource actor.ActorRef[ + chainsource.ChainSourceMsg, chainsource.ChainSourceResp, + ] + proof *recovery.Proof + callerID string + probeTimeout time.Duration + cleanupTimeout time.Duration + log btclog.Logger +} + +// HasAuthoritativeNegativeEvidence reports false because chainsource's current +// subscription API emits positives but has no completed-scan absence result. +func (r *chainSourceReconciler) HasAuthoritativeNegativeEvidence() bool { + return false +} + +// NewChainSourceReconciler constructs a chainsource-backed +// ChainReconciler. The proof is required because chainsource probes +// rely on (txid, pkScript) pairs for historical scans. +func NewChainSourceReconciler(cfg ChainSourceReconcilerConfig) ChainReconciler { + if cfg.Proof == nil { + return nil + } + probeTimeout := cfg.ProbeTimeout + if probeTimeout <= 0 { + probeTimeout = defaultReconcileProbeTimeout + } + cleanupTimeout := cfg.CleanupTimeout + if cleanupTimeout <= 0 { + cleanupTimeout = 5 * time.Second + } + + return &chainSourceReconciler{ + chainSource: cfg.ChainSource, + proof: cfg.Proof, + callerID: cfg.CallerID, + probeTimeout: probeTimeout, + cleanupTimeout: cleanupTimeout, + log: cfg.Log.UnwrapOr(btclog.Disabled), + } +} + +// ConfirmedTx probes chainsource for the current confirmation status +// of txid. The probe registers a one-shot conf watch in future mode +// using the txid + first-output pkScript from the proof, then awaits +// the future with a bounded timeout. +func (r *chainSourceReconciler) ConfirmedTx(ctx context.Context, + txid chainhash.Hash) (fn.Option[ConfirmedAnchor], error) { + + node, ok := r.proof.Node(txid) + if !ok || node == nil || node.Tx == nil || + len(node.Tx.TxOut) == 0 { + return fn.None[ConfirmedAnchor](), fmt.Errorf("%w: tx %s has "+ + "no queryable output", ErrReconcileEvidenceUnavailable, + txid) + } + pkScript := append([]byte(nil), node.Tx.TxOut[0].PkScript...) + + callerID := fmt.Sprintf("%s-conf-%s", r.callerID, txid) + probeCtx, cancel := context.WithTimeout(ctx, r.probeTimeout) + defer cancel() + + resp, err := r.chainSource.Ask( + probeCtx, &chainsource.RegisterConfRequest{ + CallerID: callerID, + Txid: &txid, + PkScript: pkScript, + TargetConfs: 1, + }, + ).Await(probeCtx).Unpack() + if err != nil { + return fn.None[ConfirmedAnchor](), fmt.Errorf("%w: register "+ + "conf probe %s: %w", ErrReconcileEvidenceUnavailable, + txid, err) + } + + confResp, ok := resp.(*chainsource.RegisterConfResponse) + if !ok || confResp.Future == nil { + return fn.None[ConfirmedAnchor](), fmt.Errorf("register conf "+ + "probe %s: unexpected response %T", txid, resp) + } + + // Schedule a best-effort cleanup unregister in case the probe + // times out: the chainsource sub-actor would otherwise stay + // alive watching for a confirmation that, by assumption, is + // never coming. The cleanup runs on a fresh background context + // so it executes even when the probe context was cancelled. + probeTxid := txid + probePkScript := pkScript + //nolint:contextcheck // cleanup intentionally uses its own context + defer r.cleanupConfWatch(callerID, &probeTxid, probePkScript) + + event, err := confResp.Future.Await(probeCtx).Unpack() + if err != nil { + if errors.Is(err, context.DeadlineExceeded) || + errors.Is(err, context.Canceled) { + + r.log.WarnS(ctx, "Reconciler conf probe timed out; "+ + "leaving checkpoint unchanged", err, + slog.String("txid", txid.String()), + slog.Duration("timeout", r.probeTimeout), + ) + } + + return fn.None[ConfirmedAnchor](), fmt.Errorf("%w: await conf "+ + "probe %s: %w", ErrReconcileEvidenceUnavailable, txid, + err) + } + + return fn.Some(ConfirmedAnchor{ + Txid: event.Txid, + Height: event.BlockHeight, + }), nil +} + +// SpentOutpoint probes chainsource for the current spend status of +// outpoint. Symmetric to ConfirmedTx: registers a future-mode spend +// watch with the outpoint's pkScript and awaits the future with a +// bounded timeout, scheduling an UnregisterSpendRequest cleanup on +// the way out. +func (r *chainSourceReconciler) SpentOutpoint(ctx context.Context, + outpoint wire.OutPoint) (fn.Option[SpendAnchor], error) { + + node, ok := r.proof.Node(outpoint.Hash) + if !ok || node == nil || node.Tx == nil { + return fn.None[SpendAnchor](), fmt.Errorf("%w: outpoint %s "+ + "has no queryable transaction", + ErrReconcileEvidenceUnavailable, outpoint) + } + if int(outpoint.Index) >= len(node.Tx.TxOut) { + return fn.None[SpendAnchor](), fmt.Errorf("%w: outpoint %s "+ + "has no queryable output", + ErrReconcileEvidenceUnavailable, outpoint) + } + pkScript := append( + []byte(nil), node.Tx.TxOut[outpoint.Index].PkScript..., + ) + + callerID := fmt.Sprintf("%s-spend-%s", r.callerID, outpoint) + probeCtx, cancel := context.WithTimeout(ctx, r.probeTimeout) + defer cancel() + + probeOutpoint := outpoint + resp, err := r.chainSource.Ask( + probeCtx, &chainsource.RegisterSpendRequest{ + CallerID: callerID, + Outpoint: &probeOutpoint, + PkScript: pkScript, + }, + ).Await(probeCtx).Unpack() + if err != nil { + return fn.None[SpendAnchor](), fmt.Errorf("%w: register spend "+ + "probe %s: %w", ErrReconcileEvidenceUnavailable, + outpoint, err) + } + + spendResp, ok := resp.(*chainsource.RegisterSpendResponse) + if !ok || spendResp.Future == nil { + return fn.None[SpendAnchor](), fmt.Errorf("register spend "+ + "probe %s: unexpected response %T", outpoint, resp) + } + + //nolint:contextcheck // cleanup intentionally uses its own context + defer r.cleanupSpendWatch(callerID, &probeOutpoint) + + event, err := spendResp.Future.Await(probeCtx).Unpack() + if err != nil { + if errors.Is(err, context.DeadlineExceeded) || + errors.Is(err, context.Canceled) { + + r.log.WarnS(ctx, "Reconciler spend probe timed "+ + "out; leaving checkpoint unchanged", err, + slog.String("outpoint", outpoint.String()), + slog.Duration("timeout", r.probeTimeout), + ) + } + + return fn.None[SpendAnchor](), fmt.Errorf("%w: await spend "+ + "probe %s: %w", ErrReconcileEvidenceUnavailable, + outpoint, err) + } + + return fn.Some(SpendAnchor{ + Outpoint: event.Outpoint, + SpendingTxid: event.SpendingTxid, + SpendingHeight: event.SpendingHeight, + }), nil +} + +// cleanupConfWatch sends a best-effort UnregisterConfRequest on a +// fresh background context so a timed-out probe does not leak the +// underlying chainsource sub-actor. Errors are intentionally ignored +// because the cleanup is purely a hygiene operation. +func (r *chainSourceReconciler) cleanupConfWatch(callerID string, + txid *chainhash.Hash, pkScript []byte) { + + cleanupCtx, cancel := context.WithTimeout( + context.Background(), r.cleanupTimeout, + ) + defer cancel() + + _, _ = r.chainSource.Ask( + cleanupCtx, &chainsource.UnregisterConfRequest{ + CallerID: callerID, + Txid: txid, + PkScript: pkScript, + TargetConfs: 1, + }, + ).Await(cleanupCtx).Unpack() +} + +// cleanupSpendWatch is the spend-side analogue of cleanupConfWatch. +func (r *chainSourceReconciler) cleanupSpendWatch(callerID string, + outpoint *wire.OutPoint) { + + cleanupCtx, cancel := context.WithTimeout( + context.Background(), r.cleanupTimeout, + ) + defer cancel() + + _, _ = r.chainSource.Ask( + cleanupCtx, &chainsource.UnregisterSpendRequest{ + CallerID: callerID, + Outpoint: outpoint, + }, + ).Await(cleanupCtx).Unpack() +} diff --git a/unroll/reconcile_chainsource_test.go b/unroll/reconcile_chainsource_test.go new file mode 100644 index 000000000..dc4aea720 --- /dev/null +++ b/unroll/reconcile_chainsource_test.go @@ -0,0 +1,317 @@ +package unroll + +import ( + "context" + "fmt" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/btcsuite/btcd/btcutil/v2" + "github.com/btcsuite/btcd/chainhash/v2" + "github.com/btcsuite/btcd/wire/v2" + "github.com/lightninglabs/wavelength/baselib/actor" + "github.com/lightninglabs/wavelength/chainsource" + fn "github.com/lightningnetwork/lnd/fn/v2" + "github.com/stretchr/testify/require" +) + +// reconcileMockBackend is a minimal ChainBackend test double tailored +// for the reconciler concurrency test. It records every RegisterConf +// call so the test can assert per-probe caller-ID uniqueness, and +// pre-arms a positive confirmation reply that fires immediately when +// the txid matches the configured target. +type reconcileMockBackend struct { + confirmTxid chainhash.Hash + confirmHeight int32 + + // registers records (callerID-like txid, pkScript) pairs as seen + // from successive RegisterConf calls. The caller-ID is not visible + // at this layer (chainsource consumes it before spawning the sub- + // actor), so the test instead asserts that the sub-actor IDs the + // system spawned are distinct via System.Has on the service-key. + mu sync.Mutex + registers []struct { + Txid chainhash.Hash + PkScript []byte + } + + // bestHeight is reported by BestBlock. + bestHeight int32 + + // epochCh / epochCancel back RegisterBlocks. + epochCh chan *chainsource.BlockEpoch + epochCancel atomic.Int32 +} + +func newReconcileMockBackend() *reconcileMockBackend { + return &reconcileMockBackend{ + bestHeight: 200, + epochCh: make(chan *chainsource.BlockEpoch, 1), + } +} + +func (m *reconcileMockBackend) EstimateFee(context.Context, uint32) ( + btcutil.Amount, error) { + + return 1000, nil +} + +func (m *reconcileMockBackend) BestBlock(context.Context) (int32, + chainhash.Hash, error) { + + return m.bestHeight, chainhash.Hash{}, nil +} + +func (m *reconcileMockBackend) TestMempoolAccept(context.Context, + ...*wire.MsgTx) ([]chainsource.MempoolAcceptResult, error) { + + return nil, nil +} + +func (m *reconcileMockBackend) BroadcastTx(context.Context, *wire.MsgTx, + string) error { + + return nil +} + +func (m *reconcileMockBackend) SubmitPackage(context.Context, []*wire.MsgTx, + *wire.MsgTx) error { + + return nil +} + +// RegisterConf records the registration and arms a one-shot positive +// confirmation when the requested txid matches the configured target. +// Every call returns a fresh ConfRegistration with its own channels so +// concurrent registrants do not contend on a shared buffered channel. +func (m *reconcileMockBackend) RegisterConf(_ context.Context, + txid *chainhash.Hash, pkScript []byte, _, _ uint32, _ bool) ( + *chainsource.ConfRegistration, error) { + + m.mu.Lock() + m.registers = append(m.registers, struct { + Txid chainhash.Hash + PkScript []byte + }{ + Txid: *txid, + PkScript: append([]byte(nil), pkScript...), + }) + m.mu.Unlock() + + confCh := make(chan *chainsource.TxConfirmation, 1) + reorged := make(chan uint64, 1) + done := make(chan struct{}, 1) + + if txid != nil && *txid == m.confirmTxid { + blockTx := wire.NewMsgTx(2) + blockHash := chainhash.Hash{0x42} + confCh <- &chainsource.TxConfirmation{ + BlockHash: &blockHash, + BlockHeight: uint32(m.confirmHeight), + Tx: blockTx, + } + } + + return &chainsource.ConfRegistration{ + Confirmed: confCh, + Reorged: reorged, + Done: done, + Cancel: func() {}, + }, nil +} + +func (m *reconcileMockBackend) RegisterSpend(context.Context, *wire.OutPoint, + []byte, uint32) (*chainsource.SpendRegistration, error) { + + return &chainsource.SpendRegistration{ + Spend: make(chan *chainsource.SpendDetail, 1), + Reorged: make(chan uint64, 1), + Done: make(chan struct{}, 1), + Cancel: func() {}, + }, nil +} + +func (m *reconcileMockBackend) RegisterBlocks(context.Context) ( + *chainsource.BlockRegistration, error) { + + return &chainsource.BlockRegistration{ + Epochs: m.epochCh, + Cancel: func() { + m.epochCancel.Add(1) + }, + }, nil +} + +func (m *reconcileMockBackend) Start() error { return nil } +func (m *reconcileMockBackend) Stop() error { return nil } + +// TestChainSourceReconcilerConcurrentProbesDoNotCollide exercises the +// invariant that two reconcilers probing the same shared proof-graph +// txid against a single chainsource actor do NOT collide on chainsource +// service keys when each reconciler is built with a target-specific +// caller-ID prefix. +// +// Chainsource keys its per-probe sub-actors on +// (CallerID, txid/pkScript, TargetConfs) (see handleRegisterConf in +// chainsource/chainsource.go). Two reconcilers built with the same +// static prefix probing the same txid would map to identical service +// keys; the second Spawn would collide with the first sub-actor's +// registration and drop or merge the second probe. Production wiring +// in waved/server.go bakes the per-actor target outpoint into the +// caller-ID prefix exactly to avoid this; this test pins that +// invariant for the chainsource-backed reconciler. +func TestChainSourceReconcilerConcurrentProbesDoNotCollide(t *testing.T) { + t.Parallel() + + proof := buildLinearProof(t) + roots := proof.RootTxids() + require.Len(t, roots, 1, "linear proof should have one root") + rootTxid := roots[0] + + backend := newReconcileMockBackend() + backend.confirmTxid = rootTxid + backend.confirmHeight = 150 + + system := actor.NewActorSystem() + defer func() { + _ = system.Shutdown(t.Context()) + }() + + chainSource := chainsource.NewChainSourceActor( + chainsource.ChainSourceConfig{ + Backend: backend, + System: system, + }, + ) + chainRef := chainsource.ChainSourceKey.Spawn( + system, "chainsource-reconcile", chainSource, + ) + + target1 := proof.TargetOutpoint() + target2 := wire.OutPoint{ + Hash: chainhash.Hash{ + 0xaa, + }, + Index: 7, + } + + mkReconciler := func(target wire.OutPoint) ChainReconciler { + return NewChainSourceReconciler(ChainSourceReconcilerConfig{ + ChainSource: chainRef, + Proof: proof, + CallerID: fmt.Sprintf( + "unroll-reconcile-%s", target, + ), + ProbeTimeout: 5 * time.Second, + }) + } + + rec1 := mkReconciler(target1) + rec2 := mkReconciler(target2) + + type probeResult struct { + anchor fn.Option[ConfirmedAnchor] + err error + } + + results := make(chan probeResult, 2) + probe := func(r ChainReconciler) { + ctx, cancel := context.WithTimeout( + t.Context(), 10*time.Second, + ) + defer cancel() + + anchor, err := r.ConfirmedTx(ctx, rootTxid) + results <- probeResult{anchor: anchor, err: err} + } + + go probe(rec1) + go probe(rec2) + + for range 2 { + select { + case res := <-results: + require.NoError( + t, res.err, + "concurrent reconciler probe failed", + ) + require.True( + t, res.anchor.IsSome(), + "reconciler did not see confirmation; a "+ + "static caller-ID collision would "+ + "silently swallow the second probe", + ) + require.Equal( + t, backend.confirmHeight, + res.anchor.UnsafeFromSome().Height, + ) + + case <-time.After(15 * time.Second): + t.Fatal( + "timed out waiting for concurrent " + + "reconciler probes; static " + + "caller-ID would cause one probe " + + "to wait forever", + ) + } + } + + // Both reconcilers must have hit the backend with a RegisterConf + // for the shared txid — confirming no internal short-circuit. + backend.mu.Lock() + defer backend.mu.Unlock() + require.Len( + t, backend.registers, 2, + "expected both reconciler probes to reach the backend", + ) + for _, r := range backend.registers { + require.Equal(t, rootTxid, r.Txid) + } +} + +// TestChainSourceReconcilerTimeoutFailsClosed proves notifier silence is +// reported as unavailable evidence for both confirmation and spend probes. It +// must never be converted into an objective absent or unspent answer. +func TestChainSourceReconcilerTimeoutFailsClosed(t *testing.T) { + t.Parallel() + + proof := buildLinearProof(t) + backend := newReconcileMockBackend() + system := actor.NewActorSystem() + defer func() { + _ = system.Shutdown(t.Context()) + }() + + chainRef := chainsource.ChainSourceKey.Spawn( + system, "chainsource-reconcile-timeout", + chainsource.NewChainSourceActor( + chainsource.ChainSourceConfig{ + Backend: backend, + System: system, + }, + ), + ) + reconciler := NewChainSourceReconciler( + ChainSourceReconcilerConfig{ + ChainSource: chainRef, + Proof: proof, + CallerID: "unroll-reconcile-timeout", + ProbeTimeout: 25 * time.Millisecond, + CleanupTimeout: time.Second, + }, + ) + + rootTxids := proof.RootTxids() + require.NotEmpty(t, rootTxids) + confirmed, err := reconciler.ConfirmedTx(t.Context(), rootTxids[0]) + require.ErrorIs(t, err, ErrReconcileEvidenceUnavailable) + require.True(t, confirmed.IsNone()) + + spent, err := reconciler.SpentOutpoint( + t.Context(), proof.TargetOutpoint(), + ) + require.ErrorIs(t, err, ErrReconcileEvidenceUnavailable) + require.True(t, spent.IsNone()) +} diff --git a/unroll/reconcile_test.go b/unroll/reconcile_test.go new file mode 100644 index 000000000..4cd3b44e3 --- /dev/null +++ b/unroll/reconcile_test.go @@ -0,0 +1,614 @@ +package unroll + +import ( + "context" + "errors" + "testing" + "time" + + "github.com/btcsuite/btcd/chainhash/v2" + "github.com/btcsuite/btcd/wire/v2" + "github.com/btcsuite/btclog/v2" + "github.com/lightninglabs/wavelength/baselib/actor" + "github.com/lightninglabs/wavelength/lib/recovery" + "github.com/lightninglabs/wavelength/unrollplan" + fn "github.com/lightningnetwork/lnd/fn/v2" + "github.com/stretchr/testify/require" +) + +// stubChainReconciler is a fully controllable ChainReconciler test +// double. The two maps drive the ConfirmedTx / SpentOutpoint answers +// and a non-nil err field lets a test inject a transport failure on +// either query. +type stubChainReconciler struct { + confirmed map[chainhash.Hash]ConfirmedAnchor + spent map[wire.OutPoint]SpendAnchor + err error +} + +// positiveOnlyReconciler models the current chainsource subscription surface: +// it can replay historical positives, but silence is not an authoritative +// unspent answer. +type positiveOnlyReconciler struct { + *stubChainReconciler + spendCalls int +} + +// HasAuthoritativeNegativeEvidence reports that notifier silence cannot prove +// absence. +func (p *positiveOnlyReconciler) HasAuthoritativeNegativeEvidence() bool { + return false +} + +// SpentOutpoint records any attempted unanchored negative probe. +func (p *positiveOnlyReconciler) SpentOutpoint(ctx context.Context, + outpoint wire.OutPoint) (fn.Option[SpendAnchor], error) { + + p.spendCalls++ + + return p.stubChainReconciler.SpentOutpoint(ctx, outpoint) +} + +// ConfirmedTx returns the configured anchor for txid, or fn.None. +func (s *stubChainReconciler) ConfirmedTx(_ context.Context, + txid chainhash.Hash) (fn.Option[ConfirmedAnchor], error) { + + if s.err != nil { + return fn.None[ConfirmedAnchor](), s.err + } + + anchor, ok := s.confirmed[txid] + if !ok { + return fn.None[ConfirmedAnchor](), nil + } + + return fn.Some(anchor), nil +} + +// SpentOutpoint returns the configured anchor for outpoint, or fn.None. +func (s *stubChainReconciler) SpentOutpoint(_ context.Context, + outpoint wire.OutPoint) (fn.Option[SpendAnchor], error) { + + if s.err != nil { + return fn.None[SpendAnchor](), s.err + } + + anchor, ok := s.spent[outpoint] + if !ok { + return fn.None[SpendAnchor](), nil + } + + return fn.Some(anchor), nil +} + +// TestPositiveOnlyReconcilerSkipsUnanchoredSpendProbe verifies that an ordinary +// restart does not wait for notifier silence to prove the target unspent. With +// no persisted spend anchor there is no positive fact to reconcile; the actor +// re-arms its live spend watch and the durable relive guard protects the +// offline uncertainty window. +func TestPositiveOnlyReconcilerSkipsUnanchoredSpendProbe(t *testing.T) { + proof := buildLinearProof(t) + rootTxid := proof.RootTxids()[0] + reconciler := &positiveOnlyReconciler{ + stubChainReconciler: &stubChainReconciler{ + confirmed: map[chainhash.Hash]ConfirmedAnchor{ + rootTxid: { + Txid: rootTxid, + Height: 101, + }, + }, + }, + } + checkpoint := &actorCheckpoint{ + Version: checkpointVersion, + Height: 101, + Started: true, + State: unrollplan.State{ + ConfirmedTxids: []chainhash.Hash{ + rootTxid, + }, + }, + } + + err := reconcileCheckpoint( + t.Context(), reconciler, proof, checkpoint, + ) + require.NoError(t, err) + require.Zero(t, reconciler.spendCalls) + require.Equal( + t, []chainhash.Hash{rootTxid}, checkpoint.State.ConfirmedTxids, + ) +} + +// restoreHarness boots a fresh unroll behavior from a fabricated +// checkpoint so the reconcile tests can inspect the post-reconciliation +// FSM state without running the rest of the actor's IO surface. +func restoreHarness(t *testing.T, proof *recovery.Proof, + checkpoint *actorCheckpoint, reconciler ChainReconciler) (*behavior, + *fakeTxConfirmRef, *fakeChainSourceRef) { + + t.Helper() + + desc := testDescriptor(t, proof.TargetOutpoint(), proof.CSVDelay()) + store := newMemCheckpointStore() + txconfirmRef := &fakeTxConfirmRef{} + chainRef := &fakeChainSourceRef{} + + raw, err := encodeCheckpoint(checkpoint) + require.NoError(t, err) + + err = store.SaveCheckpoint(t.Context(), actor.CheckpointParams{ + ActorID: "reconcile-test", + StateType: checkpointStateType, + StateData: raw, + Version: checkpointVersion, + }) + require.NoError(t, err) + + cfg := Config{ + TargetOutpoint: proof.TargetOutpoint(), + ActorID: "reconcile-test", + DeliveryStore: store, + ProofAssembler: &mockProofAssembler{ + proof: proof, + }, + VTXOStore: &mockVTXOStore{ + desc: desc, + }, + TxConfirmRef: txconfirmRef, + ChainSource: chainRef, + Wallet: &fakeSweepWallet{}, + Log: fn.Some(btclog.Disabled), + } + if reconciler != nil { + // Wrap the stub in a factory that ignores the proof, so + // the production code path (which always calls the + // factory) sees the test stub unchanged. + stub := reconciler + cfg.ChainReconcilerFactory = fn.Some( + ChainReconcilerFactory( + func(wire.OutPoint, + *recovery.Proof) ChainReconciler { + + return stub + }, + ), + ) + } + + beh := &behavior{ + cfg: cfg, + log: btclog.Disabled, + } + require.NoError(t, beh.restoreCheckpoint(t.Context())) + + return beh, txconfirmRef, chainRef +} + +// TestReconcileTargetReorgedOfflineResumesMaterialization restores a +// checkpoint whose target tx is recorded as confirmed but is no longer +// on the canonical chain. After reconciliation the actor must restart +// without the stale TargetConfirmHeight, in PhaseMaterializing, so it +// does not broadcast a sweep on a target that has been reorged out. +func TestReconcileTargetReorgedOfflineResumesMaterialization(t *testing.T) { + proof := buildLinearProof(t) + rootTxid := proof.RootTxids()[0] + targetTxid := proof.TargetOutpoint().Hash + + checkpoint := &actorCheckpoint{ + Version: checkpointVersion, + Height: 103, + Started: true, + Trigger: TriggerRestart, + State: unrollplan.State{ + ConfirmedTxids: []chainhash.Hash{ + rootTxid, targetTxid, + }, + TargetConfirmHeight: fn.Some[int32](102), + }, + } + + // Reconciler reports the root is still on chain at H=101 but the + // target tx is absent — its confirming block was reorged out + // while the daemon was offline. + reconciler := &stubChainReconciler{ + confirmed: map[chainhash.Hash]ConfirmedAnchor{ + rootTxid: { + Txid: rootTxid, + Height: 101, + }, + }, + } + + beh, _, _ := restoreHarness(t, proof, checkpoint, reconciler) + + actorInstance := actor.NewActor(actor.ActorConfig[Msg, Resp]{ + ID: "reconcile-test", + Behavior: &txExecAdapter{b: beh, ax: newMemExecFor(beh)}, + MailboxSize: 16, + }) + beh.selfRef = actorInstance.TellRef() + actorInstance.Start() + t.Cleanup(actorInstance.Stop) + + mustAsk(t, actorInstance.Ref(), &ResumeUnrollRequest{Height: 103}) + + require.Eventually(t, func() bool { + resp, ok := mustAsk( + t, actorInstance.Ref(), &GetStateRequest{}, + ).(*GetStateResp) + require.True(t, ok) + + // Target anchor must be gone, planner must be back in + // materialization, and no FailReason should have leaked + // through. + for _, h := range resp.PlannerState.ConfirmedTxids { + if h == targetTxid { + return false + } + } + + return resp.Phase == PhaseMaterializing && + resp.PlannerState.TargetConfirmHeight.IsNone() && + resp.FailReason == "" + }, testTimeout, 10*time.Millisecond, + "reconciliation never cleared the stale target anchor") +} + +// TestReconcileSweepReorgedOfflineDowngradesSweep restores a +// checkpoint with a Confirmed sweep that is no longer on chain. After +// reconciliation the sweep state must reset so the next planner pass +// re-emits NeedSweep and the cached sweep tx is re-broadcast. +func TestReconcileSweepReorgedOfflineDowngradesSweep(t *testing.T) { + proof := buildLinearProof(t) + rootTxid := proof.RootTxids()[0] + targetTxid := proof.TargetOutpoint().Hash + sweepTxid := chainhash.Hash{0xaa} + + checkpoint := &actorCheckpoint{ + Version: checkpointVersion, + Height: 110, + Started: true, + Trigger: TriggerRestart, + State: unrollplan.State{ + ConfirmedTxids: []chainhash.Hash{ + rootTxid, targetTxid, + }, + TargetConfirmHeight: fn.Some[int32](102), + Sweep: unrollplan.SweepState{ + Status: unrollplan.SweepStatusConfirmed, + Txid: fn.Some(sweepTxid), + ConfirmHeight: fn.Some[int32](108), + }, + }, + } + + // Reconciler: every proof anchor still confirmed, but the sweep + // is no longer on chain. + reconciler := &stubChainReconciler{ + confirmed: map[chainhash.Hash]ConfirmedAnchor{ + rootTxid: { + Txid: rootTxid, + Height: 101, + }, + targetTxid: { + Txid: targetTxid, + Height: 102, + }, + }, + } + + beh, _, _ := restoreHarness(t, proof, checkpoint, reconciler) + + actorInstance := actor.NewActor(actor.ActorConfig[Msg, Resp]{ + ID: "reconcile-test", + Behavior: &txExecAdapter{b: beh, ax: newMemExecFor(beh)}, + MailboxSize: 16, + }) + beh.selfRef = actorInstance.TellRef() + actorInstance.Start() + t.Cleanup(actorInstance.Stop) + + mustAsk(t, actorInstance.Ref(), &ResumeUnrollRequest{Height: 110}) + + require.Eventually(t, func() bool { + resp, ok := mustAsk( + t, actorInstance.Ref(), &GetStateRequest{}, + ).(*GetStateResp) + require.True(t, ok) + + // The target anchor must survive a sweep-only reorg — + // CSV maturity is still valid even though the sweep + // itself needs to be re-broadcast. The planner should + // move the actor back into AwaitingSweepBroadcast (or + // have already requested a fresh sweep build). + sweep := resp.PlannerState.Sweep + + return resp.PlannerState.TargetConfirmHeight.IsSome() && + (sweep.Status == unrollplan.SweepStatusPending || + sweep.Status == + unrollplan.SweepStatusBroadcasted) && + sweep.ConfirmHeight.IsNone() + }, testTimeout, 10*time.Millisecond, + "reconciliation never downgraded the stale sweep anchor") +} + +// TestReconcileRefreshesTargetHeightAfterOfflineReMine restores a +// checkpoint that recorded the target tx confirmed at one height, but +// the reconciler reports the target as still confirmed at a DIFFERENT +// (higher) height — i.e. the target tx was re-mined into a different +// block while the daemon was offline. The reconciler must refresh +// TargetConfirmHeight to the live value so CSV maturity is computed +// off the correct on-chain anchor, not the stale checkpoint height. +func TestReconcileRefreshesTargetHeightAfterOfflineReMine(t *testing.T) { + proof := buildLinearProof(t) + rootTxid := proof.RootTxids()[0] + targetTxid := proof.TargetOutpoint().Hash + + const ( + staleHeight = int32(102) + liveHeight = int32(105) + ) + + checkpoint := &actorCheckpoint{ + Version: checkpointVersion, + Height: staleHeight, + Started: true, + Trigger: TriggerRestart, + State: unrollplan.State{ + ConfirmedTxids: []chainhash.Hash{ + rootTxid, targetTxid, + }, + TargetConfirmHeight: fn.Some(staleHeight), + }, + } + + reconciler := &stubChainReconciler{ + confirmed: map[chainhash.Hash]ConfirmedAnchor{ + rootTxid: { + Txid: rootTxid, + Height: 101, + }, + targetTxid: { + Txid: targetTxid, + Height: liveHeight, + }, + }, + } + + err := reconcileCheckpoint(t.Context(), reconciler, proof, checkpoint) + require.NoError(t, err) + + require.True(t, checkpoint.State.TargetConfirmHeight.IsSome()) + require.Equal( + t, liveHeight, + checkpoint.State.TargetConfirmHeight.UnsafeFromSome(), + "TargetConfirmHeight must refresh to the live anchor height", + ) + require.GreaterOrEqual( + t, checkpoint.Height, liveHeight, + "checkpoint best-height should not lag the live anchor", + ) +} + +// TestReconcileTransportErrorSurfacesUpward asserts that a transport +// failure from the reconciler propagates out of ensureLoaded rather +// than silently letting the actor proceed off an unverified +// checkpoint. +func TestReconcileTransportErrorSurfacesUpward(t *testing.T) { + proof := buildLinearProof(t) + rootTxid := proof.RootTxids()[0] + + checkpoint := &actorCheckpoint{ + Version: checkpointVersion, + Height: 101, + Started: true, + Trigger: TriggerRestart, + State: unrollplan.State{ + ConfirmedTxids: []chainhash.Hash{ + rootTxid, + }, + }, + } + + reconciler := &stubChainReconciler{ + err: errors.New("backend unreachable"), + } + + beh, _, _ := restoreHarness(t, proof, checkpoint, reconciler) + + err := beh.ensureLoaded(t.Context()) + require.Error(t, err) + require.Contains(t, err.Error(), "backend unreachable") +} + +// TestReconcileExternalSpendStillLiveRestoresParkedActor restores a +// checkpoint that already carries a ProvisionalExternalSpend anchor +// and verifies that the reconciler keeps it set when the chain still +// reports the target as spent — the actor must enter +// AwaitingExternalSpendFinality on the next Resume rather than +// resuming planner-driven materialization. +func TestReconcileExternalSpendStillLiveRestoresParkedActor(t *testing.T) { + proof := buildLinearProof(t) + rootTxid := proof.RootTxids()[0] + externalTxid := chainhash.Hash{0xee} + + checkpoint := &actorCheckpoint{ + Version: checkpointVersion, + Height: 155, + Started: true, + Trigger: TriggerRestart, + State: unrollplan.State{ + ConfirmedTxids: []chainhash.Hash{ + rootTxid, + }, + }, + ProvisionalExternalSpend: fn.Some(ExternalSpendAnchor{ + SpendingTxid: externalTxid, + SpendingHeight: 155, + }), + } + + reconciler := &stubChainReconciler{ + confirmed: map[chainhash.Hash]ConfirmedAnchor{ + rootTxid: { + Txid: rootTxid, + Height: 101, + }, + }, + spent: map[wire.OutPoint]SpendAnchor{ + proof.TargetOutpoint(): { + Outpoint: proof.TargetOutpoint(), + SpendingTxid: externalTxid, + SpendingHeight: 155, + }, + }, + } + + beh, _, _ := restoreHarness(t, proof, checkpoint, reconciler) + + actorInstance := actor.NewActor(actor.ActorConfig[Msg, Resp]{ + ID: "reconcile-test", + Behavior: &txExecAdapter{b: beh, ax: newMemExecFor(beh)}, + MailboxSize: 16, + }) + beh.selfRef = actorInstance.TellRef() + actorInstance.Start() + t.Cleanup(actorInstance.Stop) + + mustAsk(t, actorInstance.Ref(), &ResumeUnrollRequest{Height: 155}) + + require.Eventually(t, func() bool { + resp, ok := mustAsk( + t, actorInstance.Ref(), &GetStateRequest{}, + ).(*GetStateResp) + require.True(t, ok) + + return resp.Phase == PhaseExternalSpendObserved && + resp.FailReason == "" + }, testTimeout, 10*time.Millisecond, + "actor did not restore into PhaseExternalSpendObserved") +} + +// TestReconcileExternalSpendReorgedOfflineResumesActor restores a +// checkpoint carrying a ProvisionalExternalSpend anchor but the +// reconciler reports the target outpoint as currently unspent: the +// spending block was reorged out while the daemon was offline. +// Reconciliation must clear the anchor so the actor resumes +// materialization instead of staying parked. +func TestReconcileExternalSpendReorgedOfflineResumesActor(t *testing.T) { + proof := buildLinearProof(t) + rootTxid := proof.RootTxids()[0] + externalTxid := chainhash.Hash{0xee} + + checkpoint := &actorCheckpoint{ + Version: checkpointVersion, + Height: 155, + Started: true, + Trigger: TriggerRestart, + State: unrollplan.State{ + InFlightTxids: []chainhash.Hash{ + rootTxid, + }, + }, + ProvisionalExternalSpend: fn.Some(ExternalSpendAnchor{ + SpendingTxid: externalTxid, + SpendingHeight: 155, + }), + } + + // Reconciler reports the target outpoint as unspent. The + // proof root is still in-flight; no confirmed anchors to verify + // here. + reconciler := &stubChainReconciler{} + + beh, _, _ := restoreHarness(t, proof, checkpoint, reconciler) + + actorInstance := actor.NewActor(actor.ActorConfig[Msg, Resp]{ + ID: "reconcile-test", + Behavior: &txExecAdapter{b: beh, ax: newMemExecFor(beh)}, + MailboxSize: 16, + }) + beh.selfRef = actorInstance.TellRef() + actorInstance.Start() + t.Cleanup(actorInstance.Stop) + + mustAsk(t, actorInstance.Ref(), &ResumeUnrollRequest{Height: 155}) + + require.Eventually(t, func() bool { + resp, ok := mustAsk( + t, actorInstance.Ref(), &GetStateRequest{}, + ).(*GetStateResp) + require.True(t, ok) + + return resp.Phase == PhaseMaterializing && + resp.FailReason == "" + }, testTimeout, 10*time.Millisecond, + "actor did not resume materialization after offline "+ + "external-spend reorg") +} + +// TestReconcileExternalSpendObservedOfflineParksActor restores a +// checkpoint that does NOT carry a ProvisionalExternalSpend anchor, +// but the reconciler reports the target outpoint as currently spent +// by a txid that is neither in the proof graph nor our sweep. The +// reconciler must install the anchor so the actor enters +// AwaitingExternalSpendFinality on the next Resume. +func TestReconcileExternalSpendObservedOfflineParksActor(t *testing.T) { + proof := buildLinearProof(t) + rootTxid := proof.RootTxids()[0] + externalTxid := chainhash.Hash{0xee} + + checkpoint := &actorCheckpoint{ + Version: checkpointVersion, + Height: 155, + Started: true, + Trigger: TriggerRestart, + State: unrollplan.State{ + ConfirmedTxids: []chainhash.Hash{ + rootTxid, + }, + }, + } + + reconciler := &stubChainReconciler{ + confirmed: map[chainhash.Hash]ConfirmedAnchor{ + rootTxid: { + Txid: rootTxid, + Height: 101, + }, + }, + spent: map[wire.OutPoint]SpendAnchor{ + proof.TargetOutpoint(): { + Outpoint: proof.TargetOutpoint(), + SpendingTxid: externalTxid, + SpendingHeight: 155, + }, + }, + } + + beh, _, _ := restoreHarness(t, proof, checkpoint, reconciler) + + actorInstance := actor.NewActor(actor.ActorConfig[Msg, Resp]{ + ID: "reconcile-test", + Behavior: &txExecAdapter{b: beh, ax: newMemExecFor(beh)}, + MailboxSize: 16, + }) + beh.selfRef = actorInstance.TellRef() + actorInstance.Start() + t.Cleanup(actorInstance.Stop) + + mustAsk(t, actorInstance.Ref(), &ResumeUnrollRequest{Height: 155}) + + require.Eventually(t, func() bool { + resp, ok := mustAsk( + t, actorInstance.Ref(), &GetStateRequest{}, + ).(*GetStateResp) + require.True(t, ok) + + return resp.Phase == PhaseExternalSpendObserved && + resp.FailReason == "" + }, testTimeout, 10*time.Millisecond, + "actor did not park in PhaseExternalSpendObserved after "+ + "offline external spend") +} diff --git a/unroll/registry.go b/unroll/registry.go index be41ec765..13361beba 100644 --- a/unroll/registry.go +++ b/unroll/registry.go @@ -103,8 +103,8 @@ type RegistryStore interface { ListNonTerminalRecords(ctx context.Context) ([]RegistryRecord, error) // MarkTerminal persists one terminal target state. recoverable marks a - // no-footprint failure that boot-time reconciliation may roll back to - // live. + // no-footprint failure with objective canonical-absence evidence that + // boot-time reconciliation may roll back to live. MarkTerminal(ctx context.Context, target wire.OutPoint, phase Phase, recoverable bool, failReason string, sweepTxid *chainhash.Hash) error @@ -166,14 +166,25 @@ type RegistryConfig struct { FraudCheckpointSafetyMargin int32 // VTXOExitObserver, when set, receives an ExitOutcomeNotification each - // time a child unroll job reaches a terminal phase: a clean failure - // (no on-chain footprint) asks the VTXO manager to roll the VTXO back - // to live, and a completed exit asks it to retire the VTXO to spent. - // This is the feedback edge that keeps VTXO lifecycle gated on the - // unroll job's terminal on-chain outcome rather than the user's intent - // to exit (wavelength#602). When None, terminal outcomes are not - // forwarded (used by tests that don't exercise the manager). + // time a child unroll job reaches a terminal phase: a failure with no + // footprint and objective canonical-absence evidence asks the VTXO + // manager to roll the VTXO back to live, and a completed exit asks it + // to retire the VTXO to spent. This is the feedback edge that keeps + // VTXO lifecycle gated on the unroll job's terminal on-chain outcome + // rather than the user's intent to exit (wavelength#602). When None, + // terminal outcomes are not forwarded (used by tests that don't + // exercise the manager). VTXOExitObserver fn.Option[actor.TellOnlyRef[vtxo.ManagerMsg]] + + // ChainReconcilerFactory, when set, is forwarded to each spawned + // per-target actor so it can verify its checkpoint anchors + // against the canonical chain on restart. The factory is invoked + // once per actor lifetime after the proof loads. When None + // children skip reconciliation; production wiring should pass a + // factory backed by NewChainSourceReconciler so an offline + // reorg window does not silently leave actors driving side + // effects off stale planner state. + ChainReconcilerFactory fn.Option[ChainReconcilerFactory] } // UnrollRegistryActor wraps the thin unroll registry actor. @@ -236,6 +247,9 @@ func NewUnrollRegistryActor(cfg RegistryConfig) *UnrollRegistryActor { active: make(map[wire.OutPoint]*VTXOUnrollActor), pending: make(map[wire.OutPoint]RegistryRecord), persisting: make(map[wire.OutPoint]RegistryRecord), + terminalPolicyKinds: make( + map[wire.OutPoint]ExitPolicyKind, + ), } registry := actor.NewActor(actor.ActorConfig[RegistryMsg, RegistryResp]{ @@ -264,6 +278,11 @@ type registryBehavior struct { pending map[wire.OutPoint]RegistryRecord persisting map[wire.OutPoint]RegistryRecord + // terminalPolicyKinds retains the message-authoritative policy kind + // while a failed terminal store write is being retried. It must not be + // stamped onto a record without its matching durable ref. + terminalPolicyKinds map[wire.OutPoint]ExitPolicyKind + spawnFunc func(context.Context, wire.OutPoint) (*VTXOUnrollActor, error) } @@ -464,9 +483,9 @@ func (r *registryBehavior) handleEnsure(ctx context.Context, // the durable store. Re-spawning a fresh actor on top of an existing // record would clobber the recorded sweep txid or fail reason. // - // A recoverable failure is the exception: the prior exit failed - // cleanly with no on-chain footprint and the VTXO was rolled back to - // live (wavelength#602), so a fresh exit must be admittable rather + // A recoverable failure is the exception: objective canonical-absence + // evidence allowed the VTXO to roll back to live (wavelength#602), so a + // fresh exit must be admittable rather // than deduped against the dead attempt. Fall through to the spawn // path, whose UpsertRecord overwrites the stale recoverable row. if record, ok := r.pending[req.Outpoint]; ok && @@ -544,14 +563,14 @@ func (r *registryBehavior) handleEnsure(ctx context.Context, }) } - // Terminal record. A recoverable failure (clean, no on-chain - // footprint) means the VTXO was rolled back to live, so a new - // exit is allowed: fall through to the spawn path below, whose - // UpsertRecord overwrites the stale record. Any other terminal - // record (Completed, or a footprint-bearing Failed) is a real - // end state — dedup against it and return the historical - // ActorID so the recorded sweep txid / failure reason are - // never clobbered. + // Terminal record. A recoverable failure backed by objective + // canonical-absence evidence means the VTXO was rolled back to + // live, so a new exit is allowed: fall through to the spawn + // path below, whose UpsertRecord overwrites the stale record. + // Any other terminal record (Completed, or a footprint-bearing + // Failed) is a real end state — dedup against it and return the + // historical ActorID so the recorded sweep txid / failure + // reason are never clobbered. if !existing.RecoverableFailure { return fn.Ok[RegistryResp](&EnsureUnrollResp{ ActorID: existing.ActorID, @@ -625,11 +644,11 @@ func (r *registryBehavior) handleEnsure(ctx context.Context, // notifyRegistryIfTerminal so the registry record // cannot stay PhasePending forever. // - // 2. Real start error — proof assembly, store, planner. - // Hiding it under a Created=true would silently strand - // the user's funds in unilateral_exit with no - // progress. Mark the durable row PhaseFailed so - // GetUnrollStatus surfaces a terminal status. + // 2. Real start error — proof assembly, store, planner, or + // outbox failure after submission. The registry cannot + // distinguish those positions, so it preserves the durable + // hold and surfaces the error. A fresh Ensure or restart + // retries restoration without reliving value. if isCancellationRace(err) { r.watchChildAdmissionResult( context.WithoutCancel(ctx), req.Outpoint, child, @@ -648,7 +667,7 @@ func (r *registryBehavior) handleEnsure(ctx context.Context, }) } - r.failAdmittedChild( + r.parkAdmittedChild( ctx, req.Outpoint, child, fmt.Errorf("start child: %w", err), ) @@ -659,7 +678,7 @@ func (r *registryBehavior) handleEnsure(ctx context.Context, state, err := r.childState(startCtx, child) if err != nil { if !isCancellationRace(err) { - r.failAdmittedChild( + r.parkAdmittedChild( ctx, req.Outpoint, child, fmt.Errorf("read child state: %w", err), ) @@ -741,9 +760,8 @@ func (r *registryBehavior) watchChildAdmissionResult(ctx context.Context, }() } -// handleChildAdmissionResult converts a late child start failure into the -// same terminal registry state that a synchronous start failure would have -// produced. +// handleChildAdmissionResult parks a late child start failure in the same +// fail-closed non-terminal state as a synchronous start failure. func (r *registryBehavior) handleChildAdmissionResult(ctx context.Context, req *childAdmissionResultMsg) fn.Result[RegistryResp] { @@ -752,7 +770,7 @@ func (r *registryBehavior) handleChildAdmissionResult(ctx context.Context, return fn.Ok[RegistryResp](&RegistryAckResp{}) } - r.failAdmittedChild( + r.parkAdmittedChild( ctx, req.Outpoint, child, fmt.Errorf("start child: %s", req.Err), ) @@ -760,63 +778,60 @@ func (r *registryBehavior) handleChildAdmissionResult(ctx context.Context, return fn.Ok[RegistryResp](&RegistryAckResp{}) } -// failAdmittedChild records a terminal failure for a child that already has -// a durable pending row. This keeps GetUnrollStatus from falling back to -// "not found" after VTXO ownership has moved to unilateral exit. -// -// These failures (start / proof-assembly / restore errors) all occur before -// the child broadcasts anything, so the VTXO has no on-chain footprint and is -// recoverable to live. The record is persisted as a recoverable failure and -// the VTXO manager is notified so the VTXO is rolled back rather than -// stranded in unilateral exit (wavelength#602). -func (r *registryBehavior) failAdmittedChild(ctx context.Context, +// parkAdmittedChild stops a child whose admission result is ambiguous while +// preserving its durable non-terminal hold. A child stages state before +// routing outbox work, and an external txconfirm submission can succeed before +// a later actor/store error reaches the registry. Therefore a start or status +// error is not proof of a clean no-footprint failure and must never relive the +// VTXO. A fresh Ensure or restart restores the pending record and retries. +func (r *registryBehavior) parkAdmittedChild(ctx context.Context, target wire.OutPoint, child *VTXOUnrollActor, err error) { child.Stop() delete(r.active, target) record := RegistryRecord{ - TargetOutpoint: target, - ActorID: child.Ref().ID(), - Phase: PhaseFailed, - FailReason: err.Error(), - RecoverableFailure: true, + TargetOutpoint: target, + ActorID: child.Ref().ID(), + Phase: PhasePending, + FailReason: err.Error(), } if pending, ok := r.pending[target]; ok { record = cloneRegistryRecord(pending) - record.Phase = PhaseFailed + if record.IsTerminal() { + + // A terminal child handoff won the mailbox race. + // Preserve its objective result; the late admission + // error carries no higher priority than that durable + // terminal state. + return + } record.FailReason = err.Error() - record.RecoverableFailure = true + record.RecoverableFailure = false } r.pending[target] = cloneRegistryRecord(record) - // Roll the VTXO back to live. The terminal record below is the durable - // backstop if this best-effort notification is lost. A recovery-only - // target is held in exit instead (see notifyVTXOExit / the manager), - // so its policy rides along. - r.notifyVTXOExit(context.WithoutCancel(ctx), &UnrollTerminatedMsg{ - Outpoint: target, - ActorID: record.ActorID, - Phase: PhaseFailed, - FailReason: err.Error(), - HadOnChainFootprint: false, - }, record.ExitPolicyKind) - - markErr := r.cfg.Store.MarkTerminal( - context.WithoutCancel(ctx), target, PhaseFailed, true, - err.Error(), nil, + markErr := r.cfg.Store.UpsertRecord( + context.WithoutCancel(ctx), cloneRegistryRecord(record), ) if markErr != nil { - r.log.WarnS(ctx, "Failed to mark admitted unroll child "+ - "terminal", markErr, + r.log.WarnS(ctx, "Failed to preserve parked unroll child", + markErr, slog.String("outpoint", target.String()), slog.String("actor_id", child.Ref().ID()), ) // Registry persistence retries are actor-owned follow-up work. //nolint:contextcheck r.requestPersist(target, 0) + + return } + + // The durable non-terminal row now owns retry. Dropping the cache lets + // a fresh Ensure reach the store-backed inline-restore branch instead + // of deduping forever against a child that was deliberately stopped. + delete(r.pending, target) } // isCancellationRace reports whether admission should preserve the pending @@ -836,8 +851,8 @@ func (r *registryBehavior) failAdmittedChild(ctx context.Context, // baselib/actor's durable Ask short-circuits and returns that error // BEFORE calling mailbox.Send when the actor's own ctx is already done, // so there is no persisted message to trust. Letting it fall through to -// failAdmittedChild surfaces a deterministic terminal record instead of -// pretending the job is in flight. +// parkAdmittedChild preserves the fail-closed durable hold instead of treating +// the process error as objective no-footprint evidence. func isCancellationRace(err error) bool { return errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) @@ -954,17 +969,19 @@ func (r *registryBehavior) handleGetStatus(ctx context.Context, // stopped only after a queued state probe has drained through it. That // keeps the registry from synchronously cancelling the child while the // child is still acking the terminal durable message that notified us. -// The snapshot is then enqueued for async persistence via requestPersist. -// Terminal writes intentionally stay on the async retry path — unlike -// admission, a failed terminal write does not orphan the job (the -// in-memory pending record keeps answering GetStatus), so there is no -// reason to block the registry goroutine on a flaky store. +// The terminal row is written synchronously before any VTXO outcome is +// forwarded. If that write fails, the in-memory record enters the existing +// async retry path and the outcome stays withheld until a retry succeeds. This +// durability barrier is load-bearing for recoverable failures: business state +// must never become live before its terminal invalidation is durable. func (r *registryBehavior) handleTerminated(ctx context.Context, req *UnrollTerminatedMsg) fn.Result[RegistryResp] { - // A terminal failure with no on-chain footprint is recoverable: the - // VTXO never left off-chain custody, so it can be rolled back to live. - recoverable := req.Phase == PhaseFailed && !req.HadOnChainFootprint + // A terminal failure is recoverable only when this actor recorded no + // footprint and objective canonical-absence evidence cleared the + // chain-boundary uncertainty guard. + recoverable := req.Phase == PhaseFailed && + !req.HadOnChainFootprint && !req.ReliveUnsafe record := RegistryRecord{ TargetOutpoint: req.Outpoint, @@ -998,9 +1015,6 @@ func (r *registryBehavior) handleTerminated(ctx context.Context, } r.pending[req.Outpoint] = cloneRegistryRecord(record) - // Registry persistence retries are actor-owned follow-up work. - //nolint:contextcheck - r.requestPersist(req.Outpoint, 0) // The child's terminal message carries its durable exit policy, which // outlives r.pending: a completed async persist can evict the cached @@ -1013,6 +1027,38 @@ func (r *registryBehavior) handleTerminated(ctx context.Context, if policyKind == "" { policyKind = record.ExitPolicyKind } + if r.terminalPolicyKinds == nil { + r.terminalPolicyKinds = make( + map[wire.OutPoint]ExitPolicyKind, + ) + } + r.terminalPolicyKinds[req.Outpoint] = policyKind + + // Persist terminal invalidation before forwarding any outcome that can + // restore business state. A crash after reliving the VTXO but before + // the terminal row landed would let restart restore the old + // non-terminal job and reissue on-chain work against a live coin. + persistCtx := context.WithoutCancel(ctx) + err := r.cfg.Store.MarkTerminal( + persistCtx, req.Outpoint, req.Phase, recoverable, + req.FailReason, req.SweepTxid, + ) + if err != nil { + r.log.WarnS( + ctx, + "Failed to persist terminal unroll state; holding VTXO", + err, + slog.String("outpoint", req.Outpoint.String()), + slog.String("phase", string(req.Phase)), + ) + // Registry persistence retries are actor-owned follow-up work. + // The VTXO outcome stays withheld until + // handlePersistRecordResult sees a successful durable write. + //nolint:contextcheck + r.requestPersist(req.Outpoint, 0) + + return fn.Ok[RegistryResp](&RegistryAckResp{}) + } // Forward the terminal outcome to the VTXO manager so the VTXO's // lifecycle tracks the unroll job's terminal on-chain result rather @@ -1021,17 +1067,18 @@ func (r *registryBehavior) handleTerminated(ctx context.Context, // policy rides along so the manager can hold a recovery-only target in // exit rather than relive it as a live coin. r.notifyVTXOExit(context.WithoutCancel(ctx), req, policyKind) + delete(r.pending, req.Outpoint) + delete(r.terminalPolicyKinds, req.Outpoint) return fn.Ok[RegistryResp](&RegistryAckResp{}) } // notifyVTXOExit forwards a child's terminal outcome to the VTXO manager. // -// - PhaseFailed with no on-chain footprint: the unroll never broadcast, -// so the VTXO is still live from the operator's perspective. Ask the -// manager to roll it back to live (ExitOutcomeRecoverable). -// - PhaseCompleted: the exit was swept and confirmed on-chain, so ask the -// manager to retire the VTXO to spent (ExitOutcomeConfirmed). +// - PhaseFailed with no footprint and objective canonical-absence evidence: +// roll the VTXO back live (ExitOutcomeRecoverable). +// - PhaseCompleted: the target was durably consumed by the final sweep or a +// finalized external spend, so retire it (ExitOutcomeConfirmed). // - PhaseFailed with an on-chain footprint: the exit has begun on-chain; // leave the VTXO in unilateral-exit (no notification). // @@ -1054,7 +1101,9 @@ func (r *registryBehavior) notifyVTXOExit(ctx context.Context, case req.Phase == PhaseCompleted: outcome = vtxo.ExitOutcomeConfirmed - case req.Phase == PhaseFailed && !req.HadOnChainFootprint: + case req.Phase == PhaseFailed && !req.HadOnChainFootprint && + !req.ReliveUnsafe: + outcome = vtxo.ExitOutcomeRecoverable default: @@ -1248,13 +1297,43 @@ func (r *registryBehavior) tryRestoreOne(ctx context.Context, return nil, fmt.Errorf("spawn failed on restore: %w", err) } - _, err = child.Ref().Ask(ctx, &ResumeUnrollRequest{ - Height: height, - }).Await(ctx).Unpack() + stateRaw, err := child.Ref().Ask( + ctx, &GetStateRequest{}, + ).Await(ctx).Unpack() + if err != nil { + child.Stop() + + return nil, fmt.Errorf("read checkpoint state on restore: %w", + err) + } + state, ok := stateRaw.(*GetStateResp) + if !ok { + child.Stop() + + return nil, fmt.Errorf("read checkpoint state on restore: "+ + "unexpected response %T", stateRaw) + } + + var restoreReq Msg = &ResumeUnrollRequest{Height: height} + if !state.Started { + // Admission may have failed before the first checkpoint Stage. + // Recreate the original start identity from the durable + // registry row instead of inventing a + // TriggerRestart/standard-policy job. + restoreReq = &StartUnrollRequest{ + Height: height, + Trigger: record.Trigger, + ExitPolicyKind: record.ExitPolicyKind, + ExitPolicyRef: record.ExitPolicyRef, + } + } + + _, err = child.Ref().Ask(ctx, restoreReq).Await(ctx).Unpack() if err != nil { child.Stop() - return nil, fmt.Errorf("resume failed on restore: %w", err) + return nil, fmt.Errorf("start/resume failed on restore: %w", + err) } return child, nil @@ -1341,6 +1420,32 @@ func (r *registryBehavior) handlePersistRecordResult(ctx context.Context, if req.Err == "" { record, ok := r.pending[req.Outpoint] if ok && sameRegistryRecord(record, req.Record) { + if record.IsTerminal() { + policyKind := record.ExitPolicyKind + outpoint := req.Outpoint + policyKinds := r.terminalPolicyKinds + pendingKind, exists := policyKinds[outpoint] + if exists { + policyKind = pendingKind + } + r.notifyVTXOExit( + context.WithoutCancel(ctx), + &UnrollTerminatedMsg{ + Outpoint: req.Outpoint, + ActorID: record.ActorID, + Phase: record.Phase, + FailReason: record.FailReason, + SweepTxid: copyHash( + record.SweepTxid, + ), + HadOnChainFootprint: !record. + RecoverableFailure, + ExitPolicyKind: policyKind, + }, + policyKind, + ) + delete(r.terminalPolicyKinds, req.Outpoint) + } delete(r.pending, req.Outpoint) } else if ok { // Registry persistence retries are actor-owned work. @@ -1489,6 +1594,7 @@ func (r *registryBehavior) childConfig(target wire.OutPoint) Config { ExitSpendPolicyResolver: r.cfg.ExitSpendPolicyResolver, FraudCheckpointSafetyMargin: r.cfg.FraudCheckpointSafetyMargin, RegistryRef: r.selfRef, + ChainReconcilerFactory: r.cfg.ChainReconcilerFactory, } } diff --git a/unroll/registry_exit_test.go b/unroll/registry_exit_test.go index 852c07d9a..bab2cc77d 100644 --- a/unroll/registry_exit_test.go +++ b/unroll/registry_exit_test.go @@ -12,6 +12,7 @@ import ( "github.com/btcsuite/btcd/wire/v2" "github.com/lightninglabs/wavelength/baselib/actor" "github.com/lightninglabs/wavelength/lib/actormsg" + "github.com/lightninglabs/wavelength/txconfirm" "github.com/lightninglabs/wavelength/vtxo" fn "github.com/lightningnetwork/lnd/fn/v2" "github.com/stretchr/testify/require" @@ -88,10 +89,15 @@ func newExitObserverRegistry(target wire.OutPoint) (*registryBehavior, return behavior, observer } -// TestRegistryForwardsCleanFailureAsRecoverable verifies a terminal failure -// with no on-chain footprint is forwarded to the VTXO manager as a -// recoverable exit so the VTXO is rolled back to live (wavelength#602). -func TestRegistryForwardsCleanFailureAsRecoverable(t *testing.T) { +// TestRegistryForwardsAuthoritativelyCleanFailureAsRecoverable verifies a +// terminal failure whose canonical-absence reconciliation cleared +// ReliveUnsafe is forwarded to the VTXO manager as a recoverable exit. The +// live actor keeps ReliveUnsafe set across every chain-boundary attempt; this +// message shape is reserved for objective negative evidence supplied by a +// stronger reconciler. +func TestRegistryForwardsAuthoritativelyCleanFailureAsRecoverable( + t *testing.T) { + target := wire.OutPoint{Hash: chainhash.Hash{1}, Index: 0} behavior, observer := newExitObserverRegistry(target) @@ -111,6 +117,25 @@ func TestRegistryForwardsCleanFailureAsRecoverable(t *testing.T) { require.Equal(t, "min relay fee not met", notes[0].Reason) } +// TestRegistryDoesNotReliveRestartUncertainty verifies a clean-looking failure +// after restart remains fail-closed when canonical absence was not proven. +func TestRegistryDoesNotReliveRestartUncertainty(t *testing.T) { + target := wire.OutPoint{Hash: chainhash.Hash{9}, Index: 0} + behavior, observer := newExitObserverRegistry(target) + + _, err := behavior.handleTerminated(t.Context(), &UnrollTerminatedMsg{ + Outpoint: target, + ActorID: actorIDForTarget(target), + Phase: PhaseFailed, + FailReason: "reissue rejected", + HadOnChainFootprint: false, + ReliveUnsafe: true, + }).Unpack() + require.NoError(t, err) + require.Empty(t, observer.notifications()) + require.False(t, behavior.pending[target].RecoverableFailure) +} + // TestRegistryForwardsExitPolicyFromTerminalMsg verifies the exit policy the // VTXO manager sees on a recoverable failure comes from the child's terminal // message, not the registry's in-memory pending cache. The pending record is @@ -158,8 +183,9 @@ func TestRegistryForwardsExitPolicyFromTerminalMsg(t *testing.T) { // stamped kind would overwrite the store's (kind, ref) identity with // (kind, ""). The no-cache record therefore leaves the policy empty, // letting registryExitPolicy preserve the durable admission identity. - persisted, ok := behavior.pending[target] - require.True(t, ok) + persisted, err := behavior.cfg.Store.GetRecord(t.Context(), target) + require.NoError(t, err) + require.NotNil(t, persisted) require.Empty( t, persisted.ExitPolicyKind, "the terminal record must not "+ "carry a ref-less kind that clobbers the store "+ @@ -209,29 +235,48 @@ func TestRegistryForwardsCompletionAsConfirmed(t *testing.T) { require.Equal(t, vtxo.ExitOutcomeConfirmed, notes[0].Outcome) } -// TestRegistryRecoversCleanFailureEndToEnd is the in-process integration test -// for the wavelength#602 recovery path. Unlike the unit tests above (which -// call handleTerminated directly with a synthetic UnrollTerminatedMsg), this -// drives a REAL child unroll actor through admission, proof submission, and a -// txconfirm broadcast rejection to terminal Failed, then asserts the registry -// computes HadOnChainFootprint=false from the real FSM/planner state and -// forwards ExitOutcomeRecoverable to the VTXO manager observer. +// TestRegistryDoesNotReliveBroadcastFailureEndToEnd is the in-process +// integration test for a failure racing a chain-boundary side effect. Unlike +// the unit tests above (which call handleTerminated directly with a synthetic +// UnrollTerminatedMsg), this drives a real child through admission, proof +// submission, and a txconfirm rejection to terminal Failed. // -// This is the seam the unit tests cannot cover: the bug was an emergent -// lifecycle gap across the child actor → registry → observer chain, and the -// footprint determination is the change's highest-risk assumption. Here it -// runs against the real machinery rather than a hand-built message. -func TestRegistryRecoversCleanFailureEndToEnd(t *testing.T) { +// A rejection is not objective evidence that the transaction was never +// accepted or confirmed: a historical confirmation callback may already be +// queued behind it. The real actor must therefore keep ReliveUnsafe set, and +// the registry must withhold a recovery notification even though its local +// planner recorded no confirmed or in-flight transaction. +func TestRegistryDoesNotReliveBroadcastFailureEndToEnd(t *testing.T) { proof := buildLinearProof(t) desc := testDescriptor(t, proof.TargetOutpoint(), proof.CSVDelay()) + store := newMemRegistryStore() observer := &captureExitObserver{} var observerRef actor.TellOnlyRef[vtxo.ManagerMsg] = observer + checkpoints := newMemCheckpointStore() + guardPersistedBeforeSubmit := make(chan bool, 1) txconfirmRef := &fakeTxConfirmRef{} + txconfirmRef.onAsk = func(_ *txconfirm.EnsureConfirmedReq) { + checkpoint, loadErr := checkpoints.LoadCheckpoint( + context.Background(), + actorIDForTarget( + proof.TargetOutpoint(), + ), + ) + if loadErr != nil || checkpoint == nil { + guardPersistedBeforeSubmit <- false + + return + } + + decoded, decodeErr := decodeCheckpoint(checkpoint.StateData) + guardPersistedBeforeSubmit <- decodeErr == nil && + decoded.ReliveUnsafe + } cfg := RegistryConfig{ - Store: newMemRegistryStore(), - DeliveryStore: newMemCheckpointStore(), + Store: store, + DeliveryStore: checkpoints, ProofAssembler: &mockProofAssembler{ proof: proof, }, @@ -249,10 +294,9 @@ func TestRegistryRecoversCleanFailureEndToEnd(t *testing.T) { registry := newRegistryHarnessWithSpawn(t, cfg) t.Cleanup(registry.Stop) - // Reject the first proof tx the child submits (the #602 trigger: e.g. a - // sub-dust proof tx that can't meet min relay fee). The child drives a - // terminal TxFailedEvent with nothing confirmed and nothing left - // in-flight, so the job is a clean failure with no on-chain footprint. + // Reject the first proof tx the child submits (for example, a sub-dust + // proof tx that cannot meet the relay fee). This proves only that this + // submission failed; it does not prove canonical absence. txconfirmRef.setImmediateFailed(proof.RootTxids()[0], "min relay fee") _, err := registry.Ref().Ask(t.Context(), &EnsureUnrollRequest{ @@ -260,33 +304,44 @@ func TestRegistryRecoversCleanFailureEndToEnd(t *testing.T) { Trigger: TriggerManual, }).Await(t.Context()).Unpack() require.NoError(t, err) + require.True( + t, <-guardPersistedBeforeSubmit, "fresh admission must "+ + "persist the fail-closed relive guard before "+ + "txconfirm submission", + ) require.Eventually(t, func() bool { - return len(observer.notifications()) == 1 + record, err := store.GetRecord( + t.Context(), proof.TargetOutpoint(), + ) + require.NoError(t, err) + + return record != nil && record.Phase == PhaseFailed }, testTimeout, 10*time.Millisecond, - "registry should forward a recovery notification") + "registry should durably record the terminal failure") - notes := observer.notifications() - require.Equal(t, proof.TargetOutpoint(), notes[0].Outpoint) - require.Equal( - t, vtxo.ExitOutcomeRecoverable, notes[0].Outcome, - "a clean no-broadcast failure must be reported as recoverable", + record, err := store.GetRecord(t.Context(), proof.TargetOutpoint()) + require.NoError(t, err) + require.False(t, record.RecoverableFailure) + require.Empty( + t, observer.notifications(), + "a broadcast rejection without canonical-absence evidence "+ + "must not relive the target", ) } -// TestRegistryFailedAdmissionNotifiesRecoverable verifies that a child whose -// start fails before any broadcast (failAdmittedChild) both persists a -// recoverable terminal record and notifies the VTXO manager to roll the VTXO -// back to live. These pre-broadcast failures move ownership to unilateral -// exit but leave no on-chain footprint, so they must be recoverable -// (wavelength#602). -func TestRegistryFailedAdmissionNotifiesRecoverable(t *testing.T) { +// TestRegistryFailedAdmissionDoesNotRelive verifies that a child start error +// is not accepted as objective no-footprint evidence. Outbox work may have +// reached txconfirm before the error surfaced, so the registry must retain a +// non-terminal hold and leave the VTXO in unilateral exit. +func TestRegistryFailedAdmissionDoesNotRelive(t *testing.T) { proof := buildLinearProof(t) desc := testDescriptor(t, proof.TargetOutpoint(), proof.CSVDelay()) store := newMemRegistryStore() observer := &captureExitObserver{} var observerRef actor.TellOnlyRef[vtxo.ManagerMsg] = observer + submissionCrossed := make(chan struct{}, 1) registry := newRegistryHarnessWithSpawn(t, RegistryConfig{ Store: store, @@ -300,13 +355,25 @@ func TestRegistryFailedAdmissionNotifiesRecoverable(t *testing.T) { }) t.Cleanup(registry.Stop) + var spawnCount int registry.behavior.spawnFunc = func(_ context.Context, target wire.OutPoint) (*VTXOUnrollActor, error) { + spawnCount++ + firstSpawn := spawnCount == 1 behavior := actor.NewFunctionBehavior( func(_ context.Context, msg Msg) fn.Result[Resp] { switch msg.(type) { case *StartUnrollRequest: + if !firstSpawn { + return fn.Ok[Resp](&AckResp{}) + } + + // Model an outbox submission succeeding + // before a later stage/commit error + // reaches the registry. + submissionCrossed <- struct{}{} + return fn.Err[Resp]( errors.New("start boom"), ) @@ -336,24 +403,36 @@ func TestRegistryFailedAdmissionNotifiesRecoverable(t *testing.T) { Trigger: TriggerManual, }).Await(t.Context()).Unpack() require.Error(t, err) + select { + case <-submissionCrossed: + case <-time.After(testTimeout): + t.Fatal("start error arrived before modeled submission") + } - // The pre-broadcast failure is reported to the manager as recoverable. - require.Eventually(t, func() bool { - return len(observer.notifications()) == 1 - }, testTimeout, 10*time.Millisecond, - "failed admission should notify the manager") - - notes := observer.notifications() - require.Equal(t, proof.TargetOutpoint(), notes[0].Outpoint) - require.Equal(t, vtxo.ExitOutcomeRecoverable, notes[0].Outcome) + require.Empty(t, observer.notifications()) - // And the durable record is the recoverable terminal variant, so boot - // reconciliation can recover it even if the notification is lost. + // The durable record stays non-terminal so a fresh Ensure or restart + // can restore it without making the ambiguous failure spendable. record, err := store.GetRecord(t.Context(), proof.TargetOutpoint()) require.NoError(t, err) require.NotNil(t, record) - require.Equal(t, PhaseFailed, record.Phase) - require.True(t, record.RecoverableFailure) + require.Equal(t, PhasePending, record.Phase) + require.False(t, record.RecoverableFailure) + require.Contains(t, record.FailReason, "start boom") + + // Once the transient error clears, a fresh caller restores the durable + // hold. It is not a new admission because ownership never returned + // live. + resp, err := registry.Ref().Ask(t.Context(), &EnsureUnrollRequest{ + Outpoint: proof.TargetOutpoint(), + Trigger: TriggerManual, + }).Await(t.Context()).Unpack() + require.NoError(t, err) + ensureResp, ok := resp.(*EnsureUnrollResp) + require.True(t, ok) + require.False(t, ensureResp.Created) + require.Equal(t, 2, spawnCount) + require.Empty(t, observer.notifications()) } // TestRegistryReadmitsTargetAfterRecoverableFailure verifies that once a VTXO @@ -363,92 +442,34 @@ func TestRegistryFailedAdmissionNotifiesRecoverable(t *testing.T) { // trade a guaranteed strand for a deferred one: the VTXO would recover to live // but could never be unrolled again, because the terminal record blocks every // subsequent admission. This exercises the in-memory pending-cache arm of the -// dedup gate (the record lingers in r.pending after failAdmittedChild). +// dedup gate after a real child terminal handoff has classified the failure. func TestRegistryReadmitsTargetAfterRecoverableFailure(t *testing.T) { proof := buildLinearProof(t) desc := testDescriptor(t, proof.TargetOutpoint(), proof.CSVDelay()) target := proof.TargetOutpoint() store := newMemRegistryStore() - observer := &captureExitObserver{} - var observerRef actor.TellOnlyRef[vtxo.ManagerMsg] = observer - registry := newRegistryHarnessWithSpawn(t, RegistryConfig{ - Store: store, - DeliveryStore: newMemCheckpointStore(), - ProofAssembler: &mockProofAssembler{proof: proof}, - VTXOStore: &mockVTXOStore{desc: desc}, - TxConfirmRef: &fakeTxConfirmRef{}, - ChainSource: &fakeRegistryChainSourceRef{height: 200}, - Wallet: &fakeSweepWallet{}, - VTXOExitObserver: fn.Some(observerRef), + Store: store, + DeliveryStore: newMemCheckpointStore(), + ProofAssembler: &mockProofAssembler{proof: proof}, + VTXOStore: &mockVTXOStore{desc: desc}, + TxConfirmRef: &fakeTxConfirmRef{}, + ChainSource: &fakeRegistryChainSourceRef{height: 200}, + Wallet: &fakeSweepWallet{}, }) t.Cleanup(registry.Stop) - // The first spawn fails its start (pre-broadcast admission failure, - // the #602 trigger); the second spawn is a healthy child. A counter - // distinguishes the two admissions deterministically. - var spawnCount int - registry.behavior.spawnFunc = func(_ context.Context, - spawnTarget wire.OutPoint) (*VTXOUnrollActor, error) { - - spawnCount++ - firstSpawn := spawnCount == 1 - childBehavior := actor.NewFunctionBehavior( - func(_ context.Context, msg Msg) fn.Result[Resp] { - switch msg.(type) { - case *StartUnrollRequest: - if firstSpawn { - return fn.Err[Resp]( - errors.New( - "start boom", - ), - ) - } - - return fn.Ok[Resp](&AckResp{}) - - case *GetStateRequest: - return fn.Ok[Resp](&GetStateResp{ - Started: !firstSpawn, - Phase: PhasePending, - }) - - default: - return fn.Err[Resp]( - fmt.Errorf("unexpected msg %T", - msg), - ) - } - }, - ) - - // Test children are owned by t.Cleanup after creation. - //nolint:contextcheck - return newTestUnrollChild(t, spawnTarget, childBehavior), nil + seeded := RegistryRecord{ + TargetOutpoint: target, + ActorID: "prior-clean-failure", + Trigger: TriggerManual, + Phase: PhaseFailed, + FailReason: "proof tx rejected before broadcast", + RecoverableFailure: true, } - - // First admission fails pre-broadcast: failAdmittedChild records a - // recoverable terminal row, leaves it in r.pending, and notifies the - // manager to roll the VTXO back to live. - _, err := registry.Ref().Ask(t.Context(), &EnsureUnrollRequest{ - Outpoint: target, - Trigger: TriggerManual, - }).Await(t.Context()).Unpack() - require.Error(t, err) - - require.Eventually(t, func() bool { - return len(observer.notifications()) == 1 - }, testTimeout, 10*time.Millisecond, - "failed admission should roll the VTXO back to live") - - pre, err := store.GetRecord(t.Context(), target) - require.NoError(t, err) - require.NotNil(t, pre) - require.True( - t, pre.RecoverableFailure, - "the failed admission must leave a recoverable terminal record", - ) + require.NoError(t, store.UpsertRecord(t.Context(), seeded)) + registry.behavior.pending[target] = cloneRegistryRecord(seeded) // Second admission for the SAME outpoint must re-admit (Created=true) // rather than dedup against the recoverable record. @@ -465,9 +486,6 @@ func TestRegistryReadmitsTargetAfterRecoverableFailure(t *testing.T) { "re-admittable, not deduped against the dead "+ "recoverable record", ) - require.Equal( - t, 2, spawnCount, "the second Ensure must spawn a fresh child", - ) // The stale recoverable record is overwritten by the fresh admission, // so the VTXO can progress through a new exit. @@ -478,7 +496,7 @@ func TestRegistryReadmitsTargetAfterRecoverableFailure(t *testing.T) { t, post.RecoverableFailure, "re-admission must overwrite the stale recoverable record", ) - require.Equal(t, PhasePending, post.Phase) + require.Equal(t, PhaseMaterializing, post.Phase) } // TestRegistryReadmitsTargetAfterRecoverableFailureAcrossRestart verifies the diff --git a/unroll/registry_messages.go b/unroll/registry_messages.go index e6dc0a538..089c4d906 100644 --- a/unroll/registry_messages.go +++ b/unroll/registry_messages.go @@ -155,13 +155,18 @@ type UnrollTerminatedMsg struct { SweepTxid *chainhash.Hash // HadOnChainFootprint reports whether the job ever published anything - // on-chain (a confirmed or in-flight proof node, or a broadcast - // sweep). It is false only for a clean failure that never broadcast, - // which is the sole case where the target VTXO is safe to roll back - // to live (the operator still considers it live). See - // wavelength#602. + // on-chain (a confirmed or in-flight proof node, or a broadcast sweep). + // False means only that the local planner recorded no footprint; + // ReliveUnsafe must also be false before the target VTXO may return to + // live. See wavelength#602. HadOnChainFootprint bool + // ReliveUnsafe reports that chain-boundary uncertainty prevents a + // terminal failure from proving the VTXO remained live, even if this + // process did not record its own on-chain footprint. It remains set + // until objective canonical-absence evidence clears it. + ReliveUnsafe bool + // ExitPolicyKind is the child's durable exit policy for this target. // The child sources it from its own persisted state, so it stays // authoritative even after the registry has evicted its in-memory diff --git a/unroll/registry_test.go b/unroll/registry_test.go index a0b61b499..32ec18cd8 100644 --- a/unroll/registry_test.go +++ b/unroll/registry_test.go @@ -179,6 +179,26 @@ func (s *terminalFlakyRegistryStore) UpsertRecord(ctx context.Context, return s.memRegistryStore.UpsertRecord(ctx, record) } +// MarkTerminal fails through the same counter as terminal UpsertRecord so the +// synchronous durability barrier falls back to the retry path under test. +func (s *terminalFlakyRegistryStore) MarkTerminal(ctx context.Context, + target wire.OutPoint, phase Phase, recoverable bool, failReason string, + sweepTxid *chainhash.Hash) error { + + s.mu.Lock() + if s.upsertErrors > 0 { + s.upsertErrors-- + s.mu.Unlock() + + return errors.New("injected terminal upsert failure") + } + s.mu.Unlock() + + return s.memRegistryStore.MarkTerminal( + ctx, target, phase, recoverable, failReason, sweepTxid, + ) +} + // alwaysFailUpsertRegistryStore rejects every terminal-phase upsert while // keeping the non-terminal initial write and all read paths backed by the // in-memory store. This matches the fail-closed admission contract that the @@ -210,6 +230,13 @@ func (s *alwaysFailUpsertRegistryStore) UpsertRecord(ctx context.Context, return errors.New("injected upsert failure") } +// MarkTerminal rejects the synchronous terminal durability barrier too. +func (s *alwaysFailUpsertRegistryStore) MarkTerminal(context.Context, + wire.OutPoint, Phase, bool, string, *chainhash.Hash) error { + + return errors.New("injected terminal failure") +} + // blockingRegistryStore holds terminal-phase upserts until released so tests // can verify that registry status remains available while persistence is // stalled. Non-terminal writes (including the fail-closed admission write @@ -255,6 +282,14 @@ func (s *blockingRegistryStore) UpsertRecord(ctx context.Context, return s.memRegistryStore.UpsertRecord(ctx, record) } +// MarkTerminal pushes this test store onto the asynchronous fallback so the +// blocking UpsertRecord can be observed without blocking the registry actor. +func (s *blockingRegistryStore) MarkTerminal(context.Context, wire.OutPoint, + Phase, bool, string, *chainhash.Hash) error { + + return errors.New("use asynchronous terminal persistence") +} + // cancelOnPendingRegistryStore cancels the caller context after the initial // pending admission row has been persisted. type cancelOnPendingRegistryStore struct { @@ -1158,9 +1193,10 @@ func TestRegistryEnsureStartsChildAfterCallerCancellation(t *testing.T) { require.NoError(t, <-startCtxErr) } -// TestRegistryEnsureMarksRealStartErrorFailed verifies that the pending-row -// safeguard does not hide non-cancellation child start failures. -func TestRegistryEnsureMarksRealStartErrorFailed(t *testing.T) { +// TestRegistryEnsureParksRealStartError verifies that a child start error +// remains retryable and fail-closed instead of being misclassified as proof +// that the VTXO is safe to relive. +func TestRegistryEnsureParksRealStartError(t *testing.T) { proof := buildLinearProof(t) desc := testDescriptor(t, proof.TargetOutpoint(), proof.CSVDelay()) store := newMemRegistryStore() @@ -1218,8 +1254,9 @@ func TestRegistryEnsureMarksRealStartErrorFailed(t *testing.T) { record, err := store.GetRecord(t.Context(), proof.TargetOutpoint()) require.NoError(t, err) require.NotNil(t, record) - require.Equal(t, PhaseFailed, record.Phase) + require.Equal(t, PhasePending, record.Phase) require.Contains(t, record.FailReason, "start boom") + require.False(t, record.RecoverableFailure) } // TestRegistryTerminalPersistRetriesUntilDurable verifies that a @@ -1356,15 +1393,18 @@ func TestRegistryTerminalStatusRemainsQueryableWhilePersistBlocked( store := newBlockingRegistryStore() checkpoints := newMemCheckpointStore() txconfirmRef := &fakeTxConfirmRef{} + observer := &captureExitObserver{} + var observerRef actor.TellOnlyRef[vtxo.ManagerMsg] = observer registry := newRegistryHarnessWithSpawn(t, RegistryConfig{ - Store: store, - DeliveryStore: checkpoints, - ProofAssembler: &mockProofAssembler{proof: proof}, - VTXOStore: &mockVTXOStore{desc: desc}, - TxConfirmRef: txconfirmRef, - ChainSource: &fakeRegistryChainSourceRef{height: 200}, - Wallet: &fakeSweepWallet{}, + Store: store, + DeliveryStore: checkpoints, + ProofAssembler: &mockProofAssembler{proof: proof}, + VTXOStore: &mockVTXOStore{desc: desc}, + TxConfirmRef: txconfirmRef, + ChainSource: &fakeRegistryChainSourceRef{height: 200}, + Wallet: &fakeSweepWallet{}, + VTXOExitObserver: fn.Some(observerRef), }) t.Cleanup(registry.Stop) @@ -1387,6 +1427,10 @@ func TestRegistryTerminalStatusRemainsQueryableWhilePersistBlocked( case <-time.After(testTimeout): t.Fatal("timed out waiting for blocked registry persist") } + require.Empty( + t, observer.notifications(), + "terminal outcome must wait for durable invalidation", + ) require.Eventually(t, func() bool { resp, err := registry.Ref().Ask( @@ -1412,6 +1456,11 @@ func TestRegistryTerminalStatusRemainsQueryableWhilePersistBlocked( return record != nil && record.Phase == PhaseFailed }, testTimeout, 10*time.Millisecond) + require.Empty( + t, observer.notifications(), + "a real broadcast failure remains held without objective "+ + "canonical-absence evidence", + ) } // TestRegistryRestoreFailureLeavesRecordRetryable verifies that a @@ -1550,12 +1599,11 @@ func TestRegistryRestoreFailureLeavesRecordRetryable(t *testing.T) { require.Equal(t, PhaseMaterializing, status.Phase) } -// TestRegistryLateAdmissionFailureMarksFailed verifies that when the +// TestRegistryLateAdmissionFailureStaysPending verifies that when the // registry's synchronous admission wait times out, the eventual child -// start result is still observed and a late deterministic start failure -// becomes a terminal registry record instead of leaving PhasePending -// forever. -func TestRegistryLateAdmissionFailureMarksFailed(t *testing.T) { +// start result is still observed while the durable record remains +// non-terminal and fail-closed for a later restore. +func TestRegistryLateAdmissionFailureStaysPending(t *testing.T) { proof := buildLinearProof(t) desc := testDescriptor(t, proof.TargetOutpoint(), proof.CSVDelay()) store := newMemRegistryStore() @@ -1601,11 +1649,81 @@ func TestRegistryLateAdmissionFailureMarksFailed(t *testing.T) { require.NoError(t, err) return record != nil && - record.Phase == PhaseFailed && + record.Phase == PhasePending && + !record.RecoverableFailure && strings.Contains(record.FailReason, "late start boom") }, testTimeout, 10*time.Millisecond) } +// TestRegistryRestoreWithoutCheckpointReplaysOriginalStart verifies that a +// parked admission which failed before its first checkpoint does not resume as +// an invented restart/standard-policy job. The durable registry identity must +// seed a fresh Start request verbatim. +func TestRegistryRestoreWithoutCheckpointReplaysOriginalStart(t *testing.T) { + target := wire.OutPoint{Hash: chainhash.Hash{0xe1}, Index: 4} + startRequests := make(chan *StartUnrollRequest, 1) + behavior := ®istryBehavior{} + behavior.spawnFunc = func(_ context.Context, + spawnTarget wire.OutPoint) (*VTXOUnrollActor, error) { + + require.Equal(t, target, spawnTarget) + childBehavior := actor.NewFunctionBehavior( + func(_ context.Context, msg Msg) fn.Result[Resp] { + switch request := msg.(type) { + case *GetStateRequest: + return fn.Ok[Resp](&GetStateResp{ + Started: false, + Phase: PhasePending, + }) + + case *StartUnrollRequest: + startRequests <- request + + return fn.Ok[Resp](&AckResp{}) + + case *ResumeUnrollRequest: + return fn.Err[Resp]( + errors.New("unexpected resume"), + ) + + default: + return fn.Err[Resp]( + fmt.Errorf("unexpected msg %T", + msg), + ) + } + }, + ) + + // Test children are owned by the returned handle below. + //nolint:contextcheck + return newTestUnrollChild(t, spawnTarget, childBehavior), nil + } + + record := RegistryRecord{ + TargetOutpoint: target, + ActorID: actorIDForTarget(target), + Trigger: TriggerFraudSpend, + ExitPolicyKind: ExitPolicyKind("custom-recovery-policy"), + ExitPolicyRef: "recovery-job-42", + Phase: PhasePending, + } + child, err := behavior.tryRestoreOne(t.Context(), record, 321) + require.NoError(t, err) + t.Cleanup(child.Stop) + + select { + case request := <-startRequests: + require.Equal(t, int32(321), request.Height) + require.Equal(t, record.Trigger, request.Trigger) + require.Equal(t, record.ExitPolicyKind, request.ExitPolicyKind) + require.Equal(t, record.ExitPolicyRef, request.ExitPolicyRef) + + case <-time.After(testTimeout): + t.Fatal("restore did not replay original start identity") + } +} + // TestRegistryEnsureRestoresFailedNonTerminalRecord verifies that when a // non-terminal record exists in the durable store but no child is // active (because a prior RestoreNonTerminal hit a transient error and diff --git a/unroll/reorg_safety_test.go b/unroll/reorg_safety_test.go new file mode 100644 index 000000000..0425270bb --- /dev/null +++ b/unroll/reorg_safety_test.go @@ -0,0 +1,781 @@ +package unroll + +import ( + "context" + "sync" + "testing" + "time" + + "github.com/btcsuite/btcd/chainhash/v2" + "github.com/btcsuite/btclog/v2" + "github.com/lightninglabs/wavelength/baselib/actor" + "github.com/lightninglabs/wavelength/txconfirm" + "github.com/lightninglabs/wavelength/unrollplan" + fn "github.com/lightningnetwork/lnd/fn/v2" + "github.com/stretchr/testify/require" +) + +// TestProofRootReorgBlocksDownstreamMaterialization drives a proof root +// through Confirmed -> Reorged -> Confirmed and asserts that the unroll +// actor does NOT submit the target transaction while the root anchor is +// torn down. Only once the root reconfirms does the actor advance the +// proof graph. +// +// This is the load-bearing reorg-safety case: before the rollback work +// the actor treated the first TxConfirmedMsg as a monotonic fact and +// would have submitted the target while the chain was still missing the +// root. After rollback, the target submission is gated on the root +// anchor being live. +func TestProofRootReorgBlocksDownstreamMaterialization(t *testing.T) { + proof := buildLinearProof(t) + desc := testDescriptor(t, proof.TargetOutpoint(), proof.CSVDelay()) + unrollActor, _, txconfirmRef, _ := newActorHarness(t, proof, desc) + + mustAsk(t, unrollActor.Ref(), &StartUnrollRequest{ + Height: 100, + Trigger: TriggerManual, + }) + + rootTxid := proof.RootTxids()[0] + targetTxid := proof.TargetOutpoint().Hash + + // Wait for the actor to register the root with txconfirm. + require.Eventually(t, func() bool { + return txconfirmRef.requestCountForTxid(rootTxid) == 1 + }, testTimeout, 10*time.Millisecond, "root never submitted") + + // 1. Root confirms; the actor should immediately submit the target. + txconfirmRef.emitConfirmed(t, 0, rootTxid, 101) + require.Eventually(t, func() bool { + return txconfirmRef.requestCountForTxid(targetTxid) == 1 + }, testTimeout, 10*time.Millisecond, "target never submitted") + + // 2. Root reorgs out. The actor must drop the root anchor AND the + // dependent target anchor from in-flight, since State.Validate + // requires every in-flight node to have confirmed parents in the + // proof graph. + targetSubmissionsBeforeReorg := txconfirmRef.requestCountForTxid( + targetTxid, + ) + txconfirmRef.emitReorged(t, 0, rootTxid) + + require.Eventually(t, func() bool { + resp, ok := mustAsk( + t, unrollActor.Ref(), &GetStateRequest{}, + ).(*GetStateResp) + require.True(t, ok) + for _, h := range resp.PlannerState.ConfirmedTxids { + if h == rootTxid { + return false + } + } + for _, h := range resp.PlannerState.InFlightTxids { + if h == targetTxid { + return false + } + } + + return true + }, testTimeout, 10*time.Millisecond, + "root + target anchors never cleared after reorg") + + // The actor naturally re-submits the root on its ready frontier + // after the reorg drops it from ConfirmedTxids — txconfirm dedup + // absorbs that re-submit. What matters for reorg safety is that the + // TARGET was NOT re-submitted while its parent anchor was missing: + // driving target broadcast off a non-anchored parent is exactly the + // unsafe behavior this work fixes. + time.Sleep(50 * time.Millisecond) + require.Equal( + t, targetSubmissionsBeforeReorg, + txconfirmRef.requestCountForTxid(targetTxid), + "target was re-submitted while its parent was reorged out", + ) + + // 3. Root reconfirms in a different block. The actor should keep + // progressing without resubmitting target (txconfirm dedup already + // holds the in-flight subscription, but the planner state must be + // coherent). + txconfirmRef.emitConfirmed(t, 0, rootTxid, 102) + require.Eventually(t, func() bool { + resp, ok := mustAsk( + t, unrollActor.Ref(), &GetStateRequest{}, + ).(*GetStateResp) + require.True(t, ok) + for _, h := range resp.PlannerState.ConfirmedTxids { + if h == rootTxid { + return true + } + } + + return false + }, testTimeout, 10*time.Millisecond, + "root anchor never re-recorded after reconfirm") +} + +// TestTargetReorgClearsCSVMaturity confirms the target tx, waits for the +// sweep to broadcast, then reorgs the target out. The actor must clear +// TargetConfirmHeight and downgrade the sweep so the planner stops +// reporting Done. +func TestTargetReorgClearsCSVMaturity(t *testing.T) { + proof := buildLinearProof(t) + desc := testDescriptor(t, proof.TargetOutpoint(), proof.CSVDelay()) + unrollActor, _, txconfirmRef, _ := newActorHarness(t, proof, desc) + + mustAsk(t, unrollActor.Ref(), &StartUnrollRequest{ + Height: 100, + Trigger: TriggerManual, + }) + + rootTxid := proof.RootTxids()[0] + targetTxid := proof.TargetOutpoint().Hash + + require.Eventually(t, func() bool { + return txconfirmRef.requestCountForTxid(rootTxid) == 1 + }, testTimeout, 10*time.Millisecond) + + txconfirmRef.emitConfirmed(t, 0, rootTxid, 101) + require.Eventually(t, func() bool { + return txconfirmRef.requestCountForTxid(targetTxid) == 1 + }, testTimeout, 10*time.Millisecond) + + txconfirmRef.emitConfirmed(t, 1, targetTxid, 102) + + // Advance height past CSV maturity so the sweep is built and + // broadcast. + mustAsk(t, unrollActor.Ref(), &HeightObservedMsg{ + Height: 102 + int32(proof.CSVDelay()) + 1, + }) + + require.Eventually(t, func() bool { + return txconfirmRef.requestCount() >= 3 + }, testTimeout, 10*time.Millisecond, + "sweep was never broadcast") + + sweepReq := txconfirmRef.lastRequest(t) + sweepTxid := sweepReq.Tx.TxHash() + + // Sweep confirms. + txconfirmRef.emitConfirmed(t, 2, sweepTxid, 110) + require.Eventually(t, func() bool { + resp, ok := mustAsk( + t, unrollActor.Ref(), &GetStateRequest{}, + ).(*GetStateResp) + require.True(t, ok) + + return resp.Phase == PhaseCompleted + }, testTimeout, 10*time.Millisecond, + "sweep never reached Completed") + + // 1. Target reorg invalidates the CSV anchor AND the sweep + // confirmation (which depended on it). + txconfirmRef.emitReorged(t, 1, targetTxid) + + require.Eventually(t, func() bool { + resp, ok := mustAsk( + t, unrollActor.Ref(), &GetStateRequest{}, + ).(*GetStateResp) + require.True(t, ok) + + csvCleared := resp.PlannerState.TargetConfirmHeight.IsNone() + targetCleared := true + for _, h := range resp.PlannerState.ConfirmedTxids { + if h == targetTxid { + targetCleared = false + } + } + + return csvCleared && targetCleared + }, testTimeout, 10*time.Millisecond, + "target anchor never cleared after reorg") +} + +// recordingRegistryRef is a stub RegistryRef that captures every +// message delivered by the per-target actor's notifyRegistryIfTerminal +// path so a test can assert "the registry has / has not been told the +// actor is terminal". +type recordingRegistryRef struct { + id string + mu sync.Mutex + msgs []RegistryMsg +} + +// ID returns the stub registry ref identifier. +func (r *recordingRegistryRef) ID() string { + return r.id +} + +// Tell records the inbound RegistryMsg and acks. +func (r *recordingRegistryRef) Tell(_ context.Context, msg RegistryMsg) error { + r.mu.Lock() + defer r.mu.Unlock() + r.msgs = append(r.msgs, msg) + + return nil +} + +// terminatedCount returns how many UnrollTerminatedMsg messages have +// been delivered so far. +func (r *recordingRegistryRef) terminatedCount() int { + r.mu.Lock() + defer r.mu.Unlock() + + count := 0 + for _, m := range r.msgs { + if _, ok := m.(*UnrollTerminatedMsg); ok { + count++ + } + } + + return count +} + +// terminatedMessages returns a copy of every terminal message captured by the +// registry stub. +func (r *recordingRegistryRef) terminatedMessages() []*UnrollTerminatedMsg { + r.mu.Lock() + defer r.mu.Unlock() + + var messages []*UnrollTerminatedMsg + for _, msg := range r.msgs { + terminated, ok := msg.(*UnrollTerminatedMsg) + if !ok { + continue + } + + messages = append(messages, terminated) + } + + return messages +} + +// TestRestartFailureCarriesDurableReliveGuard verifies that restart marks the +// job unsafe to relive before it reissues an in-flight transaction, and that a +// synchronous rejection carries the guard through the real actor terminal +// handoff. This closes the process-boundary gap where a conflicting offline +// spend could otherwise make the reissue fail "cleanly" and revive a spent +// VTXO. +func TestRestartFailureCarriesDurableReliveGuard(t *testing.T) { + proof := buildLinearProof(t) + desc := testDescriptor(t, proof.TargetOutpoint(), proof.CSVDelay()) + store := newMemCheckpointStore() + txconfirmRef := &fakeTxConfirmRef{} + registryRef := &recordingRegistryRef{id: "restart-reg-stub"} + rootTxid := proof.RootTxids()[0] + + raw, err := encodeCheckpoint(&actorCheckpoint{ + Version: checkpointVersion, + Height: 100, + Started: true, + Trigger: TriggerManual, + State: unrollplan.State{ + InFlightTxids: []chainhash.Hash{rootTxid}, + }, + }) + require.NoError(t, err) + require.NoError( + t, + store.SaveCheckpoint( + t.Context(), actor.CheckpointParams{ + ActorID: "restart-relive-guard-test", + StateType: checkpointStateType, + StateData: raw, + Version: checkpointVersion, + }, + ), + ) + + persistedBeforeReissue := make(chan bool, 1) + txconfirmRef.onAsk = func(_ *txconfirm.EnsureConfirmedReq) { + checkpoint, loadErr := store.LoadCheckpoint( + context.Background(), + "restart-relive-guard-test", + ) + if loadErr != nil || checkpoint == nil { + persistedBeforeReissue <- false + + return + } + + decoded, decodeErr := decodeCheckpoint(checkpoint.StateData) + persisted := decodeErr == nil && decoded.ReliveUnsafe + persistedBeforeReissue <- persisted + } + txconfirmRef.setImmediateFailed(rootTxid, "conflicting spend") + + cfg := Config{ + TargetOutpoint: proof.TargetOutpoint(), + ActorID: "restart-relive-guard-test", + DeliveryStore: store, + ProofAssembler: &mockProofAssembler{ + proof: proof, + }, + VTXOStore: &mockVTXOStore{ + desc: desc, + }, + TxConfirmRef: txconfirmRef, + ChainSource: &fakeChainSourceRef{}, + Wallet: &fakeSweepWallet{}, + Log: fn.Some(btclog.Disabled), + RegistryRef: registryRef, + } + resumeBehavior := &behavior{cfg: cfg, log: btclog.Disabled} + require.NoError(t, resumeBehavior.restoreCheckpoint(t.Context())) + + resumedActor := actor.NewActor(actor.ActorConfig[Msg, Resp]{ + ID: "restart-relive-guard-test", + Behavior: adaptTx(resumeBehavior), + MailboxSize: 64, + }) + resumeBehavior.selfRef = resumedActor.TellRef() + resumedActor.Start() + t.Cleanup(resumedActor.Stop) + + mustAsk(t, resumedActor.Ref(), &ResumeUnrollRequest{Height: 101}) + require.True( + t, <-persistedBeforeReissue, + "relive guard must be durable before txconfirm reissue", + ) + require.Eventually(t, func() bool { + return registryRef.terminatedCount() == 1 + }, testTimeout, 10*time.Millisecond) + + messages := registryRef.terminatedMessages() + require.Len(t, messages, 1) + require.Equal(t, PhaseFailed, messages[0].Phase) + require.False(t, messages[0].HadOnChainFootprint) + require.True(t, messages[0].ReliveUnsafe) +} + +// TestSweepCompletionStaysProvisionalUntilFinalized proves the Phase 7 +// invariant: a per-target actor that has reached PhaseCompleted does +// NOT notify the registry as terminal until a TxFinalizedMsg for the +// sweep txid arrives. Until then the actor stays alive so a reorg of +// the sweep confirmation has a live actor to deliver the rollback to. +func TestSweepCompletionStaysProvisionalUntilFinalized(t *testing.T) { + proof := buildLinearProof(t) + desc := testDescriptor(t, proof.TargetOutpoint(), proof.CSVDelay()) + + store := newMemCheckpointStore() + txconfirmRef := &fakeTxConfirmRef{} + registryRef := &recordingRegistryRef{id: "reg-stub"} + + cfg := Config{ + TargetOutpoint: proof.TargetOutpoint(), + ActorID: "unroll-finality-test", + DeliveryStore: store, + ProofAssembler: &mockProofAssembler{ + proof: proof, + }, + VTXOStore: &mockVTXOStore{ + desc: desc, + }, + TxConfirmRef: txconfirmRef, + ChainSource: &fakeChainSourceRef{}, + Wallet: &fakeSweepWallet{}, + Log: fn.Some(btclog.Disabled), + RegistryRef: registryRef, + } + beh := &behavior{cfg: cfg, log: btclog.Disabled} + require.NoError(t, beh.restoreCheckpoint(t.Context())) + + actorInstance := actor.NewActor(actor.ActorConfig[Msg, Resp]{ + ID: "unroll-finality-test", + Behavior: &txExecAdapter{b: beh, ax: newMemExecFor(beh)}, + MailboxSize: 32, + }) + beh.selfRef = actorInstance.TellRef() + actorInstance.Start() + t.Cleanup(actorInstance.Stop) + + mustAsk(t, actorInstance.Ref(), &StartUnrollRequest{ + Height: 100, + Trigger: TriggerManual, + }) + + rootTxid := proof.RootTxids()[0] + targetTxid := proof.TargetOutpoint().Hash + + require.Eventually(t, func() bool { + return txconfirmRef.requestCountForTxid(rootTxid) == 1 + }, testTimeout, 10*time.Millisecond) + + txconfirmRef.emitConfirmed(t, 0, rootTxid, 101) + require.Eventually(t, func() bool { + return txconfirmRef.requestCountForTxid(targetTxid) == 1 + }, testTimeout, 10*time.Millisecond) + + txconfirmRef.emitConfirmed(t, 1, targetTxid, 102) + mustAsk(t, actorInstance.Ref(), &HeightObservedMsg{ + Height: 102 + int32(proof.CSVDelay()) + 1, + }) + + require.Eventually(t, func() bool { + return txconfirmRef.requestCount() >= 3 + }, testTimeout, 10*time.Millisecond) + + sweepTxid := txconfirmRef.lastRequest(t).Tx.TxHash() + txconfirmRef.emitConfirmed(t, 2, sweepTxid, 110) + + // Wait for the actor to settle on PhaseCompleted. + require.Eventually(t, func() bool { + resp, ok := mustAsk( + t, actorInstance.Ref(), &GetStateRequest{}, + ).(*GetStateResp) + require.True(t, ok) + + return resp.Phase == PhaseCompleted + }, testTimeout, 10*time.Millisecond) + + // 1. PROVISIONAL: registry must NOT have been told the actor is + // terminal. The sweep can still reorg out. + time.Sleep(50 * time.Millisecond) + require.Equal( + t, 0, registryRef.terminatedCount(), + "registry was told terminal before TxFinalized arrived", + ) + + // 2. Sweep reorgs out; the live actor rolls back to + // AwaitingSweepConfirmation. Registry still has not been + // terminated. + txconfirmRef.emitReorged(t, 2, sweepTxid) + require.Eventually(t, func() bool { + resp, ok := mustAsk( + t, actorInstance.Ref(), &GetStateRequest{}, + ).(*GetStateResp) + require.True(t, ok) + + return resp.Phase == PhaseSweepConfirmation + }, testTimeout, 10*time.Millisecond, + "actor never rolled back to AwaitingSweepConfirmation") + require.Equal( + t, 0, registryRef.terminatedCount(), + "registry was told terminal during sweep reorg recovery", + ) + + // 3. Sweep re-confirms; actor returns to PhaseCompleted. + txconfirmRef.emitConfirmed(t, 2, sweepTxid, 111) + require.Eventually(t, func() bool { + resp, ok := mustAsk( + t, actorInstance.Ref(), &GetStateRequest{}, + ).(*GetStateResp) + require.True(t, ok) + + return resp.Phase == PhaseCompleted + }, testTimeout, 10*time.Millisecond) + require.Equal( + t, 0, registryRef.terminatedCount(), + "registry was told terminal after sweep re-confirm but "+ + "before TxFinalized", + ) + + // 4. Finality. TxFinalizedMsg matches the recorded sweep txid, + // the actor latches sweepFinalized, and the next driveEvent + // fires UnrollTerminatedMsg. + txconfirmRef.emitFinalized(t, 2, sweepTxid) + require.Eventually(t, func() bool { + return registryRef.terminatedCount() == 1 + }, testTimeout, 10*time.Millisecond, + "registry was never told terminal after TxFinalized") +} + +// TestExternalSpendReorgResumesActor observes an external spend of the +// target outpoint while materialization is in flight, then drives a +// reorg of the spending block. The actor must park in +// AwaitingExternalSpendFinality while the spend is provisional, then +// resume normal planning once the spend reorgs out. Without this +// behavior, a transient reorg-out replacement of the target would +// permanently terminate the recovery job. +func TestExternalSpendReorgResumesActor(t *testing.T) { + proof := buildLinearProof(t) + desc := testDescriptor(t, proof.TargetOutpoint(), proof.CSVDelay()) + unrollActor, beh, txconfirmRef, _ := newActorHarness(t, proof, desc) + + chainSource, ok := beh.cfg.ChainSource.(*fakeChainSourceRef) + require.True(t, ok) + + mustAsk(t, unrollActor.Ref(), &StartUnrollRequest{ + Height: 100, + Trigger: TriggerManual, + }) + + rootTxid := proof.RootTxids()[0] + require.Eventually(t, func() bool { + return txconfirmRef.requestCountForTxid(rootTxid) == 1 + }, testTimeout, 10*time.Millisecond, + "root never submitted") + + // Ensure spend watch on the target is registered with the + // reorg / done refs wired before driving any events. + require.Eventually(t, func() bool { + var targetRegistered bool + for _, op := range chainSource.spendRegistrations() { + if op == proof.TargetOutpoint() { + targetRegistered = true + break + } + } + if !targetRegistered { + return false + } + + chainSource.mu.Lock() + defer chainSource.mu.Unlock() + + return chainSource.spendReorgedRef != nil && + chainSource.spendFinalizedRef != nil + }, testTimeout, 10*time.Millisecond) + + // 1. An external party broadcasts a spend of the target outpoint. + externalTxid := chainhash.Hash{0xee} + chainSource.emitSpendForOutpoint( + t, proof.TargetOutpoint(), externalTxid, 110, + ) + + // The actor must park in AwaitingExternalSpendFinality rather + // than transitioning to Failed; the spend is provisional. + require.Eventually(t, func() bool { + resp, ok := mustAsk( + t, unrollActor.Ref(), &GetStateRequest{}, + ).(*GetStateResp) + require.True(t, ok) + + return resp.Phase == PhaseExternalSpendObserved && + resp.FailReason == "" + }, testTimeout, 10*time.Millisecond, + "actor never parked in PhaseExternalSpendObserved") + + // 2. The spending block is reorged out. The actor must clear the + // provisional anchor and resume normal planning. Because the + // proof root is still in-flight (no confirmation has arrived + // yet), the actor goes back to PhaseMaterializing. + chainSource.emitSpendReorged(t) + + require.Eventually(t, func() bool { + resp, ok := mustAsk( + t, unrollActor.Ref(), &GetStateRequest{}, + ).(*GetStateResp) + require.True(t, ok) + + return resp.Phase == PhaseMaterializing && + resp.FailReason == "" + }, testTimeout, 10*time.Millisecond, + "actor did not resume materialization after spend reorg") +} + +// TestSweepConfirmationReorgReversesCompleted drives the full happy path +// to Completed and then reorgs the sweep confirmation out. The actor +// must drop SweepStatusConfirmed back to Broadcasted and clear the +// stored ConfirmHeight so the planner stops reporting Done. +func TestSweepConfirmationReorgReversesCompleted(t *testing.T) { + proof := buildLinearProof(t) + desc := testDescriptor(t, proof.TargetOutpoint(), proof.CSVDelay()) + unrollActor, _, txconfirmRef, _ := newActorHarness(t, proof, desc) + + mustAsk(t, unrollActor.Ref(), &StartUnrollRequest{ + Height: 100, + Trigger: TriggerManual, + }) + + rootTxid := proof.RootTxids()[0] + targetTxid := proof.TargetOutpoint().Hash + + require.Eventually(t, func() bool { + return txconfirmRef.requestCountForTxid(rootTxid) == 1 + }, testTimeout, 10*time.Millisecond) + + txconfirmRef.emitConfirmed(t, 0, rootTxid, 101) + require.Eventually(t, func() bool { + return txconfirmRef.requestCountForTxid(targetTxid) == 1 + }, testTimeout, 10*time.Millisecond) + + txconfirmRef.emitConfirmed(t, 1, targetTxid, 102) + mustAsk(t, unrollActor.Ref(), &HeightObservedMsg{ + Height: 102 + int32(proof.CSVDelay()) + 1, + }) + + require.Eventually(t, func() bool { + return txconfirmRef.requestCount() >= 3 + }, testTimeout, 10*time.Millisecond) + + sweepReq := txconfirmRef.lastRequest(t) + sweepTxid := sweepReq.Tx.TxHash() + + txconfirmRef.emitConfirmed(t, 2, sweepTxid, 110) + require.Eventually(t, func() bool { + resp, ok := mustAsk( + t, unrollActor.Ref(), &GetStateRequest{}, + ).(*GetStateResp) + require.True(t, ok) + + return resp.Phase == PhaseCompleted + }, testTimeout, 10*time.Millisecond) + + // 1. Sweep confirmation reorgs out. The actor must roll back to + // AwaitingSweepConfirmation: the signed sweep is still durable, the + // txconfirm subscription is still live, but the planner stops + // reporting Done. + txconfirmRef.emitReorged(t, 2, sweepTxid) + + require.Eventually(t, func() bool { + resp, ok := mustAsk( + t, unrollActor.Ref(), &GetStateRequest{}, + ).(*GetStateResp) + require.True(t, ok) + + return resp.Phase == PhaseSweepConfirmation && + resp.PlannerState.Sweep.Status == + unrollplan.SweepStatusBroadcasted && + resp.PlannerState.Sweep.ConfirmHeight.IsNone() + }, testTimeout, 10*time.Millisecond, + "sweep reorg did not roll the actor back to "+ + "AwaitingSweepConfirmation") + + // 2. Sweep reconfirms in a different block. The actor must move + // back to Completed. + txconfirmRef.emitConfirmed(t, 2, sweepTxid, 111) + require.Eventually(t, func() bool { + resp, ok := mustAsk( + t, unrollActor.Ref(), &GetStateRequest{}, + ).(*GetStateResp) + require.True(t, ok) + + sweep := resp.PlannerState.Sweep + + return resp.Phase == PhaseCompleted && + sweep.ConfirmHeight.IsSome() && + sweep.ConfirmHeight.UnsafeFromSome() == 111 + }, testTimeout, 10*time.Millisecond, + "sweep did not re-Complete after reconfirm") +} + +// TestOfflineReorgRestartReconcilesBeforeSideEffects bundles the three +// offline-reorg scenarios (target reorged out, sweep reorged out, and +// external-spend reorged out) into one restart and asserts that the +// reconciler runs BEFORE the FSM session is built — i.e. before the +// actor can broadcast a stale sweep or park in +// AwaitingExternalSpendFinality on a vanished spender. +// +// The setup mimics a daemon that committed a checkpoint reflecting: +// +// - root + target proof nodes confirmed +// - a sweep tx broadcast AND confirmed at H=108 +// - an external spender provisionally observed at H=110 +// +// While the daemon was offline, the chain reorged: the target's +// confirming block was orphaned (so the sweep that spent it can no +// longer be confirmed either), and the external spender's block was +// also dropped. The reconciler the actor consults on restart returns +// "root still confirmed" and "everything else absent". +// +// After Resume the actor must end up in the post-reconcile state +// (sweep downgraded to Pending, TargetConfirmHeight cleared, target +// pruned from ConfirmedTxids, ProvisionalExternalSpend cleared) and +// must NOT have re-submitted the stale sweep tx to txconfirm before +// the planner re-derives broadcast intent from the post-reconcile +// PlannerState. The latter is the load-bearing safety invariant: a +// re-broadcast off a stale checkpoint would race a fresh wallet +// pkScript against any future re-mining of the target. +func TestOfflineReorgRestartReconcilesBeforeSideEffects(t *testing.T) { + proof := buildLinearProof(t) + rootTxid := proof.RootTxids()[0] + targetTxid := proof.TargetOutpoint().Hash + staleSweepTxid := chainhash.Hash{0xaa} + staleExternalTxid := chainhash.Hash{0xbb} + + checkpoint := &actorCheckpoint{ + Version: checkpointVersion, + Height: 112, + Started: true, + Trigger: TriggerRestart, + State: unrollplan.State{ + ConfirmedTxids: []chainhash.Hash{ + rootTxid, targetTxid, + }, + TargetConfirmHeight: fn.Some[int32](102), + Sweep: unrollplan.SweepState{ + Status: unrollplan.SweepStatusConfirmed, + Txid: fn.Some(staleSweepTxid), + ConfirmHeight: fn.Some[int32](108), + }, + }, + ProvisionalExternalSpend: fn.Some(ExternalSpendAnchor{ + SpendingTxid: staleExternalTxid, + SpendingHeight: 110, + }), + } + + // Reconciler reports: + // - root still confirmed (so the planner keeps the partial + // proof-graph progress) + // - target absent (offline reorg) + // - stale sweep absent (offline reorg) + // - target outpoint unspent (external spender reorged out) + reconciler := &stubChainReconciler{ + confirmed: map[chainhash.Hash]ConfirmedAnchor{ + rootTxid: { + Txid: rootTxid, + Height: 101, + }, + }, + } + + beh, txconfirmRef, _ := restoreHarness( + t, proof, checkpoint, reconciler, + ) + + actorInstance := actor.NewActor(actor.ActorConfig[Msg, Resp]{ + ID: "reconcile-test", + Behavior: &txExecAdapter{b: beh, ax: newMemExecFor(beh)}, + MailboxSize: 16, + }) + beh.selfRef = actorInstance.TellRef() + actorInstance.Start() + t.Cleanup(actorInstance.Stop) + + mustAsk(t, actorInstance.Ref(), &ResumeUnrollRequest{Height: 112}) + + // The actor must end up in the post-reconcile state: sweep + // downgraded, target-derived height cleared, target pruned from + // ConfirmedTxids, external spend cleared, root preserved. + require.Eventually(t, func() bool { + resp, ok := mustAsk( + t, actorInstance.Ref(), &GetStateRequest{}, + ).(*GetStateResp) + require.True(t, ok) + + for _, h := range resp.PlannerState.ConfirmedTxids { + if h == targetTxid { + return false + } + } + sweep := resp.PlannerState.Sweep + rootPresent := false + for _, h := range resp.PlannerState.ConfirmedTxids { + if h == rootTxid { + rootPresent = true + break + } + } + + return rootPresent && + sweep.Status == unrollplan.SweepStatusPending && + sweep.Txid.IsNone() && + sweep.ConfirmHeight.IsNone() && + resp.PlannerState.TargetConfirmHeight.IsNone() && + resp.Phase == PhaseMaterializing && + resp.FailReason == "" + }, testTimeout, 10*time.Millisecond, + "actor did not reach post-reconcile state on restart") + + // Load-bearing safety invariant: every txconfirm request issued + // after restart must target a tx the post-reconcile PlannerState + // authorizes. In particular, the orphaned sweep txid must never + // be re-submitted — that would replay a sweep against a target + // the chain says no longer exists. + require.Zero( + t, txconfirmRef.requestCountForTxid(staleSweepTxid), + "stale sweep tx was re-submitted to txconfirm before "+ + "reconciliation downgraded it", + ) +} diff --git a/unroll/snapshot.go b/unroll/snapshot.go index 9ba2733af..e5ae92ed2 100644 --- a/unroll/snapshot.go +++ b/unroll/snapshot.go @@ -2,10 +2,13 @@ package unroll import ( "bytes" + "encoding/binary" "fmt" + "github.com/btcsuite/btcd/chainhash/v2" "github.com/btcsuite/btcd/wire/v2" "github.com/lightninglabs/wavelength/unrollplan" + fn "github.com/lightningnetwork/lnd/fn/v2" "github.com/lightningnetwork/lnd/tlv" ) @@ -93,21 +96,90 @@ const ( // checkpointExitPolicyRefRecordType carries the policy-specific // durable-state reference. checkpointExitPolicyRefRecordType tlv.Type = 21 + + // checkpointExternalSpendRecordType is optional; present only when + // the actor has observed an external spend of the target outpoint + // that has not yet been finalized. Payload is a fixed-layout + // 36-byte blob (32-byte SpendingTxid + 4-byte big-endian + // SpendingHeight) so a daemon restarting mid-spend-finality-window + // can rehydrate into AwaitingExternalSpendFinality instead of + // dropping the provisional anchor and broadcasting a sweep on a + // target the chain says no longer exists. + checkpointExternalSpendRecordType tlv.Type = 23 + + // checkpointSweepFinalizedRecordType is optional; present (value 1) + // only when the sweep has finalized past the backend's reorg-safety + // depth. It makes the in-memory sweep-finalized latch durable so a + // PhaseCompleted entry whose terminal handoff was deferred or failed + // does not restart as permanently "provisional completed" (the latch + // gates notifyRegistryIfTerminal). Omitted when false. + checkpointSweepFinalizedRecordType tlv.Type = 25 + + // checkpointExternalSpendFinalizedRecordType is optional; present + // (value 1) when a target spend crossed policy finality. This prevents + // restart from misclassifying an objectively consumed target as a + // recoverable failure. + checkpointExternalSpendFinalizedRecordType tlv.Type = 27 + + // checkpointReliveUnsafeRecordType is optional; present (value 1) + // before a job issues or reissues chain-boundary work. Until + // authoritative negative evidence clears the guard, a later failure + // must hold the VTXO in exit rather than relive it. + checkpointReliveUnsafeRecordType tlv.Type = 29 ) +// externalSpendBlobSize is the canonical wire size of the persisted +// ProvisionalExternalSpend payload: 32-byte txid + 4-byte big-endian +// height. +const externalSpendBlobSize = chainhash.HashSize + 4 + // actorCheckpoint is the durable checkpoint shape for one VTXO unroll actor. type actorCheckpoint struct { - Version uint8 - Height int32 - Started bool - Trigger StartTrigger - State unrollplan.State - ExitPolicyKind ExitPolicyKind - ExitPolicyRef string - SweepTx *wire.MsgTx - Fail string - SweepAttempts int - DeferredCheckpoints []DeferredCheckpoint + Version uint8 + Height int32 + Started bool + Trigger StartTrigger + State unrollplan.State + ExitPolicyKind ExitPolicyKind + ExitPolicyRef string + SweepTx *wire.MsgTx + Fail string + SweepAttempts int + DeferredCheckpoints []DeferredCheckpoint + ProvisionalExternalSpend fn.Option[ExternalSpendAnchor] + SweepFinalized bool + ExternalSpendFinalized bool + ReliveUnsafe bool +} + +// encodeExternalSpendBlob serializes an ExternalSpendAnchor into the +// fixed-layout payload carried by checkpointExternalSpendRecordType. +func encodeExternalSpendBlob(anchor ExternalSpendAnchor) []byte { + blob := make([]byte, externalSpendBlobSize) + copy(blob[:chainhash.HashSize], anchor.SpendingTxid[:]) + binary.BigEndian.PutUint32( + blob[chainhash.HashSize:], uint32(anchor.SpendingHeight), + ) + + return blob +} + +// decodeExternalSpendBlob parses a fixed-layout external-spend payload +// back into an ExternalSpendAnchor. +func decodeExternalSpendBlob(blob []byte) (ExternalSpendAnchor, error) { + if len(blob) != externalSpendBlobSize { + return ExternalSpendAnchor{}, fmt.Errorf("external spend blob "+ + "has %d bytes, want %d", len(blob), + externalSpendBlobSize) + } + + var anchor ExternalSpendAnchor + copy(anchor.SpendingTxid[:], blob[:chainhash.HashSize]) + anchor.SpendingHeight = int32( + binary.BigEndian.Uint32(blob[chainhash.HashSize:]), + ) + + return anchor, nil } // encodeCheckpoint serializes one actor checkpoint into canonical TLV @@ -226,6 +298,46 @@ func encodeCheckpoint(value *actorCheckpoint) ([]byte, error) { ) } + value.ProvisionalExternalSpend.WhenSome( + func(anchor ExternalSpendAnchor) { + blob := encodeExternalSpendBlob(anchor) + records = append( + records, tlv.MakePrimitiveRecord( + checkpointExternalSpendRecordType, + &blob, + ), + ) + }, + ) + + if value.SweepFinalized { + finalized := uint8(1) + records = append( + records, tlv.MakePrimitiveRecord( + checkpointSweepFinalizedRecordType, &finalized, + ), + ) + } + + if value.ExternalSpendFinalized { + finalized := uint8(1) + records = append( + records, tlv.MakePrimitiveRecord( + checkpointExternalSpendFinalizedRecordType, + &finalized, + ), + ) + } + + if value.ReliveUnsafe { + unsafe := uint8(1) + records = append( + records, tlv.MakePrimitiveRecord( + checkpointReliveUnsafeRecordType, &unsafe, + ), + ) + } + stream, err := tlv.NewStream(records...) if err != nil { return nil, fmt.Errorf("create checkpoint stream: %w", err) @@ -255,17 +367,21 @@ func encodeCheckpoint(value *actorCheckpoint) ([]byte, error) { // truncated state. func decodeCheckpoint(raw []byte) (*actorCheckpoint, error) { var ( - version uint8 - height uint32 - started uint8 - trigger uint32 - stateBytes []byte - sweepBytes []byte - failBytes []byte - attempts uint32 - deferredBytes []byte - policyKind []byte - policyRef []byte + version uint8 + height uint32 + started uint8 + trigger uint32 + stateBytes []byte + sweepBytes []byte + failBytes []byte + attempts uint32 + deferredBytes []byte + policyKind []byte + policyRef []byte + extSpendBlob []byte + sweepFinalized uint8 + extFinalized uint8 + reliveUnsafe uint8 ) stream, err := tlv.NewStream( @@ -302,6 +418,19 @@ func decodeCheckpoint(raw []byte) (*actorCheckpoint, error) { tlv.MakePrimitiveRecord( checkpointExitPolicyRefRecordType, &policyRef, ), + tlv.MakePrimitiveRecord( + checkpointExternalSpendRecordType, &extSpendBlob, + ), + tlv.MakePrimitiveRecord( + checkpointSweepFinalizedRecordType, &sweepFinalized, + ), + tlv.MakePrimitiveRecord( + checkpointExternalSpendFinalizedRecordType, + &extFinalized, + ), + tlv.MakePrimitiveRecord( + checkpointReliveUnsafeRecordType, &reliveUnsafe, + ), ) if err != nil { return nil, fmt.Errorf("create checkpoint stream: %w", err) @@ -367,6 +496,26 @@ func decodeCheckpoint(raw []byte) (*actorCheckpoint, error) { checkpoint.DeferredCheckpoints = checkpoints } + if _, ok := parsed[checkpointExternalSpendRecordType]; ok { + anchor, err := decodeExternalSpendBlob(extSpendBlob) + if err != nil { + return nil, fmt.Errorf("decode external spend: %w", err) + } + checkpoint.ProvisionalExternalSpend = fn.Some(anchor) + } + + if _, ok := parsed[checkpointSweepFinalizedRecordType]; ok { + checkpoint.SweepFinalized = sweepFinalized != 0 + } + + if _, ok := parsed[checkpointExternalSpendFinalizedRecordType]; ok { + checkpoint.ExternalSpendFinalized = extFinalized != 0 + } + + if _, ok := parsed[checkpointReliveUnsafeRecordType]; ok { + checkpoint.ReliveUnsafe = reliveUnsafe != 0 + } + return checkpoint, nil } diff --git a/unroll/snapshot_test.go b/unroll/snapshot_test.go index 33c40a62f..964abece6 100644 --- a/unroll/snapshot_test.go +++ b/unroll/snapshot_test.go @@ -81,6 +81,28 @@ func TestCheckpointCodecRoundTripHandcrafted(t *testing.T) { SweepAttempts: 1, }, }, + { + name: "sweep_finalized", + checkpoint: &actorCheckpoint{ + Version: checkpointVersion, + Height: 210, + Started: true, + Trigger: TriggerManual, + State: unrollplan.State{ + ConfirmedTxids: []chainhash.Hash{ + targetTxid, + }, + Sweep: unrollplan.SweepState{ + Status: unrollplan. + SweepStatusBroadcasted, + Txid: fn.Some(sweepTxid), + }, + }, + SweepTx: sweepTx, + SweepAttempts: 1, + SweepFinalized: true, + }, + }, { name: "failed", checkpoint: &actorCheckpoint{ @@ -111,6 +133,51 @@ func TestCheckpointCodecRoundTripHandcrafted(t *testing.T) { }}, }, }, + { + name: "provisional_external_spend", + checkpoint: &actorCheckpoint{ + Version: checkpointVersion, + Height: 155, + Started: true, + Trigger: TriggerManual, + State: unrollplan.State{ + ConfirmedTxids: []chainhash.Hash{ + targetTxid, + }, + TargetConfirmHeight: fn.Some[int32]( + 154, + ), + }, + ProvisionalExternalSpend: fn.Some( + ExternalSpendAnchor{ + SpendingTxid: hashFromByteCk( + 0xee, + ), + SpendingHeight: 155, + }, + ), + }, + }, + { + name: "finalized_external_spend", + checkpoint: &actorCheckpoint{ + Version: checkpointVersion, + Height: 185, + Started: true, + Trigger: TriggerManual, + ExternalSpendFinalized: true, + }, + }, + { + name: "restart_relive_unsafe", + checkpoint: &actorCheckpoint{ + Version: checkpointVersion, + Height: 190, + Started: true, + Trigger: TriggerManual, + ReliveUnsafe: true, + }, + }, } for _, tc := range cases { @@ -573,6 +640,10 @@ func requireCheckpointEqual(t *testing.T, want, got *actorCheckpoint) { require.Equal(t, want.Trigger, got.Trigger) require.Equal(t, want.Fail, got.Fail) require.Equal(t, want.SweepAttempts, got.SweepAttempts) + require.Equal( + t, want.ExternalSpendFinalized, got.ExternalSpendFinalized, + ) + require.Equal(t, want.ReliveUnsafe, got.ReliveUnsafe) require.ElementsMatch( t, want.DeferredCheckpoints, got.DeferredCheckpoints, ) @@ -597,6 +668,16 @@ func requireCheckpointEqual(t *testing.T, want, got *actorCheckpoint) { t, txsEqualCk(want.SweepTx, got.SweepTx), "sweep transactions differ", ) + require.Equal( + t, want.ProvisionalExternalSpend.IsSome(), + got.ProvisionalExternalSpend.IsSome(), + ) + if want.ProvisionalExternalSpend.IsSome() { + require.Equal( + t, want.ProvisionalExternalSpend.UnsafeFromSome(), + got.ProvisionalExternalSpend.UnsafeFromSome(), + ) + } } // hashFromByteCk builds a chainhash.Hash with the first byte set to b, useful diff --git a/unroll/state_snapshot.go b/unroll/state_snapshot.go index d0693ba6c..b8e14aad7 100644 --- a/unroll/state_snapshot.go +++ b/unroll/state_snapshot.go @@ -40,20 +40,21 @@ func checkpointFromState(state State, sweepTx *wire.MsgTx) *actorCheckpoint { } checkpoint.Fail = job.FailReason checkpoint.SweepAttempts = job.SweepAttempts + checkpoint.ProvisionalExternalSpend = job.ProvisionalExternalSpend + checkpoint.ExternalSpendFinalized = job.ExternalSpendFinalized + checkpoint.ReliveUnsafe = job.ReliveUnsafe return checkpoint } -// jobHadOnChainFootprint reports whether the job ever published anything -// on-chain. A footprint exists if any proof node confirmed or is still -// in-flight (submitted to txconfirm, so potentially in the mempool), or if -// the sweep advanced past pending. It is false only for a clean failure -// that never broadcast, which is the sole case where the target VTXO is -// safe to roll back to live: any footprint means the unilateral exit has -// begun on-chain and the operator no longer treats the VTXO as live. See -// wavelength#602. +// jobHadOnChainFootprint reports whether the job published anything on-chain +// or observed a finalized external spend. A false result means only that the +// local planner recorded no footprint; the independent ReliveUnsafe guard must +// also be cleared by objective canonical-absence evidence before the VTXO may +// return to live. See wavelength#602. // -// This reflects THIS job's own footprint. An exit driven on-chain by a +// Except for ExternalSpendFinalized, this reflects THIS job's own footprint. +// An exit driven on-chain by a // third party (the operator, or a prior holder of a fraudulently re-spent // VTXO) does not reach the recoverable branch: our submission of the same // proof node is an ignorable "already known" broadcast rather than a hard @@ -66,9 +67,13 @@ func jobHadOnChainFootprint(job *JobState) bool { return false } + sweepPublished := job.PlannerState.Sweep.Status != + unrollplan.SweepStatusPending + return len(job.PlannerState.ConfirmedTxids) > 0 || len(job.PlannerState.InFlightTxids) > 0 || - job.PlannerState.Sweep.Status != unrollplan.SweepStatusPending + sweepPublished || + job.ExternalSpendFinalized } // effectiveSweepTxid returns the durable sweep txid from planner state when @@ -102,14 +107,19 @@ func stateFromCheckpoint(checkpoint *actorCheckpoint) State { deferred := copyDeferredCheckpoints(checkpoint.DeferredCheckpoints) job := &JobState{ - Height: checkpoint.Height, - Trigger: checkpoint.Trigger, - ExitPolicyKind: exitPolicyKind(checkpoint.ExitPolicyKind), - ExitPolicyRef: checkpoint.ExitPolicyRef, - PlannerState: copyPlannerState(checkpoint.State), - DeferredCheckpoints: deferred, - FailReason: checkpoint.Fail, - SweepAttempts: checkpoint.SweepAttempts, + Height: checkpoint.Height, + Trigger: checkpoint.Trigger, + ExitPolicyKind: exitPolicyKind( + checkpoint.ExitPolicyKind, + ), + ExitPolicyRef: checkpoint.ExitPolicyRef, + PlannerState: copyPlannerState(checkpoint.State), + DeferredCheckpoints: deferred, + FailReason: checkpoint.Fail, + SweepAttempts: checkpoint.SweepAttempts, + ProvisionalExternalSpend: checkpoint.ProvisionalExternalSpend, + ExternalSpendFinalized: checkpoint.ExternalSpendFinalized, + ReliveUnsafe: checkpoint.ReliveUnsafe, } switch phaseFromPlannerState(job) { @@ -119,6 +129,9 @@ func stateFromCheckpoint(checkpoint *actorCheckpoint) State { case PhaseFailed: return &Failed{Job: job} + case PhaseExternalSpendObserved: + return &AwaitingExternalSpendFinality{Job: job} + case PhaseSweepConfirmation: return &AwaitingSweepConfirmation{Job: job} @@ -152,6 +165,9 @@ func phaseFromState(state State) Phase { case *AwaitingSweepConfirmation: return PhaseSweepConfirmation + case *AwaitingExternalSpendFinality: + return PhaseExternalSpendObserved + case *Completed: return PhaseCompleted @@ -174,6 +190,19 @@ func phaseFromPlannerState(job *JobState) Phase { return PhaseFailed } + if job.ExternalSpendFinalized { + return PhaseCompleted + } + + // A persisted provisional external spend takes precedence over the + // sweep-based phase derivation: the actor was parked waiting for either + // a reorg (which clears the anchor) or finality (which resolves it as + // permanently consumed). Surfacing this phase keeps restart + // reconciliation and the live reducer aligned on the same state. + if job.ProvisionalExternalSpend.IsSome() { + return PhaseExternalSpendObserved + } + switch { case job.PlannerState.Sweep.Status == unrollplan.SweepStatusConfirmed: return PhaseCompleted @@ -207,6 +236,9 @@ func stateJob(state State) *JobState { case *AwaitingSweepConfirmation: return s.Job.Copy() + case *AwaitingExternalSpendFinality: + return s.Job.Copy() + case *Completed: return s.Job.Copy() diff --git a/unroll/state_snapshot_test.go b/unroll/state_snapshot_test.go index aad87a4f6..004681327 100644 --- a/unroll/state_snapshot_test.go +++ b/unroll/state_snapshot_test.go @@ -87,6 +87,17 @@ func TestCheckpointRoundTripByPhase(t *testing.T) { }, typ: &Completed{}, }, + { + name: "external_spend_completed", + state: &Completed{ + Job: &JobState{ + Height: 106, + Trigger: TriggerRestart, + ExternalSpendFinalized: true, + }, + }, + typ: &Completed{}, + }, { name: "failed", state: &Failed{ @@ -145,6 +156,14 @@ func TestCheckpointRoundTripByPhase(t *testing.T) { t, stateJob(tc.state).DeferredCheckpoints, stateJob(restored).DeferredCheckpoints, ) + require.Equal( + t, stateJob(tc.state).ExternalSpendFinalized, + stateJob(restored).ExternalSpendFinalized, + ) + require.Equal( + t, stateJob(tc.state).ReliveUnsafe, + stateJob(restored).ReliveUnsafe, + ) }) } } diff --git a/vhtlcrecovery/coordinator/service.go b/vhtlcrecovery/coordinator/service.go index 5560e41aa..91cd4502e 100644 --- a/vhtlcrecovery/coordinator/service.go +++ b/vhtlcrecovery/coordinator/service.go @@ -565,7 +565,13 @@ func (s *Service) reconcileLoaded(ctx context.Context, switch status.UnrollPhase { case unroll.PhasePending, unroll.PhaseMaterializing, unroll.PhaseCSVPending, unroll.PhaseSweepBroadcast, - unroll.PhaseSweepConfirmation: + unroll.PhaseSweepConfirmation, + unroll.PhaseExternalSpendObserved: + // PhaseExternalSpendObserved is a parked, non-terminal state: + // the unroll actor observed an unfinalized external spend of + // the target and is holding (a reorg can resurrect it), so the + // recovery job likewise holds its current state until the + // unroll resolves one way or the other. return status, nil case unroll.PhaseCompleted: diff --git a/waved/rpc_server.go b/waved/rpc_server.go index 5bae39408..59f517eb5 100644 --- a/waved/rpc_server.go +++ b/waved/rpc_server.go @@ -5610,6 +5610,19 @@ func unrollPhaseToProto(phase unroll.Phase) waverpc.UnrollJobStatus { case unroll.PhaseFailed: return waverpc.UnrollJobStatus_UNROLL_JOB_STATUS_FAILED + case unroll.PhaseMaterializing: + return waverpc.UnrollJobStatus_UNROLL_JOB_STATUS_MATERIALIZING + + // PhaseExternalSpendObserved is deliberately reported as + // MATERIALIZING: it is a transient, reorg-reversible parked phase (an + // external spend was seen but has not finalized) with no dedicated + // proto status, and the true phase is always reconstructed from the + // durable checkpoint on restore. The collapse only coarsens the + // operator-facing status during the parked window; add a dedicated + // proto value if that window needs to be independently observable. + case unroll.PhaseExternalSpendObserved: + return waverpc.UnrollJobStatus_UNROLL_JOB_STATUS_MATERIALIZING + default: return waverpc.UnrollJobStatus_UNROLL_JOB_STATUS_MATERIALIZING } From d183a1917f6679704dd3dc48d251b6c53e2cdb11 Mon Sep 17 00:00:00 2001 From: Elle Mouton Date: Thu, 16 Jul 2026 10:49:18 -0700 Subject: [PATCH 2/4] wallet: Make boarding sweeps reorg recoverable Boarding sweeps previously collapsed the first confirmation or spend notification into terminal wallet state. If that evidence was later reorged out, the actor could no longer distinguish a canonical spend from a provisional one and could restore or retire inputs incorrectly. Keep the sweep confirmation and a separate spend watch for every consumed boarding input active until the chain source reports policy finality. Record positive observations provisionally, remove them on Reorged, and finalize each consumed input durably so a later sweep-level failure cannot make already spent value available again. Treat a broadcast failure as an uncertain attempt instead of evidence that no transaction reached the chain. Recovery remains bounded and tip-paced, and a restart reconstructs the outstanding watches from persisted actor state. Add focused actor and store coverage plus a real LND/Bitcoin system test that drives a boarding sweep through confirmed, reorged, and reconfirmed states. --- db/boarding_sweep_store.go | 59 +++- db/boarding_wallet_test.go | 109 ++++++ systest/boarding_sweep_reorg_test.go | 307 ++++++++++++++++ wallet/boarding_sweep_actor.go | 501 ++++++++++++++++++--------- wallet/boarding_sweep_actor_test.go | 490 +++++++++++++++++++++++--- 5 files changed, 1240 insertions(+), 226 deletions(-) create mode 100644 systest/boarding_sweep_reorg_test.go diff --git a/db/boarding_sweep_store.go b/db/boarding_sweep_store.go index dad9f5b5f..5b40d48b0 100644 --- a/db/boarding_sweep_store.go +++ b/db/boarding_sweep_store.go @@ -174,12 +174,36 @@ func (b *BoardingWalletStore) MarkBoardingSweepFailed(ctx context.Context, now := b.clock.Now().Unix() return b.db.ExecTx(ctx, WriteTxOption(), func(q BoardingStore) error { + sweep, err := q.GetBoardingSweep(ctx, txid[:]) + if err != nil { + return fmt.Errorf("get sweep: %w", err) + } + if sweep.Status != sweepStatusPending && + sweep.Status != sweepStatusPublished { + + // Terminal evidence is monotonic. A duplicate or stale + // failure notification must never downgrade a + // confirmed/external sweep or release any of the value + // it consumed. + return nil + } + inputs, err := q.ListBoardingSweepInputs(ctx, txid[:]) if err != nil { return fmt.Errorf("list sweep inputs: %w", err) } for _, input := range inputs { + if input.Status != inputStatusPending && + input.Status != inputStatusPublished { + + // A terminal spend is objective chain evidence. + // A later local broadcaster failure may release + // the other inputs, but must never restore this + // intent to a spendable state. + continue + } + err = q.UpdateBoardingIntentStatus( ctx, sqlc.UpdateBoardingIntentStatusParams{ OutpointHash: input.OutpointHash, @@ -374,6 +398,22 @@ func (b *BoardingWalletStore) MarkBoardingSweepInputSpent(ctx context.Context, return sql.ErrNoRows } + // Finality for one input is sufficient to make that specific + // boarding intent permanently spent, even while sibling inputs + // are still unresolved. This prevents a later sweep-level + // failure from restoring an objectively consumed intent. + err = q.UpdateBoardingIntentStatus( + ctx, sqlc.UpdateBoardingIntentStatusParams{ + OutpointHash: outpoint.Hash[:], + OutpointIndex: int32(outpoint.Index), + Status: "swept", + LastUpdateTime: now, + }, + ) + if err != nil { + return fmt.Errorf("mark intent swept: %w", err) + } + count, err := q.CountUnresolvedBoardingSweepInputs( ctx, sweepTxid[:], ) @@ -389,24 +429,11 @@ func (b *BoardingWalletStore) MarkBoardingSweepInputSpent(ctx context.Context, if err != nil { return fmt.Errorf("list sweep inputs: %w", err) } - for _, input := range inputs { - err = q.UpdateBoardingIntentStatus( - ctx, sqlc.UpdateBoardingIntentStatusParams{ - OutpointHash: input.OutpointHash, - OutpointIndex: input.OutpointIndex, - Status: "swept", - LastUpdateTime: now, - }, - ) - if err != nil { - return fmt.Errorf("mark intent swept: %w", err) - } - } - sweepStatus := sweepStatusExternalResolved + sweepStatus := sweepStatusConfirmed for _, input := range inputs { - if input.Status == inputStatusSpent { - sweepStatus = sweepStatusConfirmed + if input.Status != inputStatusSpent { + sweepStatus = sweepStatusExternalResolved break } } diff --git a/db/boarding_wallet_test.go b/db/boarding_wallet_test.go index 3ec512bc1..2e8651b10 100644 --- a/db/boarding_wallet_test.go +++ b/db/boarding_wallet_test.go @@ -2,6 +2,7 @@ package db import ( "database/sql" + "errors" "testing" btcaddr "github.com/btcsuite/btcd/address/v2" @@ -822,6 +823,94 @@ func TestBoardingSweepFailedRestoresIntent(t *testing.T) { require.Equal(t, wallet.BoardingStatusConfirmed, updated.Status) } +// TestBoardingSweepFailedPreservesTerminalInput verifies that a local +// sweep-level failure restores only unresolved siblings. An input with a +// final external spend must remain unavailable forever. +func TestBoardingSweepFailedPreservesTerminalInput(t *testing.T) { + t.Parallel() + + ctx := t.Context() + store, _ := newBoardingStoreForTest(t) + terminal := createSweepStoreIntentWithSeed(t, store, 31) + unresolved := createSweepStoreIntentWithSeed(t, store, 32) + + sweepTx := wire.NewMsgTx(2) + sweepTx.AddTxIn(&wire.TxIn{ + PreviousOutPoint: terminal.Outpoint, + }) + sweepTx.AddTxIn(&wire.TxIn{ + PreviousOutPoint: unresolved.Outpoint, + }) + sweepTx.AddTxOut(&wire.TxOut{ + Value: 19_000, + PkScript: []byte{txscript.OP_TRUE}, + }) + sweepTxid := sweepTx.TxHash() + + err := store.CreatePendingBoardingSweep(ctx, wallet.NewBoardingSweep{ + Tx: sweepTx, + TotalAmount: 20_000, + FeeAmount: 1_000, + VBytes: 300, + Inputs: []wallet.NewBoardingSweepInput{ + { + Outpoint: terminal.Outpoint, + Amount: terminal.ChainInfo.Amount, + PreviousStatus: terminal.Status, + }, + { + Outpoint: unresolved.Outpoint, + Amount: unresolved.ChainInfo.Amount, + PreviousStatus: unresolved.Status, + }, + }, + }) + require.NoError(t, err) + + externalTxid := chainhash.Hash{0xfa} + resolved, err := store.MarkBoardingSweepInputSpent( + ctx, terminal.Outpoint, externalTxid, 333, + ) + require.NoError(t, err) + require.False(t, resolved) + + terminalIntent, err := store.GetIntent(ctx, terminal.Outpoint) + require.NoError(t, err) + require.Equal(t, wallet.BoardingStatusSwept, terminalIntent.Status) + + err = store.MarkBoardingSweepFailed( + ctx, sweepTxid, errors.New("conflicting input"), + ) + require.NoError(t, err) + + terminalIntent, err = store.GetIntent(ctx, terminal.Outpoint) + require.NoError(t, err) + require.Equal(t, wallet.BoardingStatusSwept, terminalIntent.Status) + + unresolvedIntent, err := store.GetIntent(ctx, unresolved.Outpoint) + require.NoError(t, err) + require.Equal( + t, wallet.BoardingStatusConfirmed, unresolvedIntent.Status, + ) + + record, err := store.GetBoardingSweep(ctx, sweepTxid) + require.NoError(t, err) + require.NotNil(t, record) + require.Equal(t, wallet.BoardingSweepStatusFailed, record.Status) + statusByOutpoint := make(map[wire.OutPoint]string, len(record.Inputs)) + for _, input := range record.Inputs { + statusByOutpoint[input.Outpoint] = input.Status + } + require.Equal( + t, wallet.BoardingSweepInputStatusExternalSpent, + statusByOutpoint[terminal.Outpoint], + ) + require.Equal( + t, wallet.BoardingSweepInputStatusFailed, + statusByOutpoint[unresolved.Outpoint], + ) +} + // TestBoardingSweepExternalSpendResolvesSeparately verifies an externally // spent sweep input does not make the aggregate look confirmed by our tx. func TestBoardingSweepExternalSpendResolvesSeparately(t *testing.T) { @@ -880,6 +969,26 @@ func TestBoardingSweepExternalSpendResolvesSeparately(t *testing.T) { ) require.NoError(t, err) require.Empty(t, confirmed) + + // A stale broadcaster failure cannot overwrite the objective terminal + // spend or restore the consumed boarding intent. + err = store.MarkBoardingSweepFailed( + ctx, sweepTxid, errors.New("late broadcaster failure"), + ) + require.NoError(t, err) + + record, err := store.GetBoardingSweep(ctx, sweepTxid) + require.NoError(t, err) + require.Equal( + t, wallet.BoardingSweepStatusExternalResolved, record.Status, + ) + require.Equal( + t, wallet.BoardingSweepInputStatusExternalSpent, + dbSweepInputStatus(t, record.Inputs), + ) + updatedIntent, err := store.GetIntent(ctx, intent.Outpoint) + require.NoError(t, err) + require.Equal(t, wallet.BoardingStatusSwept, updatedIntent.Status) } // TestActiveBoardingSweepInputUnique verifies the DB enforces that one diff --git a/systest/boarding_sweep_reorg_test.go b/systest/boarding_sweep_reorg_test.go new file mode 100644 index 000000000..7c8fe56d5 --- /dev/null +++ b/systest/boarding_sweep_reorg_test.go @@ -0,0 +1,307 @@ +//go:build systest + +package systest + +import ( + "context" + "crypto/sha256" + "testing" + "time" + + btcaddr "github.com/btcsuite/btcd/address/v2" + "github.com/btcsuite/btcd/btcutil/v2" + "github.com/btcsuite/btcd/chaincfg/v2" + "github.com/btcsuite/btcd/txscript/v2" + "github.com/lightninglabs/wavelength/baselib/actor" + "github.com/lightninglabs/wavelength/txconfirm" + "github.com/lightninglabs/wavelength/wallet" + "github.com/stretchr/testify/require" +) + +// recordingBoardingSweepRef collects the BoardingSweepTxNotification +// values that txconfirm's lifecycle, fanned through the boarding +// sweep MapNotification, delivers to the actor's mailbox. The test +// uses this to assert that each chainsource event maps to the right +// BoardingSweepTxStatus rather than collapsing reorgs into a failure. +type recordingBoardingSweepRef struct { + id string + msgs chan wallet.BoardingSweepTxNotification +} + +func newRecordingBoardingSweepRef(id string) *recordingBoardingSweepRef { + return &recordingBoardingSweepRef{ + id: id, + msgs: make(chan wallet.BoardingSweepTxNotification, 16), + } +} + +// ID returns the subscriber identifier. +func (r *recordingBoardingSweepRef) ID() string { + return r.id +} + +// Tell records the inbound boarding-sweep notification on the channel +// for the test to consume. +func (r *recordingBoardingSweepRef) Tell(_ context.Context, + msg wallet.WalletMsg) error { + + notif, ok := msg.(wallet.BoardingSweepTxNotification) + if !ok { + + // Not a notification we care about (and there should be + // no others on this subscriber path). + return nil + } + + r.msgs <- notif + + return nil +} + +// TryTell mirrors Tell for the non-blocking TellOnlyRef contract; the +// recording ref never applies backpressure, so it simply delegates. +func (r *recordingBoardingSweepRef) TryTell(ctx context.Context, + msg wallet.WalletMsg) error { + + return r.Tell(ctx, msg) +} + +// await pulls the next boarding-sweep notification or fails the test +// on timeout. +func (r *recordingBoardingSweepRef) await( + t *testing.T) wallet.BoardingSweepTxNotification { + + t.Helper() + + select { + case msg := <-r.msgs: + return msg + + case <-time.After(txConfirmSystestEventTimeout): + t.Fatalf("timeout waiting for boarding-sweep notification (%s)", + txConfirmSystestEventTimeout) + + return wallet.BoardingSweepTxNotification{} + } +} + +// TestBoardingSweepReorgRoundTrip exercises the boarding sweep +// MapNotification under a real bitcoind reorg, end-to-end through the +// full txconfirm pipeline. It is the boarding-sweep complement of +// TestTxConfirmReorgRoundTrip: the same chain-event substrate, but +// asserting on the wallet-level lifecycle statuses that drive +// handleSweepTxNotification rather than the raw txconfirm +// notifications. +// +// Lifecycle asserted: +// +// BoardingSweepTxStatusConfirmed (first conf landed) +// BoardingSweepTxStatusReorged (conf block disconnected) +// BoardingSweepTxStatusConfirmed (re-confirmation on canonical chain) +// +// TxFinalized is exercised by TestTxConfirmReorgRoundTrip — that test +// proves the chainsource finality-depth synthesizer fires after the +// reorg-safety horizon, which is the same wire that would deliver +// BoardingSweepTxStatusFinalized to the wallet handler. Including a +// dedicated Finalized assertion here would require mining to the +// 31-confirmation terminal boundary without exercising any +// boarding-sweep-specific code +// path that the unit tests in +// wallet/boarding_sweep_actor_test.go::TestSweepTxNotificationFinalizedIsBenign +// do not already cover. +// +// What this test specifically pins: +// +// - The boarding-sweep MapNotification correctly classifies a real +// chainsource Confirmed event as +// BoardingSweepTxStatusConfirmed (not Failed). +// - The boarding-sweep MapNotification correctly classifies a real +// chainsource Reorged event as +// BoardingSweepTxStatusReorged (not Failed). This is the load- +// bearing regression the wallet refactor was designed to prevent. +// - The handler is multi-shot: txconfirm keeps the watch armed +// past the first TxConfirmed, so a re-confirmation on the +// canonical chain re-fires +// BoardingSweepTxStatusConfirmed. +// - BlockHeight is preserved across the wire on each Confirmed +// event. +func TestBoardingSweepReorgRoundTrip(t *testing.T) { + ParallelN(t) + + h := NewSysTestHarness(t) + ctx := h.Context() + + chainSource := h.NewChainSourceActor() + + // Spawn a real txconfirm actor over the real chainsource. Wallet + // is nil because the systest tx is non-anchor; CPFP fee-input + // selection is never triggered. + txconfBehavior := txconfirm.NewTxBroadcasterActor(txconfirm.Config{ + ChainSource: chainSource, + }) + txconfInstance := actor.NewActor(actor.ActorConfig[ + txconfirm.Msg, txconfirm.Resp, + ]{ + ID: "txconfirm-boarding-sweep-reorg", + Behavior: txconfBehavior, + MailboxSize: 64, + }) + txconfBehavior.SetSelfRef(txconfInstance.TellRef()) + txconfInstance.Start() + t.Cleanup(txconfInstance.Stop) + + // Synthetic watched pkScript — same trick as + // TestTxConfirmReorgRoundTrip. + pubKeyHash := sha256.Sum256([]byte(t.Name())) + addr, err := btcaddr.NewAddressWitnessPubKeyHash( + pubKeyHash[:20], &chaincfg.RegressionNetParams, + ) + require.NoError(t, err, "build synthetic P2WPKH address") + pkScript, err := txscript.PayToAddrScript(addr) + require.NoError(t, err, "derive pkScript for synthetic address") + + heightHint := h.Harness.BlockCount() + + signedTx := h.Harness.SignedV3Tx( + pkScript, btcutil.Amount(btcutil.SatoshiPerBitcoin/100), + ) + txidVal := signedTx.TxHash() + txid := &txidVal + t.Logf("constructed v3 tx: txid=%s", txid) + + // Build the subscriber chain that the production code uses: + // + // walletNotif (TellOnlyRef[BoardingSweepTxNotification], wrap + // via MapMessage so it can sit on a WalletMsg pipe) + // -> wallet.MapBoardingSweepNotification (the production + // classifier from boarding_sweep_actor.go) hosted on a + // txconfirm subscriber. + // + // The recording ref stands in for the wallet actor's mailbox; the + // MapNotification function is the same one the production + // submitSweepConfirmer wires up. + walletRecorder := newRecordingBoardingSweepRef( + "boarding-sweep-systest-sub", + ) + subscriber := wallet.NewBoardingSweepTxconfirmSubscriber( + walletRecorder, + ) + + // Register the EnsureConfirmedReq BEFORE mining so we exercise + // the live-detection path. + _, err = txconfInstance.Ref().Ask( + ctx, &txconfirm.EnsureConfirmedReq{ + Tx: signedTx, + ConfirmationPkScript: pkScript, + Label: "systest-boarding-sweep-reorg", + HeightHint: heightHint, + TargetConfs: 1, + Subscriber: subscriber, + }, + ).Await(ctx).Unpack() + require.NoError(t, err, "EnsureConfirmedReq failed") + + // 1. Mine the block that confirms the systest tx. + originalBlocks := h.Harness.Generate(1) + require.Len(t, originalBlocks, 1) + originalBlock := originalBlocks[0] + + // 2. Expect Status=Confirmed at the original block height with + // at least one confirmation. NumConfs being non-zero pins the + // classifier's assignment of ev.NumConfs through the + // txconfirm.TxConfirmed -> BoardingSweepTxNotification mapping; + // a regression that zeroed it (e.g. by accidentally pulling + // from the wrong field on the event) would otherwise be silently + // invisible to consumers that read NumConfs to gate on + // confirmation depth. + first := walletRecorder.await(t) + require.Equal( + t, wallet.BoardingSweepTxStatusConfirmed, first.Status, "fir"+ + "st notification must be Confirmed, got status=%d", + first.Status, + ) + require.Equal(t, *txid, first.Txid) + require.Equal( + t, int32(originalBlock.Height), first.BlockHeight, + "first Confirmed BlockHeight should match the mined block", + ) + require.GreaterOrEqual( + t, first.NumConfs, uint32(1), + "first Confirmed NumConfs must be at least the target (1)", + ) + t.Logf( + "first Confirmed: txid=%s height=%d num_confs=%d", first.Txid, + first.BlockHeight, first.NumConfs, + ) + + // 3. Reorg the conf block out, mine a strictly longer replacement + // branch, wait for lnd to chain-sync. + reorg := h.Harness.Reorg(1, 2) + require.Equal( + t, originalBlock.Hash, reorg.Disconnected[0].Hash, + "the reorg should have disconnected the conf block", + ) + require.Len(t, reorg.Connected, 2) + t.Logf( + "reorg: disconnected=%d connected=%d fork=%d", + len(reorg.Disconnected), len(reorg.Connected), + reorg.ForkPoint.Height, + ) + + // 4. Expect Status=Reorged. The load-bearing assertion: a real + // chainsource Reorged event must NOT classify as Failed (which + // would trigger MarkBoardingSweepFailed in the production + // handler). + second := walletRecorder.await(t) + require.Equal( + t, wallet.BoardingSweepTxStatusReorged, second.Status, "seco"+ + "nd notification must be Reorged (NOT Failed); got "+ + "status=%d. A Reorged-as-Failed regression is "+ + "exactly what this systest exists to catch", + second.Status, + ) + require.Equal(t, *txid, second.Txid) + require.Empty( + t, second.Reason, "Reorged must not carry a failure reason", + ) + t.Logf("Reorged: txid=%s", second.Txid) + + // 5. Expect a fresh Status=Confirmed on the canonical chain. The + // re-confirmation must land in one of the new connected blocks + // (NumConfs is also asserted to pin the field-through invariant + // under the re-confirmation path — the classifier's TxConfirmed + // arm is exercised twice in this lifecycle). Note: the + // re-confirmation can legitimately land at the SAME block height + // as the disconnected block — Reorg(1, 2) replaces 1 block with + // 2, so the first new block sits at the same height as the old + // one, just with a different hash. The notification doesn't + // carry BlockHash so we can't directly assert hash inequality; + // the canonical-height membership check below is the strongest + // available signal that this is a fresh confirmation on the new + // tip rather than a recycled stale event. + third := walletRecorder.await(t) + require.Equal( + t, wallet.BoardingSweepTxStatusConfirmed, third.Status, "thi"+ + "rd notification must be Confirmed "+ + "(re-confirmation), got status=%d", third.Status, + ) + require.Equal(t, *txid, third.Txid) + canonicalHeights := make(map[int32]struct{}, len(reorg.Connected)) + for _, blk := range reorg.Connected { + canonicalHeights[int32(blk.Height)] = struct{}{} + } + _, onCanonical := canonicalHeights[third.BlockHeight] + require.True( + t, onCanonical, "re-confirmation BlockHeight=%d must be one "+ + "of the new connected block heights %v", + third.BlockHeight, reorg.Connected, + ) + require.GreaterOrEqual( + t, third.NumConfs, uint32(1), + "re-confirmation NumConfs must be at least the target (1)", + ) + t.Logf( + "re-Confirmed: txid=%s height=%d num_confs=%d", third.Txid, + third.BlockHeight, third.NumConfs, + ) +} diff --git a/wallet/boarding_sweep_actor.go b/wallet/boarding_sweep_actor.go index b8c36e340..77363398a 100644 --- a/wallet/boarding_sweep_actor.go +++ b/wallet/boarding_sweep_actor.go @@ -210,19 +210,63 @@ func (m *ReplayPendingIntentsResponse) MessageType() string { func (m *ReplayPendingIntentsResponse) walletRespSealed() {} -// BoardingSweepSpendNotification is a Tell carrying a chainsource spend -// event for a boarding-sweep input. Emitted by the chainsource subscription -// the wallet actor sets up via MapSpendEvent. +// BoardingSweepSpendStatus identifies which point in the chainsource +// spend-watch reorg-aware lifecycle drove this notification. Splitting +// the status out makes it impossible for the handler to confuse a +// reorg-out with a fresh spend: a reorged-out spending block should +// not advance the input's persistent state machine further than the +// canonical chain has. +type BoardingSweepSpendStatus int + +const ( + // BoardingSweepSpendStatusUnknown is the zero value; receiving it + // indicates a programmer error (MapSpendEvent fell through). + BoardingSweepSpendStatusUnknown BoardingSweepSpendStatus = iota + + // BoardingSweepSpendStatusSpent reports that the watched outpoint + // was observed spent on the canonical chain by SpendingTxid at + // SpendingHeight. The observation is provisional until + // BoardingSweepSpendStatusDone arrives — a reorg can roll it back + // via BoardingSweepSpendStatusReorged. + BoardingSweepSpendStatusSpent + + // BoardingSweepSpendStatusReorged reports that a previously + // delivered Spent observation was reorged out. The handler logs + // the divergence and leaves the watch armed; the chainsource + // sub-actor stays alive in reorg-aware mode so a subsequent + // re-spend on the new canonical chain re-fires + // BoardingSweepSpendStatusSpent. + BoardingSweepSpendStatusReorged + + // BoardingSweepSpendStatusDone reports that the spend observation + // is past the chainsource backend's reorg-safety depth and is no + // longer reversible. Synthesized by the spend-actor's + // FinalityDepth synthesizer when the backend (notably lndclient + // over gRPC) does not surface a native Done signal. + BoardingSweepSpendStatusDone +) + +// BoardingSweepSpendNotification is a Tell carrying one event from the +// chainsource spend-watch reorg-aware lifecycle for a boarding-sweep +// input. Emitted by the subscription the wallet actor sets up via +// MapSpendEvent / MapSpendReorgedEvent / MapSpendDoneEvent. type BoardingSweepSpendNotification struct { actor.BaseMessage - // Outpoint is the boarding UTXO that was spent. + // Status identifies which lifecycle event this notification + // carries. See BoardingSweepSpendStatus. + Status BoardingSweepSpendStatus + + // Outpoint is the boarding UTXO whose spend lifecycle changed. Outpoint wire.OutPoint - // SpendingTxid is the transaction that confirmed the spend. + // SpendingTxid is the transaction that spent the outpoint when + // Status=Spent; zero on Reorged / Done (the wire-level event + // carries no spender identity past the first Spent observation). SpendingTxid chainhash.Hash - // SpendingHeight is the block height of the spending transaction. + // SpendingHeight is the block height of the spending transaction + // when Status=Spent; zero on Reorged / Done. SpendingHeight int32 } @@ -257,8 +301,7 @@ const ( // delivered TxConfirmed for this sweep was reorged out. The // handler must NOT call MarkBoardingSweepFailed; instead it // leaves the spend watches and pending state armed so the next - // TxConfirmed on the new canonical chain (or an eventual - // TxFailed) drives the terminal decision. + // TxConfirmed on the new canonical chain can drive finality. BoardingSweepTxStatusReorged // BoardingSweepTxStatusFinalized reports that the sweep @@ -268,9 +311,9 @@ const ( BoardingSweepTxStatusFinalized // BoardingSweepTxStatusFailed reports a terminal failure from - // txconfirm (broadcast rejected, retry budget exhausted, etc.). - // The handler marks the sweep failed in the store and cleans up - // pending state. + // txconfirm. It is terminal for that broadcaster attempt, but is not + // objective evidence that every input is unspent. The wallet therefore + // keeps the sweep reserved for the normal tip-paced resume path. BoardingSweepTxStatusFailed ) @@ -298,8 +341,8 @@ func classifyTxconfirmNotificationForBoardingSweep( case *txconfirm.TxReorged: // Reorg-out: do NOT mark the sweep as failed in the // handler. Leave spend watches and pending state armed so - // the next TxConfirmed on the new canonical chain (or an - // eventual TxFailed) drives the terminal decision. + // the next TxConfirmed on the new canonical chain can drive + // finality. return BoardingSweepTxNotification{ Status: BoardingSweepTxStatusReorged, Txid: ev.Txid, @@ -416,6 +459,12 @@ type pendingSweepState struct { // chainsource spend-watch deregistration. inputs map[wire.OutPoint]string + // provisionalSpends keeps the spender identity that arrives with a + // Spent event until the matching Done event seals it. Reorged removes + // the observation, while restart reconstructs it from the still-pending + // durable row and historical spend watch. + provisionalSpends map[wire.OutPoint]provisionalSweepSpend + // totalAmount is the gross input value of the sweep. totalAmount btcutil.Amount @@ -437,6 +486,13 @@ type pendingSweepState struct { submitted bool } +// provisionalSweepSpend is the reversible positive observation retained until +// chainsource reports that the spend reached policy finality. +type provisionalSweepSpend struct { + txid chainhash.Hash + height int32 +} + // boardingSweepCallerID returns the deterministic chainsource caller-id // used to register or cancel a per-outpoint spend watch. func boardingSweepCallerID(op wire.OutPoint) string { @@ -838,22 +894,19 @@ func (a *Ark) publishBoardingSweep(ctx context.Context, ctx, args.signed.Tx, args.pkScript, uint32(args.bestHeight), ); err != nil { - failErr := a.sweepStore.MarkBoardingSweepFailed( - ctx, pending.txid, err, - ) - if failErr != nil { - log.WarnS(ctx, "Failed to roll back failed sweep", - failErr, slog.String( - "txid", pending.txid.String(), - )) - } - - delete(a.pendingSweeps, pending.txid) - a.cancelSweepSpendWatches(ctx, pending) + // A broadcaster failure proves only that this submission + // attempt failed. It cannot prove all inputs remained unspent, + // especially when a conflicting spend notification may already + // be queued. Keep the durable reservation and watches intact. + // The existing per-tip recovery pass retries at a bounded + // cadence; an immediate self-retry here could spin forever on a + // structurally invalid tx. + pending.submitted = false return fn.Ok[WalletResp]( failedSweepResponse( - fmt.Errorf("submit sweep to broadcaster: %w", + fmt.Errorf("submit sweep to broadcaster: %w; "+ + "inputs remain reserved for recovery", err), args.feeRate, args.confTarget, @@ -923,6 +976,7 @@ func (a *Ark) registerSweepSpendWatch(ctx context.Context, notify := chainsource.MapSpendEvent(a.selfRef, func(ev chainsource.SpendEvent) WalletMsg { return BoardingSweepSpendNotification{ + Status: BoardingSweepSpendStatusSpent, Outpoint: ev.Outpoint, SpendingTxid: ev.SpendingTxid, SpendingHeight: ev.SpendingHeight, @@ -930,16 +984,44 @@ func (a *Ark) registerSweepSpendWatch(ctx context.Context, }, ) + // Reorg-aware lifecycle refs. Without these, a reorged-out + // external spender would leave the input row marked + // external_spent in the persistent store with no rollback path — + // the chainsource SpendActor would either tear the registration + // down (legacy mode) or hold reorg events for nobody (reorg-aware + // mode missing the refs). Wiring them keeps the registration + // alive past the first Spent event, surfaces Reorged on rollback, + // and lets the FinalityDepth synthesizer eventually fire Done so + // the sub-actor exits cleanly. + reorgedNotify := chainsource.MapSpendReorgedEvent(a.selfRef, + func(ev chainsource.SpendReorgedEvent) WalletMsg { + return BoardingSweepSpendNotification{ + Status: BoardingSweepSpendStatusReorged, + Outpoint: ev.Outpoint, + } + }, + ) + doneNotify := chainsource.MapSpendDoneEvent(a.selfRef, + func(ev chainsource.SpendDoneEvent) WalletMsg { + return BoardingSweepSpendNotification{ + Status: BoardingSweepSpendStatusDone, + Outpoint: ev.Outpoint, + } + }, + ) + callerID := boardingSweepCallerID(op) notifyOpt := fn.Some[actor.TellOnlyRef[chainsource.SpendEvent]]( notify, ) req := &chainsource.RegisterSpendRequest{ - CallerID: callerID, - Outpoint: &op, - PkScript: out.PkScript, - HeightHint: heightHint, - NotifyActor: notifyOpt, + CallerID: callerID, + Outpoint: &op, + PkScript: out.PkScript, + HeightHint: heightHint, + NotifyActor: notifyOpt, + NotifyReorged: fn.Some(reorgedNotify), + NotifyDone: fn.Some(doneNotify), } future := a.chainSource.Ask(ctx, req) @@ -965,6 +1047,9 @@ func (a *Ark) cancelSweepSpendWatches(ctx context.Context, cancellations[op] = id } pending.inputs = make(map[wire.OutPoint]string) + pending.provisionalSpends = make( + map[wire.OutPoint]provisionalSweepSpend, + ) for op, callerID := range cancellations { op := op @@ -1069,8 +1154,9 @@ func (a *Ark) submitSweepToConfirm(ctx context.Context, tx *wire.MsgTx, return nil } -// handleSweepSpendNotification updates persistent state when an input of -// an in-flight aggregate sweep is observed spent on chain. +// handleSweepSpendNotification holds a positive spend observation in memory +// until Done seals it. Reorged discards that provisional observation, so the +// durable sweep row never advances on evidence that can still disappear. func (a *Ark) handleSweepSpendNotification(ctx context.Context, notif BoardingSweepSpendNotification) fn.Result[WalletResp] { @@ -1078,109 +1164,169 @@ func (a *Ark) handleSweepSpendNotification(ctx context.Context, return fn.Ok[WalletResp](&BoardingSweepNotificationAck{}) } - var ( - callerID string - pending *pendingSweepState - ) + var pending *pendingSweepState for _, candidate := range a.pendingSweeps { - id, ok := candidate.inputs[notif.Outpoint] + _, ok := candidate.inputs[notif.Outpoint] if !ok { continue } - callerID = id pending = candidate break } - // The sweep's own input-spend notification is only a provisional - // one-confirmation observation. txconfirm owns the reorg-aware watch - // for that transaction and will commit the store transition at - // Finalized. Deferring here prevents a Confirmed -> Reorged -> Failed - // lifecycle from leaving the store and ledger falsely - // terminal-successful. - if pending != nil && notif.SpendingTxid == pending.txid { - a.logger(ctx).DebugS( - ctx, - "Deferring own sweep spend until finality", - slog.String("outpoint", notif.Outpoint.String()), - slog.String("sweep_txid", pending.txid.String()), - ) + switch notif.Status { + case BoardingSweepSpendStatusSpent: + if pending == nil { + return fn.Ok[WalletResp]( + &BoardingSweepNotificationAck{}, + ) + } + if pending.provisionalSpends == nil { + pending.provisionalSpends = make( + map[wire.OutPoint]provisionalSweepSpend, + ) + } + + pending.provisionalSpends[notif.Outpoint] = + provisionalSweepSpend{ + txid: notif.SpendingTxid, + height: notif.SpendingHeight, + } return fn.Ok[WalletResp](&BoardingSweepNotificationAck{}) - } - resolved, err := a.sweepStore.MarkBoardingSweepInputSpent( - ctx, notif.Outpoint, notif.SpendingTxid, notif.SpendingHeight, - ) - switch { - // A duplicate spend event for an input row that has already been - // resolved (e.g. a re-org-then-respend, or a stale buffered event - // arriving after the sweep moved to a terminal state) returns - // ErrNoRows from the store. This is benign — the row is already in - // its target state — so suppress it to Debug rather than alerting on - // every duplicate. - case errors.Is(err, sql.ErrNoRows): - a.logger(ctx).DebugS(ctx, - "Boarding sweep input already resolved; "+ - "ignoring duplicate spend", + case BoardingSweepSpendStatusReorged: + if pending != nil { + delete(pending.provisionalSpends, notif.Outpoint) + } + + a.logger(ctx).WarnS(ctx, "Boarding-sweep input spend "+ + "reorged out; waiting for a canonical re-spend", + nil, slog.String("outpoint", notif.Outpoint.String()), - slog.String( - "spending_txid", notif.SpendingTxid.String(), - )) + ) return fn.Ok[WalletResp](&BoardingSweepNotificationAck{}) - case err != nil: - a.logger(ctx).WarnS(ctx, - "Failed to mark boarding sweep input spent", err, + case BoardingSweepSpendStatusDone: + if pending == nil { + return fn.Ok[WalletResp]( + &BoardingSweepNotificationAck{}, + ) + } + + spend, ok := pending.provisionalSpends[notif.Outpoint] + if !ok { + a.logger(ctx).WarnS(ctx, "Final boarding-sweep spend "+ + "arrived without its positive observation", + nil, + slog.String("outpoint", notif.Outpoint.String()), + ) + a.rearmSweepSpendWatch(ctx, pending, notif.Outpoint) + + return fn.Ok[WalletResp]( + &BoardingSweepNotificationAck{}, + ) + } + + resolved, err := a.sweepStore.MarkBoardingSweepInputSpent( + ctx, notif.Outpoint, spend.txid, spend.height, + ) + + return a.finishSweepSpend( + ctx, pending, notif.Outpoint, spend.txid, resolved, err, + ) + + default: + a.logger(ctx).WarnS(ctx, "Boarding-sweep spend notification "+ + "with unknown status", + fmt.Errorf("status=%d", notif.Status), slog.String("outpoint", notif.Outpoint.String()), - slog.String( - "spending_txid", notif.SpendingTxid.String(), - )) + ) return fn.Ok[WalletResp](&BoardingSweepNotificationAck{}) } +} - if pending != nil { - delete(pending.inputs, notif.Outpoint) - delete(a.pendingSweepInputs, notif.Outpoint) - if resolved { - delete(a.pendingSweeps, pending.txid) - } - } +// finishSweepSpend applies a terminal store result and keeps the durable row +// recoverable when persistence fails. +func (a *Ark) finishSweepSpend(ctx context.Context, pending *pendingSweepState, + outpoint wire.OutPoint, spendingTxid chainhash.Hash, resolved bool, + err error) fn.Result[WalletResp] { - if callerID != "" { - op := notif.Outpoint - if err := a.chainSource.Tell(ctx, - &chainsource.UnregisterSpendRequest{ - CallerID: callerID, - Outpoint: &op, - }); err != nil { + switch { + case errors.Is(err, sql.ErrNoRows): + a.logger(ctx).DebugS( + ctx, + "Boarding sweep input already resolved; ignoring "+ + "duplicate", + slog.String("outpoint", outpoint.String()), + slog.String("spending_txid", spendingTxid.String()), + ) - a.logger(ctx).DebugS( - ctx, - "Best-effort unregister spend failed", - err, - slog.String("outpoint", op.String()), - ) - } + case err != nil: + a.logger(ctx).WarnS( + ctx, + "Failed to finalize boarding sweep input spend", + err, + slog.String("outpoint", outpoint.String()), + slog.String("spending_txid", spendingTxid.String()), + ) + a.rearmSweepSpendWatch(ctx, pending, outpoint) + + return fn.Ok[WalletResp](&BoardingSweepNotificationAck{}) } - if resolved && pending != nil { - // Cancel any straggler spend watches for the same sweep so - // chainsource sub-actors do not leak. + delete(pending.provisionalSpends, outpoint) + delete(pending.inputs, outpoint) + delete(a.pendingSweepInputs, outpoint) + if resolved || len(pending.inputs) == 0 { + delete(a.pendingSweeps, pending.txid) a.cancelSweepSpendWatches(ctx, pending) } return fn.Ok[WalletResp](&BoardingSweepNotificationAck{}) } +// rearmSweepSpendWatch drops stale in-memory watch ownership and asks the +// normal resume path to replay the still-pending durable row. +func (a *Ark) rearmSweepSpendWatch(ctx context.Context, + pending *pendingSweepState, outpoint wire.OutPoint) { + + delete(pending.provisionalSpends, outpoint) + delete(pending.inputs, outpoint) + delete(a.pendingSweepInputs, outpoint) + + a.scheduleBoardingSweepResume(ctx) +} + +// scheduleBoardingSweepResume asks the actor-owned recovery path to reconstruct +// pending watches without inheriting notification-message cancellation. +func (a *Ark) scheduleBoardingSweepResume(ctx context.Context) { + if a.selfRef == nil { + return + } + + // The wallet actor, not the notification message, owns recovery. Keep + // trace values while detaching the re-arm from a caller context that + // may already be canceled as a chainsource sub-actor shuts down. + rearmCtx := context.WithoutCancel(ctx) + err := a.selfRef.Tell(rearmCtx, &ResumeBoardingSweepsRequest{}) + if err != nil { + a.logger(ctx).WarnS( + ctx, + "Failed to schedule boarding sweep spend re-arm", + err, + ) + } +} + // handleSweepTxNotification processes the reorg-aware txconfirm lifecycle for // a tracked aggregate sweep. Confirmed remains provisional. Finalized commits -// the store and ledger transitions and releases the in-memory watches, while -// Failed mirrors a terminal broadcaster failure into the store. +// the store and ledger transitions and releases the in-memory watches. Failed +// ends one broadcaster attempt but keeps the inputs reserved for safe replay. func (a *Ark) handleSweepTxNotification(ctx context.Context, notif BoardingSweepTxNotification) fn.Result[WalletResp] { @@ -1205,16 +1351,6 @@ func (a *Ark) handleSweepTxNotification(ctx context.Context, // alive, so a subsequent TxConfirmed on the new canonical // chain remains provisional and an eventual Finalized commits // the canonical result. - // - // Best-effort backstop only: the per-input chainsource - // spend watch fires handleSweepSpendNotification if some - // other party spends an input. That spend-watch path is - // not itself reorg-symmetric — MapSpendEvent collapses the - // chainsource SpendEvent lifecycle to a single - // BoardingSweepSpendNotification without surfacing - // Reorged / Done — so a reorged-out external spender will - // leave the input row marked external_spent until manual - // reconciliation. Closing that gap is tracked separately. a.logger(ctx).WarnS( ctx, "Boarding sweep confirmation reorged out; waiting "+ @@ -1234,7 +1370,21 @@ func (a *Ark) handleSweepTxNotification(ctx context.Context, slog.String("txid", notif.Txid.String()), slog.Int("block_height", int(notif.BlockHeight)), ) - a.reconcileSweepInputsOnFinalized(ctx, notif) + if !a.reconcileSweepInputsOnFinalized(ctx, notif) { + // The txconfirm lifecycle is terminal, but durable + // input state is not. Re-submit through the normal + // recovery path so historical confirmation replay can + // retry finalization. + pending := a.pendingSweeps[notif.Txid] + if pending != nil { + pending.submitted = false + } + a.scheduleBoardingSweepResume(ctx) + + return fn.Ok[WalletResp]( + &BoardingSweepNotificationAck{}, + ) + } a.emitSweepConfirmedLedger(ctx, notif) pending := a.pendingSweeps[notif.Txid] @@ -1252,21 +1402,31 @@ func (a *Ark) handleSweepTxNotification(ctx context.Context, // would diverge the store from the ledger. If the persisted // record shows the sweep already reached a terminal-success // status, ignore the failure rather than undo a booked sweep. - if rec, ok := a.lookupSweepRecord(ctx, notif.Txid); ok && - isTerminalSuccessSweepStatus(rec.Status) { - - a.logger(ctx).WarnS( - ctx, - "Ignoring boarding sweep failure for an "+ - "already-resolved sweep", - errors.New(notif.Reason), - slog.String("txid", notif.Txid.String()), - slog.String("status", rec.Status), - ) - - return fn.Ok[WalletResp]( - &BoardingSweepNotificationAck{}, - ) + if rec, ok := a.lookupSweepRecord(ctx, notif.Txid); ok { + switch { + case isTerminalSuccessSweepStatus(rec.Status): + a.logger(ctx).WarnS( + ctx, + "Ignoring resolved sweep failure", + errors.New(notif.Reason), + slog.String("txid", + notif.Txid.String()), + slog.String("status", rec.Status), + ) + + return fn.Ok[WalletResp]( + &BoardingSweepNotificationAck{}, + ) + + case rec.Status == BoardingSweepStatusFailed: + // A legacy/pre-submission path already + // completed the failure transition. Do not + // schedule a futile replay for a row the + // pending-sweep query will not return. + return fn.Ok[WalletResp]( + &BoardingSweepNotificationAck{}, + ) + } } a.logger(ctx).WarnS( @@ -1280,23 +1440,13 @@ func (a *Ark) handleSweepTxNotification(ctx context.Context, // operators can alert on a stuck boarding-sweep watcher. a.emitBackgroundTaskError(ctx, "boarding_sweep_watcher") - err := a.sweepStore.MarkBoardingSweepFailed( - ctx, notif.Txid, errors.New(notif.Reason), - ) - if err != nil { - a.logger(ctx).WarnS( - ctx, - "Failed to mark boarding sweep failed", - err, - slog.String("txid", notif.Txid.String()), - ) - } - - pending := a.pendingSweeps[notif.Txid] - delete(a.pendingSweeps, notif.Txid) - - if pending != nil { - a.cancelSweepSpendWatches(ctx, pending) + // The attempt failed, but that is not negative chain evidence + // for any input. Keep the durable reservation and every spend + // watch alive so a queued/conflicting spend can still reach + // Done. The tip-driven resume path creates a fresh txconfirm + // attempt at a bounded cadence until recovery succeeds. + if pending := a.pendingSweeps[notif.Txid]; pending != nil { + pending.submitted = false } default: @@ -1322,22 +1472,47 @@ func (a *Ark) handleSweepTxNotification(ctx context.Context, // The store's status guard rejects a redundant transition with sql.ErrNoRows; // that signals "input row already advanced past pending/published" and is // treated as a benign no-op. Other errors still log at warn because they -// indicate a real persistence problem. +// indicate a real persistence problem. The return value is true only when +// every durable input transition is known to be complete. func (a *Ark) reconcileSweepInputsOnFinalized(ctx context.Context, - notif BoardingSweepTxNotification) { + notif BoardingSweepTxNotification) bool { - pending, ok := a.pendingSweeps[notif.Txid] - if !ok || pending == nil { - return + record, found := a.lookupSweepRecord(ctx, notif.Txid) + if !found { + return false } + if record.Status == BoardingSweepStatusConfirmed { + return true + } + if record.Status != BoardingSweepStatusPending && + record.Status != BoardingSweepStatusPublished { - for op := range pending.inputs { - _, err := a.sweepStore.MarkBoardingSweepInputSpent( - ctx, op, notif.Txid, notif.BlockHeight, + a.logger(ctx).WarnS( + ctx, + "Finalized sweep conflicts with durable status", + nil, + slog.String("txid", notif.Txid.String()), + slog.String("status", record.Status), + ) + + return false + } + + durable := true + resolved := false + for _, input := range record.Inputs { + if input.Status != BoardingSweepInputStatusPending && + input.Status != BoardingSweepInputStatusPublished { + + continue + } + + inputResolved, err := a.sweepStore.MarkBoardingSweepInputSpent( + ctx, input.Outpoint, notif.Txid, notif.BlockHeight, ) switch { case err == nil: - // Success. + resolved = resolved || inputResolved case errors.Is(err, sql.ErrNoRows): // Row already past pending/published — likely @@ -1346,15 +1521,31 @@ func (a *Ark) reconcileSweepInputsOnFinalized(ctx context.Context, // Idempotent no-op. default: + durable = false a.logger(ctx).WarnS( ctx, "Failed to mark sweep input spent on confirm", err, - slog.String("outpoint", op.String()), + slog.String("outpoint", + input.Outpoint.String()), slog.String("txid", notif.Txid.String()), ) } } + + if !durable { + return false + } + if resolved { + return true + } + + // Every candidate raced to a terminal row between the initial read and + // its guarded update. Re-read the aggregate so a concurrent idempotent + // finalization is accepted only if the durable parent is now confirmed. + record, found = a.lookupSweepRecord(ctx, notif.Txid) + + return found && record.Status == BoardingSweepStatusConfirmed } // emitSweepConfirmedLedger emits the double-entry ledger and UTXO audit diff --git a/wallet/boarding_sweep_actor_test.go b/wallet/boarding_sweep_actor_test.go index 6f9d226ac..b76f67ddc 100644 --- a/wallet/boarding_sweep_actor_test.go +++ b/wallet/boarding_sweep_actor_test.go @@ -403,47 +403,55 @@ func TestSweepBoardingUTXOsPreviewBuildsTx(t *testing.T) { ) } -// TestSweepSpendNotificationMarksInputSpent verifies that a chainsource -// spend event for a tracked input is forwarded to the store and the -// in-memory tracking map is cleaned up when the sweep resolves. -func TestSweepSpendNotificationMarksInputSpent(t *testing.T) { +// TestSweepSpendNotificationKeepsPositiveProvisional verifies that a +// chainsource spend event remains reversible until the final Done event. +func TestSweepSpendNotificationKeepsPositiveProvisional(t *testing.T) { t.Parallel() op := wire.OutPoint{Hash: chainhash.Hash{0xab}, Index: 0} spendingTxid := chainhash.Hash{0xcd} store := &MockBoardingSweepStore{} - store.On( - "MarkBoardingSweepInputSpent", mock.Anything, op, - spendingTxid, int32(220), - ).Return(true, nil) a := newSweepTestArk(t, store, nil, 0, 0) - // Pre-populate pending sweep state so the handler clears it on - // resolution. + // Pre-populate pending sweep state. The positive observation must not + // clear it or touch durable state before the finality boundary. pendingTxid := chainhash.Hash{0xee} - a.pendingSweeps[pendingTxid] = &pendingSweepState{ + pending := &pendingSweepState{ txid: pendingTxid, inputs: map[wire.OutPoint]string{ op: boardingSweepCallerID(op), }, + provisionalSpends: map[wire.OutPoint]provisionalSweepSpend{ + op: { + txid: chainhash.Hash{ + 0xb3, + }, + height: 120, + }, + }, } + a.pendingSweeps[pendingTxid] = pending + a.pendingSweepInputs[op] = pendingTxid result := a.handleSweepSpendNotification( t.Context(), BoardingSweepSpendNotification{ + Status: BoardingSweepSpendStatusSpent, Outpoint: op, SpendingTxid: spendingTxid, SpendingHeight: 220, }, ) require.True(t, result.IsOk()) - require.Empty( - t, a.pendingSweeps, - "resolved sweep must be evicted from in-memory tracking", + require.Same(t, pending, a.pendingSweeps[pendingTxid]) + require.Equal(t, pendingTxid, a.pendingSweepInputs[op]) + require.Equal( + t, provisionalSweepSpend{ + txid: spendingTxid, height: 220, + }, pending.provisionalSpends[op], ) - - store.AssertExpectations(t) + store.AssertNotCalled(t, "MarkBoardingSweepInputSpent") } // TestSweepSpendNotificationDefersOwnSweepUntilFinality verifies that the @@ -468,6 +476,7 @@ func TestSweepSpendNotificationDefersOwnSweepUntilFinality(t *testing.T) { result := a.handleSweepSpendNotification( t.Context(), BoardingSweepSpendNotification{ + Status: BoardingSweepSpendStatusSpent, Outpoint: op, SpendingTxid: sweepTxid, SpendingHeight: 220, @@ -476,6 +485,11 @@ func TestSweepSpendNotificationDefersOwnSweepUntilFinality(t *testing.T) { require.True(t, result.IsOk()) require.Same(t, pending, a.pendingSweeps[sweepTxid]) require.Equal(t, sweepTxid, a.pendingSweepInputs[op]) + require.Equal( + t, provisionalSweepSpend{ + txid: sweepTxid, height: 220, + }, pending.provisionalSpends[op], + ) store.AssertNotCalled(t, "MarkBoardingSweepInputSpent") } @@ -983,30 +997,28 @@ func TestSweepTxNotificationMissingTxSkipsLegs(t *testing.T) { ) } -// TestSweepTxNotificationFailedMarksFailed verifies that a terminal -// txconfirm failure is mirrored into the persistent store and the -// in-memory pending state is dropped. -func TestSweepTxNotificationFailedMarksFailed(t *testing.T) { +// TestSweepTxNotificationFailedKeepsInputsReserved verifies that a terminal +// txconfirm attempt failure does not masquerade as proof that every input is +// unspent. Recovery retains the watches and schedules a fresh attempt. +func TestSweepTxNotificationFailedKeepsInputsReserved(t *testing.T) { t.Parallel() failedTxid := chainhash.Hash{0x99} store := &MockBoardingSweepStore{} - store.On( - "MarkBoardingSweepFailed", mock.Anything, failedTxid, - mock.Anything, - ).Return(nil) - // The Failed arm looks the sweep up first to ignore failures for an - // already-resolved sweep; absent record means the failure proceeds. + // The persisted lookup is absent in this actor-only test. In production + // the in-memory pending entry corresponds to the pending durable row. store.On( "GetBoardingSweep", mock.Anything, failedTxid, ).Return(nil, nil) a := newSweepTestArk(t, store, nil, 0, 0) - a.pendingSweeps[failedTxid] = &pendingSweepState{ - txid: failedTxid, - inputs: map[wire.OutPoint]string{}, + pending := &pendingSweepState{ + txid: failedTxid, + submitted: true, + inputs: map[wire.OutPoint]string{}, } + a.pendingSweeps[failedTxid] = pending result := a.handleSweepTxNotification( t.Context(), BoardingSweepTxNotification{ @@ -1016,8 +1028,9 @@ func TestSweepTxNotificationFailedMarksFailed(t *testing.T) { }, ) require.True(t, result.IsOk()) - require.Empty(t, a.pendingSweeps) - + require.Same(t, pending, a.pendingSweeps[failedTxid]) + require.False(t, pending.submitted) + store.AssertNotCalled(t, "MarkBoardingSweepFailed") store.AssertExpectations(t) } @@ -1079,6 +1092,17 @@ func TestSweepTxNotificationFinalizedCommitsAndCleansUp(t *testing.T) { "MarkBoardingSweepInputSpent", mock.Anything, op, finalizedTxid, int32(800_750), ).Return(true, nil) + store.On( + "GetBoardingSweep", mock.Anything, finalizedTxid, + ).Return(&BoardingSweepRecord{ + Txid: finalizedTxid, + Status: BoardingSweepStatusPublished, + Inputs: []BoardingSweepInputRecord{{ + Txid: finalizedTxid, + Outpoint: op, + Status: BoardingSweepInputStatusPublished, + }}, + }, nil) a := newSweepTestArk(t, store, nil, 0, 0) pending := &pendingSweepState{ @@ -1095,7 +1119,7 @@ func TestSweepTxNotificationFinalizedCommitsAndCleansUp(t *testing.T) { Status: BoardingSweepTxStatusFinalized, Txid: finalizedTxid, BlockHeight: 800_750, - NumConfs: 6, + NumConfs: 31, }, ) require.True(t, result.IsOk()) @@ -1106,6 +1130,136 @@ func TestSweepTxNotificationFinalizedCommitsAndCleansUp(t *testing.T) { store.AssertExpectations(t) } +// TestSweepTxNotificationFinalizedPersistenceFailureRearms verifies that a +// terminal chain signal cannot release watches or run later effects before all +// input transitions are durable. +func TestSweepTxNotificationFinalizedPersistenceFailureRearms(t *testing.T) { + t.Parallel() + + finalizedTxid := chainhash.Hash{0xa9} + op := wire.OutPoint{Hash: chainhash.Hash{0xb9}, Index: 0} + store := &MockBoardingSweepStore{} + store.On( + "MarkBoardingSweepInputSpent", mock.Anything, op, + finalizedTxid, int32(800_760), + ).Return(false, errors.New("database unavailable")) + store.On( + "GetBoardingSweep", mock.Anything, finalizedTxid, + ).Return(&BoardingSweepRecord{ + Txid: finalizedTxid, + Status: BoardingSweepStatusPublished, + Inputs: []BoardingSweepInputRecord{{ + Txid: finalizedTxid, + Outpoint: op, + Status: BoardingSweepInputStatusPublished, + }}, + }, nil) + + a := newSweepTestArk(t, store, nil, 0, 0) + selfRef := actor.NewChannelTellOnlyRef[WalletMsg]("wallet-self", 1) + a.selfRef = selfRef + sink, drain := newCapturingLedgerSink(t) + a.ledgerSink = fn.Some(sink) + pending := &pendingSweepState{ + txid: finalizedTxid, + submitted: true, + inputs: map[wire.OutPoint]string{ + op: boardingSweepCallerID(op), + }, + } + a.pendingSweeps[finalizedTxid] = pending + a.pendingSweepInputs[op] = finalizedTxid + + ctx, cancel := context.WithCancel(t.Context()) + cancel() + result := a.handleSweepTxNotification( + ctx, BoardingSweepTxNotification{ + Status: BoardingSweepTxStatusFinalized, + Txid: finalizedTxid, + BlockHeight: 800_760, + NumConfs: 31, + }, + ) + require.True(t, result.IsOk()) + require.Same(t, pending, a.pendingSweeps[finalizedTxid]) + require.Equal(t, finalizedTxid, a.pendingSweepInputs[op]) + require.False(t, pending.submitted) + + msg, ok := selfRef.AwaitMessage(time.Second) + require.True( + t, ok, "failed durable finalization must schedule recovery", + ) + _, ok = msg.(*ResumeBoardingSweepsRequest) + require.True(t, ok) + require.Empty( + t, drain(0), + "ledger must not run before all input transitions are durable", + ) + store.AssertExpectations(t) +} + +// TestSweepTxNotificationFinalizedCoversUnwatchedInputs verifies that the +// terminal backstop reads the complete durable input set instead of trusting +// the in-memory watch map. A registration failure must not leave one consumed +// boarding input available after the sweep itself reaches finality. +func TestSweepTxNotificationFinalizedCoversUnwatchedInputs(t *testing.T) { + t.Parallel() + + finalizedTxid := chainhash.Hash{0xaa} + armed := wire.OutPoint{Hash: chainhash.Hash{0xba}, Index: 0} + unwatched := wire.OutPoint{Hash: chainhash.Hash{0xbb}, Index: 1} + store := &MockBoardingSweepStore{} + store.On( + "GetBoardingSweep", mock.Anything, finalizedTxid, + ).Return(&BoardingSweepRecord{ + Txid: finalizedTxid, + Status: BoardingSweepStatusPublished, + Inputs: []BoardingSweepInputRecord{ + { + Txid: finalizedTxid, + Outpoint: armed, + Status: BoardingSweepInputStatusPublished, + }, + { + Txid: finalizedTxid, + Outpoint: unwatched, + Status: BoardingSweepInputStatusPublished, + }, + }, + }, nil) + store.On( + "MarkBoardingSweepInputSpent", mock.Anything, armed, + finalizedTxid, int32(800_770), + ).Return(false, nil) + store.On( + "MarkBoardingSweepInputSpent", mock.Anything, unwatched, + finalizedTxid, int32(800_770), + ).Return(true, nil) + + a := newSweepTestArk(t, store, nil, 0, 0) + pending := &pendingSweepState{ + txid: finalizedTxid, + inputs: map[wire.OutPoint]string{ + armed: boardingSweepCallerID(armed), + }, + } + a.pendingSweeps[finalizedTxid] = pending + a.pendingSweepInputs[armed] = finalizedTxid + + result := a.handleSweepTxNotification( + t.Context(), BoardingSweepTxNotification{ + Status: BoardingSweepTxStatusFinalized, + Txid: finalizedTxid, + BlockHeight: 800_770, + NumConfs: 31, + }, + ) + require.True(t, result.IsOk()) + require.Empty(t, a.pendingSweeps) + require.Empty(t, a.pendingSweepInputs) + store.AssertExpectations(t) +} + // TestSweepTxNotificationConfirmedRemainsProvisional verifies that the first // confirmation cannot mutate durable sweep success or release recovery state. func TestSweepTxNotificationConfirmedRemainsProvisional(t *testing.T) { @@ -1162,33 +1316,26 @@ func TestSweepTxNotificationReorgedAfterPendingCleared(t *testing.T) { store.AssertExpectations(t) } -// TestSweepTxNotificationFailedAfterReorgedStillTerminates verifies -// the Reorged arm does NOT suppress a subsequent terminal Failed -// notification: a sequence of Reorged then Failed must still call -// MarkBoardingSweepFailed and drop the pending entry, otherwise a -// reorg followed by a hard broadcast failure would be silently -// stranded. -func TestSweepTxNotificationFailedAfterReorgedStillTerminates(t *testing.T) { +// TestSweepTxNotificationFailedAfterReorgedStaysReserved verifies a failed +// re-confirmation attempt after a reorg cannot restore inputs whose canonical +// spend state is still uncertain. +func TestSweepTxNotificationFailedAfterReorgedStaysReserved(t *testing.T) { t.Parallel() txid := chainhash.Hash{0xa4} store := &MockBoardingSweepStore{} - store.On( - "MarkBoardingSweepFailed", mock.Anything, txid, - mock.Anything, - ).Return(nil) - // The Failed arm now looks the sweep up first to ignore a spurious - // failure for an already-resolved sweep. Here the record is absent - // (not terminal-success), so the failure path proceeds as before. store.On( "GetBoardingSweep", mock.Anything, txid, ).Return(nil, nil) a := newSweepTestArk(t, store, nil, 0, 0) + selfRef := actor.NewChannelTellOnlyRef[WalletMsg]("wallet-self", 1) + a.selfRef = selfRef pending := &pendingSweepState{ - txid: txid, - inputs: map[wire.OutPoint]string{}, + txid: txid, + submitted: true, + inputs: map[wire.OutPoint]string{}, } a.pendingSweeps[txid] = pending @@ -1203,8 +1350,8 @@ func TestSweepTxNotificationFailedAfterReorgedStillTerminates(t *testing.T) { require.True(t, reorgResult.IsOk()) require.Same(t, pending, a.pendingSweeps[txid]) - // Step 2: Failed — terminal path must still fire even though we - // passed through Reorged first. + // Step 2: Failed — the broadcaster attempt ends, but the durable + // reservation and spend-watch ownership remain fail-closed. failResult := a.handleSweepTxNotification( t.Context(), BoardingSweepTxNotification{ Status: BoardingSweepTxStatusFailed, @@ -1213,11 +1360,9 @@ func TestSweepTxNotificationFailedAfterReorgedStillTerminates(t *testing.T) { }, ) require.True(t, failResult.IsOk()) - require.Empty( - t, a.pendingSweeps, - "Failed after Reorged must still tear down pendingSweeps", - ) - + require.Same(t, pending, a.pendingSweeps[txid]) + require.False(t, pending.submitted) + store.AssertNotCalled(t, "MarkBoardingSweepFailed") store.AssertExpectations(t) } @@ -1291,3 +1436,238 @@ func TestSweepTxNotificationUnknownStatusIsBenign(t *testing.T) { store.AssertExpectations(t) } + +// TestSweepSpendNotificationReorgedDoesNotMarkSpent verifies that a +// chainsource SpendReorgedEvent on a watched boarding-sweep input is +// classified as BoardingSweepSpendStatusReorged and processed by the +// Reorged arm of handleSweepSpendNotification — specifically, the +// store's MarkBoardingSweepInputSpent must NOT be called (would +// double-spend the row's state machine) and pendingSweeps tracking +// must be left intact (the chainsource sub-actor stays alive in +// reorg-aware mode; a follow-up Spent on the canonical chain will +// drive the row through the existing happy path). +func TestSweepSpendNotificationReorgedDoesNotMarkSpent(t *testing.T) { + t.Parallel() + + op := wire.OutPoint{Hash: chainhash.Hash{0xb1}, Index: 0} + + store := &MockBoardingSweepStore{} + // CRITICAL: MarkBoardingSweepInputSpent must NOT be called on + // reorg. The testify/mock will fail the test if any unexpected + // method is invoked, so we simply do not register + // MarkBoardingSweepInputSpent here. + + a := newSweepTestArk(t, store, nil, 0, 0) + + pendingTxid := chainhash.Hash{0xb2} + pending := &pendingSweepState{ + txid: pendingTxid, + inputs: map[wire.OutPoint]string{ + op: boardingSweepCallerID(op), + }, + } + a.pendingSweeps[pendingTxid] = pending + a.pendingSweepInputs[op] = pendingTxid + + result := a.handleSweepSpendNotification( + t.Context(), BoardingSweepSpendNotification{ + Status: BoardingSweepSpendStatusReorged, + Outpoint: op, + }, + ) + require.True(t, result.IsOk()) + + // Reorg must leave both in-memory tracking maps alone — the + // sub-actor stays alive in reorg-aware mode and a follow-up + // Spent event on the canonical chain will re-drive the row. + require.Same( + t, pending, a.pendingSweeps[pendingTxid], + "reorg must not evict pending sweep tracking", + ) + require.Equal( + t, pendingTxid, a.pendingSweepInputs[op], + "reorg must not evict per-input back-pointer", + ) + _, provisional := pending.provisionalSpends[op] + require.False( + t, provisional, + "reorg must discard the non-canonical positive observation", + ) + + store.AssertExpectations(t) +} + +// TestSweepSpendNotificationDoneCommitsPositive verifies that Done seals the +// matching positive observation in the store and releases in-memory tracking. +func TestSweepSpendNotificationDoneCommitsPositive(t *testing.T) { + t.Parallel() + + op := wire.OutPoint{Hash: chainhash.Hash{0xc1}, Index: 0} + spendingTxid := chainhash.Hash{0xc3} + + store := &MockBoardingSweepStore{} + store.On( + "MarkBoardingSweepInputSpent", mock.Anything, op, + spendingTxid, int32(140), + ).Return(true, nil) + + a := newSweepTestArk(t, store, nil, 0, 0) + + pendingTxid := chainhash.Hash{0xc2} + pending := &pendingSweepState{ + txid: pendingTxid, + inputs: map[wire.OutPoint]string{ + op: boardingSweepCallerID(op), + }, + provisionalSpends: map[wire.OutPoint]provisionalSweepSpend{ + op: { + txid: spendingTxid, + height: 140, + }, + }, + } + a.pendingSweeps[pendingTxid] = pending + a.pendingSweepInputs[op] = pendingTxid + + result := a.handleSweepSpendNotification( + t.Context(), BoardingSweepSpendNotification{ + Status: BoardingSweepSpendStatusDone, + Outpoint: op, + }, + ) + require.True(t, result.IsOk()) + + require.Empty(t, a.pendingSweeps) + require.Empty(t, a.pendingSweepInputs) + require.Empty(t, pending.provisionalSpends) + + store.AssertExpectations(t) +} + +// TestSweepSpendNotificationPersistenceFailureRearms verifies that a final +// spend is never forgotten when the terminal store write fails. Recovery is +// owned by the wallet actor and therefore survives cancellation of the +// notification context. +func TestSweepSpendNotificationPersistenceFailureRearms(t *testing.T) { + t.Parallel() + + op := wire.OutPoint{Hash: chainhash.Hash{0xc4}, Index: 1} + spendingTxid := chainhash.Hash{0xc5} + pendingTxid := chainhash.Hash{0xc6} + store := &MockBoardingSweepStore{} + store.On( + "MarkBoardingSweepInputSpent", mock.Anything, op, + spendingTxid, int32(150), + ).Return(false, errors.New("database unavailable")) + + a := newSweepTestArk(t, store, nil, 0, 0) + selfRef := actor.NewChannelTellOnlyRef[WalletMsg]("wallet-self", 1) + a.selfRef = selfRef + pending := &pendingSweepState{ + txid: pendingTxid, + inputs: map[wire.OutPoint]string{ + op: boardingSweepCallerID(op), + }, + provisionalSpends: map[wire.OutPoint]provisionalSweepSpend{ + op: { + txid: spendingTxid, + height: 150, + }, + }, + } + a.pendingSweeps[pendingTxid] = pending + a.pendingSweepInputs[op] = pendingTxid + + ctx, cancel := context.WithCancel(t.Context()) + cancel() + result := a.handleSweepSpendNotification( + ctx, BoardingSweepSpendNotification{ + Status: BoardingSweepSpendStatusDone, + Outpoint: op, + }, + ) + require.True(t, result.IsOk()) + require.Same(t, pending, a.pendingSweeps[pendingTxid]) + require.NotContains(t, pending.inputs, op) + require.NotContains(t, a.pendingSweepInputs, op) + require.NotContains(t, pending.provisionalSpends, op) + + msg, ok := selfRef.AwaitMessage(time.Second) + require.True( + t, ok, "canceled notification must still schedule recovery", + ) + _, ok = msg.(*ResumeBoardingSweepsRequest) + require.True(t, ok) + store.AssertExpectations(t) +} + +// TestSweepSpendNotificationDoneWithoutPositiveRearms verifies that an +// impossible Done-first ordering fails closed and reconstructs the watch from +// the still-pending durable sweep rather than inventing spender evidence. +func TestSweepSpendNotificationDoneWithoutPositiveRearms(t *testing.T) { + t.Parallel() + + op := wire.OutPoint{Hash: chainhash.Hash{0xc7}, Index: 2} + pendingTxid := chainhash.Hash{0xc8} + store := &MockBoardingSweepStore{} + a := newSweepTestArk(t, store, nil, 0, 0) + selfRef := actor.NewChannelTellOnlyRef[WalletMsg]("wallet-self", 1) + a.selfRef = selfRef + pending := &pendingSweepState{ + txid: pendingTxid, + inputs: map[wire.OutPoint]string{ + op: boardingSweepCallerID(op), + }, + } + a.pendingSweeps[pendingTxid] = pending + a.pendingSweepInputs[op] = pendingTxid + + result := a.handleSweepSpendNotification( + t.Context(), BoardingSweepSpendNotification{ + Status: BoardingSweepSpendStatusDone, + Outpoint: op, + }, + ) + require.True(t, result.IsOk()) + store.AssertNotCalled(t, "MarkBoardingSweepInputSpent") + + msg, ok := selfRef.AwaitMessage(time.Second) + require.True(t, ok) + _, ok = msg.(*ResumeBoardingSweepsRequest) + require.True(t, ok) +} + +// TestSweepSpendNotificationUnknownStatusIsBenign verifies the default +// arm of handleSweepSpendNotification handles an unrecognised status +// without touching the store. Guards against a future chainsource +// lifecycle addition being delivered without a matching MapSpendEvent +// classification arm. +func TestSweepSpendNotificationUnknownStatusIsBenign(t *testing.T) { + t.Parallel() + + op := wire.OutPoint{Hash: chainhash.Hash{0xd1}, Index: 0} + store := &MockBoardingSweepStore{} + // No expectations — unknown must touch nothing. + + a := newSweepTestArk(t, store, nil, 0, 0) + pendingTxid := chainhash.Hash{0xd2} + pending := &pendingSweepState{ + txid: pendingTxid, + inputs: map[wire.OutPoint]string{ + op: boardingSweepCallerID(op), + }, + } + a.pendingSweeps[pendingTxid] = pending + a.pendingSweepInputs[op] = pendingTxid + + result := a.handleSweepSpendNotification( + t.Context(), BoardingSweepSpendNotification{ + Status: BoardingSweepSpendStatusUnknown, + Outpoint: op, + }, + ) + require.True(t, result.IsOk()) + require.Same(t, pending, a.pendingSweeps[pendingTxid]) + + store.AssertExpectations(t) +} From 7f102724995a20adbc04c8bbf39432244c63367c Mon Sep 17 00:00:00 2001 From: Elle Mouton Date: Thu, 16 Jul 2026 10:50:11 -0700 Subject: [PATCH 3/4] multi: Wire configurable reorg recovery policy The reorg-aware actors need one explicit policy boundary. Without shared wiring, individual subsystems can finalize at different depths or retain watches indefinitely when a backend allocates a Done channel but never writes to it. Add reorgsafetydepth as the deepest chain replacement the client promises to recover from. Default it to 30 blocks, cap it at the current cross-backend limit of 144, and begin terminal finality one confirmation beyond the selected horizon. Feed that boundary into chain-source Done synthesis and unilateral exit restart reconciliation. Define the reorg-safe Ark protocol version, but deliberately keep client advertisement pinned to v1. One-confirmation VTXO exposure remains disabled until batch lineage, admission gates, server recovery, and end-to-end safety tests land in the remaining stack. Document the configuration boundary and test default, override, validation, finality, and negotiation behavior. --- arkrpc/version.go | 15 ++++--- chainsource/chainsource.go | 8 +--- chainsource/conf_actor.go | 2 +- sample-waved.conf | 13 ++++++ waved/config.go | 45 ++++++++++++++++++++ waved/config_reorg_safety_test.go | 68 ++++++++++++++++++++++++++++++ waved/operator_negotiation.go | 4 +- waved/operator_negotiation_test.go | 10 +++++ waved/server.go | 60 +++++++++++++++++++++++--- 9 files changed, 203 insertions(+), 22 deletions(-) create mode 100644 waved/config_reorg_safety_test.go diff --git a/arkrpc/version.go b/arkrpc/version.go index ca9295ad5..d0881354a 100644 --- a/arkrpc/version.go +++ b/arkrpc/version.go @@ -7,9 +7,14 @@ package arkrpc // expectations. // // It is negotiated through the direct GetInfo bootstrap RPC and bound to a -// client runtime for its lifetime. Production currently supports only v1; a -// synthetic v2 may be configured in tests to exercise selection, binding, and -// rejection behavior, but no production default advertises a version beyond -// v1. This constant is deliberately separate from the mailbox transport -// version and the VTXO construction version, which evolve independently. +// client runtime for its lifetime. This constant is deliberately separate +// from the mailbox transport version and the VTXO construction version, which +// evolve independently. const ArkProtocolVersionV1 uint32 = 1 + +// ArkProtocolVersionV2 identifies the complete one-confirmation reorg-safety +// contract: durable lineage registration, fail-closed admission, reversible +// effects through the negotiated policy horizon, and recovery on both peers. +// Defining the compatibility boundary does not enable it; clients must not +// advertise v2 until every required gate and end-to-end proof is present. +const ArkProtocolVersionV2 uint32 = 2 diff --git a/chainsource/chainsource.go b/chainsource/chainsource.go index 0a1284d02..0b860d80c 100644 --- a/chainsource/chainsource.go +++ b/chainsource/chainsource.go @@ -20,13 +20,7 @@ const ( epochChannelSize = 10 // DefaultFinalityDepth is the conventional Bitcoin reorg-safety - // depth. After this many inclusive confirmations the ConfActor - // and SpendActor synthesize their Done events when the backend - // transport (notably lndclient over gRPC) does not surface one - // of its own. Six is the value the wider Lightning stack treats - // as final for the purposes of channel funding / settlement, so - // using it here keeps the unroll subsystem's finality threshold - // aligned with the rest of the daemon's chain assumptions. + // depth. Daemon policy may override it through ChainSourceConfig. DefaultFinalityDepth uint32 = 6 ) diff --git a/chainsource/conf_actor.go b/chainsource/conf_actor.go index 6482e6a28..97bbed7c2 100644 --- a/chainsource/conf_actor.go +++ b/chainsource/conf_actor.go @@ -34,7 +34,7 @@ type ConfActorConfig struct { // // The depth is counted inclusively (a tx confirmed at height H is // at depth 1; height-based finality fires once the actor observes - // a block at H + FinalityDepth - 1). The conventional choice is 6. + // a block at H + FinalityDepth - 1). FinalityDepth uint32 } diff --git a/sample-waved.conf b/sample-waved.conf index 41689968a..8c0f42e5c 100644 --- a/sample-waved.conf +++ b/sample-waved.conf @@ -40,6 +40,11 @@ # btcwallet, and unknown signers. Set one to force serial signing. # signingworkers=0 +# Deepest chain replacement for which provisional evidence remains +# recoverable. Terminal finality begins one confirmation later. The zero value +# uses the default of 30; the current cross-backend maximum is 144. +# reorgsafetydepth=30 + # Explicit opt-in for mainnet operation. # allow-mainnet=false @@ -223,6 +228,14 @@ # Maximum unroll fee rate in sat/vB. The zero value uses the unroll default. # unroll.maxfeeratesatpervbyte=0 +# Per-anchor restart-reconciliation probe timeout in seconds. The chainsource- +# backed ChainReconciler issues one probe per persisted anchor on Resume; a +# probe that exceeds this budget leaves the durable checkpoint unchanged and +# fails closed for a later retry. Operators running against a slow chain +# backend can raise the budget. The zero value uses the reconciler's internal +# default (10s). +# unroll.reconcileprobetimeoutsec=0 + # Swap server address for swapruntime builds. Empty uses the network+transport # default; see docs/signet.md for the public test-network endpoints. # swap.serveraddress= diff --git a/waved/config.go b/waved/config.go index d98a2671e..a80d572c4 100644 --- a/waved/config.go +++ b/waved/config.go @@ -176,6 +176,16 @@ const ( // creating an excessive number of simultaneous cryptographic jobs. MaxSigningWorkers = 64 + // DefaultReorgSafetyDepth is the deepest chain replacement the daemon + // keeps provisional and promises to recover from once reorg-safe v2 is + // enabled. Terminal finality begins one confirmation later. + DefaultReorgSafetyDepth uint32 = 30 + + // MaxReorgSafetyDepth is the largest policy horizon supported by every + // current chain backend. LND's notifier retains reversible watches for + // at most 144 disconnected blocks. + MaxReorgSafetyDepth uint32 = 144 + // DefaultWalletType is the default wallet backend. The "lwwallet" // backend uses an in-process lightweight wallet backed by // btcwallet and Esplora, requiring no external lnd node. @@ -338,6 +348,13 @@ type Config struct { // original serial behavior. SigningWorkers int `mapstructure:"signingworkers"` + // ReorgSafetyDepth is the deepest chain replacement for which evidence + // remains provisional and recoverable. Terminal finality begins at + // ReorgSafetyDepth+1 confirmations. Zero uses + // DefaultReorgSafetyDepth. This policy is not advertised until the full + // v2 safety capability and depth negotiation are enabled. + ReorgSafetyDepth uint32 `mapstructure:"reorgsafetydepth"` + // RegistrationTimeout is the maximum wall-clock duration to // wait for the server's RoundJoined admission watermark after // sending a JoinRoundRequest. If zero, the round package @@ -524,6 +541,14 @@ type UnrollConfig struct { // MaxFeeRateSatPerVByte caps fee estimates to prevent runaway // fees. Zero uses the default of 100 sat/vB. MaxFeeRateSatPerVByte int64 `mapstructure:"maxfeeratesatpervbyte"` + + // ReconcileProbeTimeoutSec bounds each per-anchor restart- + // reconciliation probe issued by the chainsource-backed + // ChainReconciler (in seconds). A probe that times out leaves the + // durable checkpoint unchanged and fails closed for a later retry; + // operators running against a slow backend can raise this budget. + // Zero uses the reconciler's internal default (10s). + ReconcileProbeTimeoutSec int64 `mapstructure:"reconcileprobetimeoutsec"` } // FeeEstimationConfig groups optional external chain fee providers used by the @@ -1184,6 +1209,7 @@ func DefaultConfig() *Config { }, MaxOperatorFeeSat: DefaultMaxOperatorFeeSat, SigningWorkers: DefaultSigningWorkers, + ReorgSafetyDepth: DefaultReorgSafetyDepth, OOR: defaultOORConfig(), FeeEstimation: &FeeEstimationConfig{ MempoolSpace: &MempoolSpaceFeeConfig{}, @@ -1243,6 +1269,10 @@ func (c *Config) Validate() error { return fmt.Errorf("signingworkers exceeds maximum %d: got %d", MaxSigningWorkers, c.SigningWorkers) } + if c.ReorgSafetyDepth > MaxReorgSafetyDepth { + return fmt.Errorf("reorgsafetydepth exceeds maximum %d: got %d", + MaxReorgSafetyDepth, c.ReorgSafetyDepth) + } if c.OOR == nil { c.OOR = defaultOORConfig() @@ -1411,6 +1441,21 @@ func (c *Config) validateWalletConfig() error { return nil } +// reorgSafetyDepth resolves the daemon's configured recovery horizon. +func (c *Config) reorgSafetyDepth() uint32 { + if c.ReorgSafetyDepth == 0 { + return DefaultReorgSafetyDepth + } + + return c.ReorgSafetyDepth +} + +// chainFinalityDepth returns the first inclusive confirmation depth beyond +// the configured recovery horizon. +func (c *Config) chainFinalityDepth() uint32 { + return c.reorgSafetyDepth() + 1 +} + // transportEndpoints holds the gRPC and REST addresses for one service. type transportEndpoints struct { grpc string diff --git a/waved/config_reorg_safety_test.go b/waved/config_reorg_safety_test.go new file mode 100644 index 000000000..c78e196ff --- /dev/null +++ b/waved/config_reorg_safety_test.go @@ -0,0 +1,68 @@ +package waved + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +// TestReorgSafetyDepthResolution pins the inclusive policy boundary and its +// zero-value default. +func TestReorgSafetyDepthResolution(t *testing.T) { + t.Parallel() + + cases := []struct { + name string + configured uint32 + wantHorizon uint32 + wantFinality uint32 + }{ + { + name: "zero uses default", + wantHorizon: DefaultReorgSafetyDepth, + wantFinality: DefaultReorgSafetyDepth + 1, + }, + { + name: "custom", + configured: 42, + wantHorizon: 42, + wantFinality: 43, + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + cfg := &Config{ReorgSafetyDepth: tc.configured} + require.Equal(t, tc.wantHorizon, cfg.reorgSafetyDepth()) + require.Equal( + t, tc.wantFinality, cfg.chainFinalityDepth(), + ) + }) + } +} + +// TestConfigValidateRejectsUnsupportedReorgSafetyDepth verifies that policy +// cannot outlive the shortest current backend observation horizon. +func TestConfigValidateRejectsUnsupportedReorgSafetyDepth(t *testing.T) { + t.Parallel() + + cfg := DefaultConfig() + cfg.Network = "regtest" + cfg.Server.Host = "127.0.0.1:10010" + cfg.Wallet.EsploraURL = "http://127.0.0.1:3000" + cfg.ReorgSafetyDepth = MaxReorgSafetyDepth + 1 + + err := cfg.Validate() + require.ErrorContains(t, err, "reorgsafetydepth exceeds maximum") +} + +// TestDefaultConfigHasReorgSafetyDepth locks in a non-zero policy default. +func TestDefaultConfigHasReorgSafetyDepth(t *testing.T) { + t.Parallel() + + cfg := DefaultConfig() + require.Equal(t, DefaultReorgSafetyDepth, cfg.ReorgSafetyDepth) + require.Equal(t, DefaultReorgSafetyDepth+1, cfg.chainFinalityDepth()) +} diff --git a/waved/operator_negotiation.go b/waved/operator_negotiation.go index a623412c4..5430a7c9c 100644 --- a/waved/operator_negotiation.go +++ b/waved/operator_negotiation.go @@ -75,8 +75,8 @@ func operatorTermsFromResponse(resp *arkrpc.GetInfoResponse) ( } // clientSupportedArkVersions returns the Ark protocol versions this client -// advertises during bootstrap, ordered by preference. Production supports only -// v1; no production default advertises a higher version. +// advertises during bootstrap, ordered by preference. V2 remains deliberately +// absent until the complete one-confirmation safety stack is enabled. func clientSupportedArkVersions() []uint32 { return []uint32{arkrpc.ArkProtocolVersionV1} } diff --git a/waved/operator_negotiation_test.go b/waved/operator_negotiation_test.go index 9439fd4e7..ba71ba643 100644 --- a/waved/operator_negotiation_test.go +++ b/waved/operator_negotiation_test.go @@ -153,6 +153,16 @@ func TestOperatorTermsFreeRefreshWindow(t *testing.T) { require.Equal(t, uint32(72), terms.FreeRefreshWindowBlocks) } +// TestClientSupportedArkVersionsKeepsV2Disabled verifies that merely defining +// the reorg-safety compatibility boundary cannot negotiate it prematurely. +func TestClientSupportedArkVersionsKeepsV2Disabled(t *testing.T) { + t.Parallel() + + versions := clientSupportedArkVersions() + require.Equal(t, []uint32{arkrpc.ArkProtocolVersionV1}, versions) + require.NotContains(t, versions, arkrpc.ArkProtocolVersionV2) +} + // TestNegotiateArkBootstrapZeroSelection proves the client refuses to bootstrap // when the operator returns a zero selection (no common version, or a // pre-versioning server). There is no legacy fallback. diff --git a/waved/server.go b/waved/server.go index f1123770f..35cb8cc4e 100644 --- a/waved/server.go +++ b/waved/server.go @@ -2259,13 +2259,13 @@ func (s *Server) registerChainSourceActor( chainsource.ChainSourceConfig{ Backend: s.chainBackend, System: s.actorSystem, - // Enable height-based Done synthesis at the default - // safety depth. Without this, conf/spend sub-actors - // driven through lndclient (whose Done channel is - // allocated-but-never-written) would never receive a - // finality signal, and reorg-aware consumers like - // txconfirm would leak watch state forever. - FinalityDepth: chainsource.DefaultFinalityDepth, + // Enable height-based Done synthesis just beyond the + // configured recovery horizon. Without this, conf/spend + // sub-actors driven through lndclient (whose Done + // channel is allocated-but-never-written) would never + // receive a finality signal, and reorg-aware consumers + // like txconfirm would leak watch state forever. + FinalityDepth: s.cfg.chainFinalityDepth(), }, ) @@ -5671,6 +5671,35 @@ func (s *Server) initUnrollSubsystem(ctx context.Context, exitObserver = fn.Some[actor.TellOnlyRef[vtxo.ManagerMsg]](ref) }) + // Build a chainsource-backed reconciler factory so every per-target + // actor re-observes persisted positive anchors before restart. The + // current subscription API cannot prove absence, so silence leaves the + // checkpoint unchanged and fails closed; it is never treated as a + // reorg. See unroll/reconcile.go for the safety rationale. + // + // The factory bakes the target outpoint into the chainsource + // caller-ID prefix so two actors that reconcile the same + // shared proof-graph ancestor concurrently land on distinct + // service keys (chainsource keys on (CallerID, Txid, PkScript, + // TargetConfs)); a static prefix would collide. + reconcileLog := s.subLogger("UREC") + probeTimeout := s.unrollReconcileProbeTimeout() + reconcilerFactory := func(target wire.OutPoint, + proof *recovery.Proof) unroll.ChainReconciler { + + return unroll.NewChainSourceReconciler( + unroll.ChainSourceReconcilerConfig{ + ChainSource: chainSourceRef, + Proof: proof, + CallerID: fmt.Sprintf( + "unroll-reconcile-%s", target, + ), + ProbeTimeout: probeTimeout, + Log: fn.Some(reconcileLog), + }, + ) + } + registry := unroll.NewUnrollRegistryActor(unroll.RegistryConfig{ Store: &unroll.DBRegistryStore{ UEStore: ueStore, @@ -5691,6 +5720,9 @@ func (s *Server) initUnrollSubsystem(ctx context.Context, Preimage: preimages, }, VTXOExitObserver: exitObserver, + ChainReconcilerFactory: fn.Some( + unroll.ChainReconcilerFactory(reconcilerFactory), + ), }) s.unrollRegistry = registry s.unrollRegistryRef = fn.Some(registry.Ref()) @@ -6169,6 +6201,20 @@ func (s *Server) unrollMaxFeeRate() int64 { return 0 } +// unrollReconcileProbeTimeout returns the configured per-anchor probe +// timeout for the chainsource-backed restart reconciler, or zero so +// the reconciler falls back to defaultReconcileProbeTimeout. +func (s *Server) unrollReconcileProbeTimeout() time.Duration { + if s.cfg.Unroll != nil && + s.cfg.Unroll.ReconcileProbeTimeoutSec > 0 { + return time.Duration( + s.cfg.Unroll.ReconcileProbeTimeoutSec, + ) * time.Second + } + + return 0 +} + // unrollBumpAfterBlocks returns the configured fee-bump cadence (in // blocks) for the shared txconfirm actor used by the unroll subsystem, // or zero to let txconfirm fall back to DefaultFeeBumpIntervalBlocks. From a599666efebd6da5d3e7aa6f106cae1e934968f6 Mon Sep 17 00:00:00 2001 From: Elle Mouton Date: Fri, 17 Jul 2026 16:06:58 -0700 Subject: [PATCH 4/4] unroll: Make VTXO reliving evidence-based Treat legacy started checkpoints and ambiguous broadcast failures as unsafe. Only an explicit no-broadcast proof may clear the relive guard, so a failed exit cannot restore a coin that may have crossed the chain boundary. --- txconfirm/messages.go | 6 ++++ unroll/AGENTS.md | 5 +-- unroll/CLAUDE.md | 5 +-- unroll/actor.go | 2 ++ unroll/actor_test.go | 64 +++++++++++++++++++++++++++++++++---- unroll/fsm_logic.go | 3 ++ unroll/fsm_types.go | 5 +++ unroll/reorg_safety_test.go | 6 ++++ unroll/snapshot.go | 22 +++++++++---- unroll/snapshot_test.go | 27 ++++++++++++++++ waved/server.go | 7 ++-- 11 files changed, 132 insertions(+), 20 deletions(-) diff --git a/txconfirm/messages.go b/txconfirm/messages.go index 24ecbb7cc..ac23c781f 100644 --- a/txconfirm/messages.go +++ b/txconfirm/messages.go @@ -171,6 +171,12 @@ type EnsureConfirmedResp struct { // Created is true when the request created a new tracking entry and // false when it attached to existing state. Created bool + + // DefinitelyNotBroadcast is true only when the responder can prove + // that no broadcast attempt crossed the chain boundary. Callers may + // use this to distinguish a locally rejected request from an ambiguous + // broadcast failure. The conservative default is false. + DefinitelyNotBroadcast bool } // MessageType returns the stable message type identifier. diff --git a/unroll/AGENTS.md b/unroll/AGENTS.md index 6420ac977..95cbd2409 100644 --- a/unroll/AGENTS.md +++ b/unroll/AGENTS.md @@ -71,8 +71,9 @@ For field-level detail, use `go doc github.com/lightninglabs/wavelength/unroll.< `ExitOutcomeRecoverable` (roll back to live), a completed exit → `ExitOutcomeConfirmed` (retire to spent). The live actor sets `ReliveUnsafe` before every chain-boundary attempt; only objective - canonical-absence evidence may clear it. A mere rejection, absence, or - timeout therefore leaves the VTXO held in unilateral exit. + canonical-absence evidence or an explicit `DefinitelyNotBroadcast` + response may clear it. An ambiguous rejection, absence, or timeout + therefore leaves the VTXO held in unilateral exit. `UnrollTerminatedMsg` carries `HadOnChainFootprint`, computed by `jobHadOnChainFootprint` (any confirmed/in-flight proof node or a non-pending sweep), and the durable `ReliveUnsafe` guard. It also carries diff --git a/unroll/CLAUDE.md b/unroll/CLAUDE.md index 6420ac977..95cbd2409 100644 --- a/unroll/CLAUDE.md +++ b/unroll/CLAUDE.md @@ -71,8 +71,9 @@ For field-level detail, use `go doc github.com/lightninglabs/wavelength/unroll.< `ExitOutcomeRecoverable` (roll back to live), a completed exit → `ExitOutcomeConfirmed` (retire to spent). The live actor sets `ReliveUnsafe` before every chain-boundary attempt; only objective - canonical-absence evidence may clear it. A mere rejection, absence, or - timeout therefore leaves the VTXO held in unilateral exit. + canonical-absence evidence or an explicit `DefinitelyNotBroadcast` + response may clear it. An ambiguous rejection, absence, or timeout + therefore leaves the VTXO held in unilateral exit. `UnrollTerminatedMsg` carries `HadOnChainFootprint`, computed by `jobHadOnChainFootprint` (any confirmed/in-flight proof node or a non-pending sweep), and the durable `ReliveUnsafe` guard. It also carries diff --git a/unroll/actor.go b/unroll/actor.go index db9c8663c..59c63eaa3 100644 --- a/unroll/actor.go +++ b/unroll/actor.go @@ -1002,6 +1002,8 @@ func (b *behavior) ensureNodeConfirmed(ctx context.Context, return b.driveEvent(ctx, ax, &TxFailedEvent{ Txid: txid, + DefinitelyNotBroadcast: ensureResp. + DefinitelyNotBroadcast, Reason: b.failureReasonForTx( txid, "txconfirm returned failed state", ), diff --git a/unroll/actor_test.go b/unroll/actor_test.go index 59922ac3d..47a82da98 100644 --- a/unroll/actor_test.go +++ b/unroll/actor_test.go @@ -137,10 +137,11 @@ func (m *mockVTXOStore) DeleteVTXO(context.Context, wire.OutPoint) error { type fakeTxConfirmRef struct { mu sync.Mutex - requests []*txconfirm.EnsureConfirmedReq - responseStates map[chainhash.Hash]txconfirm.TxState - confirmHeights map[chainhash.Hash]int32 - failureReasons map[chainhash.Hash]string + requests []*txconfirm.EnsureConfirmedReq + responseStates map[chainhash.Hash]txconfirm.TxState + confirmHeights map[chainhash.Hash]int32 + failureReasons map[chainhash.Hash]string + definitelyNotBroadcast map[chainhash.Hash]bool // onAsk, when set, is invoked with each EnsureConfirmedReq as it is // recorded (outside the store lock). Tests use it to assert ordering @@ -188,6 +189,8 @@ func (f *fakeTxConfirmRef) Ask(_ context.Context, f.requests = append(f.requests, req) state := f.responseStates[req.Tx.TxHash()] height := f.confirmHeights[req.Tx.TxHash()] + definitelyNotBroadcast := + f.definitelyNotBroadcast[req.Tx.TxHash()] onAsk := f.onAsk f.mu.Unlock() @@ -226,9 +229,10 @@ func (f *fakeTxConfirmRef) Ask(_ context.Context, promise.Complete( fn.Ok[txconfirm.Resp]( &txconfirm.EnsureConfirmedResp{ - Txid: req.Tx.TxHash(), - State: state, - Created: true, + Txid: req.Tx.TxHash(), + State: state, + Created: true, + DefinitelyNotBroadcast: definitelyNotBroadcast, }, ), ) @@ -343,6 +347,23 @@ func (f *fakeTxConfirmRef) setImmediateFailed(txid chainhash.Hash, f.failureReasons[txid] = reason } +// setImmediateDefiniteNoBroadcast configures one txid to fail before any +// broadcast attempt crosses the chain boundary. +func (f *fakeTxConfirmRef) setImmediateDefiniteNoBroadcast(txid chainhash.Hash, + reason string) { + + f.setImmediateFailed(txid, reason) + + f.mu.Lock() + defer f.mu.Unlock() + + if f.definitelyNotBroadcast == nil { + f.definitelyNotBroadcast = make(map[chainhash.Hash]bool) + } + + f.definitelyNotBroadcast[txid] = true +} + // emitConfirmed delivers a txconfirm success notification to the subscriber. func (f *fakeTxConfirmRef) emitConfirmed(t *testing.T, index int, txid chainhash.Hash, height int32) { @@ -2980,6 +3001,35 @@ func TestProofTxFailureTransitionsToFailed(t *testing.T) { " failed: txconfirm returned failed state", checkpoint.Fail, ) + require.True(t, checkpoint.ReliveUnsafe) +} + +// TestDefiniteNoBroadcastClearsReliveGuard proves that an explicit local +// rejection may recover the VTXO while an ambiguous TxStateFailed response +// remains fail-closed. +func TestDefiniteNoBroadcastClearsReliveGuard(t *testing.T) { + proof := buildLinearProof(t) + desc := testDescriptor(t, proof.TargetOutpoint(), proof.CSVDelay()) + rootTxid := proof.RootTxids()[0] + unrollActor, _, txconfirmRef, store := newActorHarness(t, proof, desc) + txconfirmRef.setImmediateDefiniteNoBroadcast(rootTxid, "rejected") + + mustAsk(t, unrollActor.Ref(), &StartUnrollRequest{ + Height: 100, + Trigger: TriggerManual, + }) + + require.Eventually(t, func() bool { + stateResp, ok := mustAsk( + t, unrollActor.Ref(), &GetStateRequest{}, + ).(*GetStateResp) + require.True(t, ok) + + return stateResp.Phase == PhaseFailed + }, testTimeout, 10*time.Millisecond) + + checkpoint := mustDecodeCheckpoint(t, store, "unroll-test") + require.False(t, checkpoint.ReliveUnsafe) } // TestResumeReissuesSweepConfirmation verifies that resume reattaches diff --git a/unroll/fsm_logic.go b/unroll/fsm_logic.go index 8d592facf..e34df3ebf 100644 --- a/unroll/fsm_logic.go +++ b/unroll/fsm_logic.go @@ -501,6 +501,9 @@ func applyFailedEvent(job *JobState, event *TxFailedEvent) { job.DeferredCheckpoints = removeDeferredCheckpoint( job.DeferredCheckpoints, event.Txid, ) + if event.DefinitelyNotBroadcast { + job.ReliveUnsafe = false + } job.FailReason = event.Reason } diff --git a/unroll/fsm_types.go b/unroll/fsm_types.go index 3eacb12f2..274a25382 100644 --- a/unroll/fsm_types.go +++ b/unroll/fsm_types.go @@ -235,6 +235,11 @@ type TxFailedEvent struct { // Reason is the stable human-readable failure reason. Reason string + + // DefinitelyNotBroadcast proves that the failed transaction never + // crossed the chain boundary. Only this explicit evidence may clear + // the fail-closed relive guard. + DefinitelyNotBroadcast bool } // eventSealed marks TxFailedEvent as an FSM event. diff --git a/unroll/reorg_safety_test.go b/unroll/reorg_safety_test.go index 0425270bb..d4ea9af59 100644 --- a/unroll/reorg_safety_test.go +++ b/unroll/reorg_safety_test.go @@ -214,6 +214,12 @@ func (r *recordingRegistryRef) Tell(_ context.Context, msg RegistryMsg) error { return nil } +func (r *recordingRegistryRef) TryTell(ctx context.Context, + msg RegistryMsg) error { + + return r.Tell(ctx, msg) +} + // terminatedCount returns how many UnrollTerminatedMsg messages have // been delivered so far. func (r *recordingRegistryRef) terminatedCount() int { diff --git a/unroll/snapshot.go b/unroll/snapshot.go index e5ae92ed2..7fb6483c8 100644 --- a/unroll/snapshot.go +++ b/unroll/snapshot.go @@ -121,10 +121,11 @@ const ( // recoverable failure. checkpointExternalSpendFinalizedRecordType tlv.Type = 27 - // checkpointReliveUnsafeRecordType is optional; present (value 1) - // before a job issues or reissues chain-boundary work. Until - // authoritative negative evidence clears the guard, a later failure - // must hold the VTXO in exit rather than relive it. + // checkpointReliveUnsafeRecordType is present for every checkpoint + // whose actor has started. A value of 1 means chain-boundary work may + // have escaped; 0 records an explicit clear by authoritative negative + // evidence. Started checkpoints written before this record existed + // omit it and decode fail-closed as unsafe. checkpointReliveUnsafeRecordType tlv.Type = 29 ) @@ -329,8 +330,11 @@ func encodeCheckpoint(value *actorCheckpoint) ([]byte, error) { ) } - if value.ReliveUnsafe { - unsafe := uint8(1) + if value.Started { + unsafe := uint8(0) + if value.ReliveUnsafe { + unsafe = 1 + } records = append( records, tlv.MakePrimitiveRecord( checkpointReliveUnsafeRecordType, &unsafe, @@ -514,6 +518,12 @@ func decodeCheckpoint(raw []byte) (*actorCheckpoint, error) { if _, ok := parsed[checkpointReliveUnsafeRecordType]; ok { checkpoint.ReliveUnsafe = reliveUnsafe != 0 + } else if checkpoint.Started { + // Checkpoints written before the guard was added have no + // record. A started job may already have crossed a chain + // boundary, so absence cannot be treated as evidence that + // reliving is safe. + checkpoint.ReliveUnsafe = true } return checkpoint, nil diff --git a/unroll/snapshot_test.go b/unroll/snapshot_test.go index 964abece6..ab89ba75b 100644 --- a/unroll/snapshot_test.go +++ b/unroll/snapshot_test.go @@ -222,6 +222,33 @@ func TestCheckpointCodecVersionMismatch(t *testing.T) { require.ErrorContains(t, err, "unsupported checkpoint version") } +// TestCheckpointCodecLegacyStartedDefaultsReliveUnsafe proves an in-flight +// checkpoint written before the relive guard existed cannot decode fail-open +// at the upgrade boundary. +func TestCheckpointCodecLegacyStartedDefaultsReliveUnsafe(t *testing.T) { + checkpoint := &actorCheckpoint{ + Version: checkpointVersion, + Started: true, + Trigger: TriggerManual, + } + raw, err := encodeCheckpoint(checkpoint) + require.NoError(t, err) + + // Record type 29 is the final canonical record and uses one-byte TLV + // type/length encodings. Removing it reproduces the pre-field wire + // shape while retaining Started=true. + reliveRecord := []byte{ + byte(checkpointReliveUnsafeRecordType), 1, 0, + } + require.True(t, bytes.HasSuffix(raw, reliveRecord)) + raw = raw[:len(raw)-len(reliveRecord)] + + decoded, err := decodeCheckpoint(raw) + require.NoError(t, err) + require.True(t, decoded.Started) + require.True(t, decoded.ReliveUnsafe) +} + // TestCheckpointCodecCorruptDataRejected asserts that malformed input (empty, // truncated, random garbage) is rejected with an error rather than panicking // or returning a zero-value checkpoint. diff --git a/waved/server.go b/waved/server.go index 35cb8cc4e..e02609fb8 100644 --- a/waved/server.go +++ b/waved/server.go @@ -6111,9 +6111,10 @@ func (f *faultyUnrollTxConfirm) Ask(ctx context.Context, promise.Complete( fn.Ok[txconfirm.Resp]( &txconfirm.EnsureConfirmedResp{ - Txid: req.Tx.TxHash(), - State: txconfirm.TxStateFailed, - Created: true, + Txid: req.Tx.TxHash(), + State: txconfirm.TxStateFailed, + Created: true, + DefinitelyNotBroadcast: true, }, ), )