persist envelop and recover full state by it

Author: domiwei

cl/phase1/forkchoice/fork_graph/fork_graph_disk.go

@@ -495,6 +495,109 @@ func (f *forkGraphDisk) hasBeaconState(blockRoot common.Hash) bool {
 	return err == nil && exists
 }
 
+// GetExecutionPayloadState reconstructs the post-execution-payload state by replaying.
+// Similar to getState() but handles GLOAS FULL/EMPTY paths.
+// [New in Gloas:EIP7732]
+func (f *forkGraphDisk) GetExecutionPayloadState(blockRoot common.Hash) (*state.CachingBeaconState, error) {
+	// 1. Read target envelope
+	targetEnvelope, err := f.ReadEnvelopeFromDisk(blockRoot)
+	if err != nil {
+		return nil, fmt.Errorf("GetExecutionPayloadState: failed to read envelope: %w", err)
+	}
+
+	// 2. Get block_state for the target block (handles FULL/EMPTY paths)
+	blockState, err := f.getStateWithEnvelopes(blockRoot)
+	if err != nil {
+		return nil, fmt.Errorf("GetExecutionPayloadState: failed to get block state: %w", err)
+	}
+
+	// 3. Apply target envelope to get exec_payload_state
+	if err := transition.DefaultMachine.ProcessExecutionPayloadEnvelope(blockState, targetEnvelope); err != nil {
+		return nil, fmt.Errorf("GetExecutionPayloadState: failed to process envelope: %w", err)
+	}
+
+	return blockState, nil
+}
+
+// getStateWithEnvelopes is like getState but handles GLOAS FULL/EMPTY parent paths.
+// When replaying blocks, if a block was built on parent's FULL path, applies parent's envelope first.
+// [New in Gloas:EIP7732]
+func (f *forkGraphDisk) getStateWithEnvelopes(blockRoot common.Hash) (*state.CachingBeaconState, error) {
+	// Collect blocks with their parent path info
+	type blockInfo struct {
+		block          *cltypes.SignedBeaconBlock
+		parentWasFull  bool
+		parentEnvelope *cltypes.SignedExecutionPayloadEnvelope
+	}
+	blocksInTheWay := []blockInfo{}
+	currentRoot := blockRoot
+	var checkpointState *state.CachingBeaconState
+
+	// Walk back to find checkpoint
+	for checkpointState == nil {
+		block, ok := f.GetBlock(currentRoot)
+		if !ok {
+			// Check if it's a header-only checkpoint
+			bHeader, headerOk := f.GetHeader(currentRoot)
+			if headerOk && bHeader.Slot%dumpSlotFrequency == 0 {
+				checkpointState, _ = f.readBeaconStateFromDisk(currentRoot)
+			}
+			if checkpointState != nil {
+				break
+			}
+			return nil, fmt.Errorf("getStateWithEnvelopes: block not found: %x", currentRoot)
+		}
+
+		// Try to read checkpoint state
+		if block.Block.Slot%dumpSlotFrequency == 0 {
+			checkpointState, _ = f.readBeaconStateFromDisk(currentRoot)
+			if checkpointState != nil {
+				break
+			}
+		}
+
+		// Check if this block was built on parent's FULL path
+		parentRoot := block.Block.ParentRoot
+		parentEnvelope, _ := f.ReadEnvelopeFromDisk(parentRoot)
+		parentWasFull := false
+		if parentEnvelope != nil && block.Block.Body.SignedExecutionPayloadBid != nil &&
+			block.Block.Body.SignedExecutionPayloadBid.Message != nil {
+			// Compare block's bid.ParentBlockHash with parent envelope's payload.BlockHash
+			bidHash := block.Block.Body.SignedExecutionPayloadBid.Message.ParentBlockHash
+			envHash := parentEnvelope.Message.Payload.BlockHash
+			parentWasFull = (bidHash == envHash)
+		}
+
+		blocksInTheWay = append(blocksInTheWay, blockInfo{
+			block:          block,
+			parentWasFull:  parentWasFull,
+			parentEnvelope: parentEnvelope,
+		})
+		currentRoot = parentRoot
+	}
+
+	// Replay forward from checkpoint
+	// Note: checkpointState is always a block_state (from DumpBeaconStateOnDisk)
+	replayState := checkpointState
+	for i := len(blocksInTheWay) - 1; i >= 0; i-- {
+		bi := blocksInTheWay[i]
+
+		// If this block was built on parent's FULL path, apply parent envelope first
+		if bi.parentWasFull && bi.parentEnvelope != nil {
+			if err := transition.DefaultMachine.ProcessExecutionPayloadEnvelope(replayState, bi.parentEnvelope); err != nil {
+				return nil, fmt.Errorf("getStateWithEnvelopes: failed to process parent envelope: %w", err)
+			}
+		}
+
+		// Apply process_block to get this block's block_state
+		if err := transition.TransitionState(replayState, bi.block, nil, false); err != nil {
+			return nil, fmt.Errorf("getStateWithEnvelopes: failed to transition state: %w", err)
+		}
+	}
+
+	return replayState, nil
+}
+
 func (f *forkGraphDisk) Prune(pruneSlot uint64) (err error) {
 	oldRoots := make([]common.Hash, 0, f.beaconCfg.SlotsPerEpoch)
 	highestStoredBeaconStateSlot := uint64(0)
@@ -529,6 +632,8 @@ func (f *forkGraphDisk) Prune(pruneSlot uint64) (err error) {
 		f.headers.Delete(root)
 		f.blockRewards.Delete(root)
 		f.fs.Remove(getBeaconStateFilename(root))
+		// [New in Gloas:EIP7732] Also remove envelope files
+		f.fs.Remove(getEnvelopeFilename(root))
 	}
 	log.Debug("Pruned old blocks", "pruneSlot", pruneSlot)
 	return

cl/phase1/forkchoice/fork_graph/fork_graph_disk_fs.go

@@ -25,6 +25,8 @@ import (
 	"github.com/golang/snappy"
 	"github.com/spf13/afero"
 
+	"github.com/erigontech/erigon/cl/clparams"
+	"github.com/erigontech/erigon/cl/cltypes"
 	"github.com/erigontech/erigon/cl/phase1/core/state"
 	"github.com/erigontech/erigon/common"
 	"github.com/erigontech/erigon/common/log/v3"
@@ -34,6 +36,12 @@ func getBeaconStateFilename(blockRoot common.Hash) string {
 	return fmt.Sprintf("%x.snappy_ssz", blockRoot)
 }
 
+// getEnvelopeFilename returns the filename for execution payload envelopes.
+// [New in Gloas:EIP7732]
+func getEnvelopeFilename(blockRoot common.Hash) string {
+	return fmt.Sprintf("%x.envelope.snappy_ssz", blockRoot)
+}
+
 func (f *forkGraphDisk) readBeaconStateFromDisk(blockRoot common.Hash) (bs *state.CachingBeaconState, err error) {
 	var file afero.File
 	f.stateDumpLock.Lock()
@@ -136,3 +144,104 @@ func (f *forkGraphDisk) DumpBeaconStateOnDisk(blockRoot common.Hash, bs *state.C
 
 	return
 }
+
+// HasEnvelope checks if an envelope exists on disk for the given block root.
+// [New in Gloas:EIP7732]
+func (f *forkGraphDisk) HasEnvelope(blockRoot common.Hash) bool {
+	exists, err := afero.Exists(f.fs, getEnvelopeFilename(blockRoot))
+	return err == nil && exists
+}
+
+// ReadEnvelopeFromDisk reads an execution payload envelope from disk.
+// [New in Gloas:EIP7732]
+func (f *forkGraphDisk) ReadEnvelopeFromDisk(blockRoot common.Hash) (envelope *cltypes.SignedExecutionPayloadEnvelope, err error) {
+	var file afero.File
+	f.stateDumpLock.Lock()
+	defer f.stateDumpLock.Unlock()
+
+	file, err = f.fs.Open(getEnvelopeFilename(blockRoot))
+	if err != nil {
+		return
+	}
+	defer file.Close()
+
+	if f.sszSnappyReader == nil {
+		f.sszSnappyReader = snappy.NewReader(file)
+	} else {
+		f.sszSnappyReader.Reset(file)
+	}
+
+	// Read the length
+	lengthBytes := make([]byte, 8)
+	var n int
+	n, err = io.ReadFull(f.sszSnappyReader, lengthBytes)
+	if err != nil {
+		return nil, fmt.Errorf("failed to read length: %w, root: %x", err, blockRoot)
+	}
+	if n != 8 {
+		return nil, fmt.Errorf("failed to read length: %d, want 8, root: %x", n, blockRoot)
+	}
+
+	f.sszBuffer = f.sszBuffer[:binary.BigEndian.Uint64(lengthBytes)]
+	n, err = io.ReadFull(f.sszSnappyReader, f.sszBuffer)
+	if err != nil {
+		return nil, fmt.Errorf("failed to read snappy buffer: %w, root: %x", err, blockRoot)
+	}
+	f.sszBuffer = f.sszBuffer[:n]
+
+	envelope = &cltypes.SignedExecutionPayloadEnvelope{}
+	if err = envelope.DecodeSSZ(f.sszBuffer, int(clparams.GloasVersion)); err != nil {
+		return nil, fmt.Errorf("failed to decode envelope: %w, root: %x, len: %d", err, blockRoot, n)
+	}
+
+	return
+}
+
+// DumpEnvelopeOnDisk dumps an execution payload envelope to disk.
+// [New in Gloas:EIP7732]
+func (f *forkGraphDisk) DumpEnvelopeOnDisk(blockRoot common.Hash, envelope *cltypes.SignedExecutionPayloadEnvelope) (err error) {
+	f.stateDumpLock.Lock()
+	defer f.stateDumpLock.Unlock()
+
+	// Encode the envelope
+	f.sszBuffer, err = envelope.EncodeSSZ(f.sszBuffer[:0])
+	if err != nil {
+		return
+	}
+
+	dumpedFile, err := f.fs.OpenFile(getEnvelopeFilename(blockRoot), os.O_TRUNC|os.O_CREATE|os.O_RDWR, 0o755)
+	if err != nil {
+		return err
+	}
+	defer dumpedFile.Close()
+
+	if f.sszSnappyWriter == nil {
+		f.sszSnappyWriter = snappy.NewBufferedWriter(dumpedFile)
+	} else {
+		f.sszSnappyWriter.Reset(dumpedFile)
+	}
+
+	// Write the length
+	length := make([]byte, 8)
+	binary.BigEndian.PutUint64(length, uint64(len(f.sszBuffer)))
+	if _, err := f.sszSnappyWriter.Write(length); err != nil {
+		log.Error("failed to write length", "err", err)
+		return err
+	}
+	// Write the envelope
+	if _, err := f.sszSnappyWriter.Write(f.sszBuffer); err != nil {
+		log.Error("failed to write ssz buffer", "err", err)
+		return err
+	}
+	if err = f.sszSnappyWriter.Flush(); err != nil {
+		log.Error("failed to flush snappy writer", "err", err)
+		return err
+	}
+
+	if err = dumpedFile.Sync(); err != nil {
+		log.Error("failed to sync dumped file", "err", err)
+		return
+	}
+
+	return
+}

cl/phase1/forkchoice/fork_graph/interface.go

@@ -58,4 +58,10 @@ type ForkGraph interface {
 	GetCurrentParticipationIndicies(epoch uint64) (*solid.ParticipationBitList, error)
 	GetPreviousParticipationIndicies(epoch uint64) (*solid.ParticipationBitList, error)
 	DumpBeaconStateOnDisk(blockRoot common.Hash, state *state.CachingBeaconState, forced bool) error
+	// [New in Gloas:EIP7732] Execution payload envelope persistence and state reconstruction
+	DumpEnvelopeOnDisk(blockRoot common.Hash, envelope *cltypes.SignedExecutionPayloadEnvelope) error
+	ReadEnvelopeFromDisk(blockRoot common.Hash) (*cltypes.SignedExecutionPayloadEnvelope, error)
+	HasEnvelope(blockRoot common.Hash) bool
+	// GetExecutionPayloadState reconstructs the post-execution-payload state by replaying the envelope.
+	GetExecutionPayloadState(blockRoot common.Hash) (*state.CachingBeaconState, error)
 }

cl/phase1/forkchoice/forkchoice.go

@@ -17,7 +17,6 @@
 package forkchoice
 
 import (
-	"fmt"
 	"slices"
 	"sort"
 	"sync"
@@ -148,10 +147,6 @@ type ForkChoiceStore struct {
 	optimisticStore         optimistic.OptimisticStore
 	probabilisticHeadGetter bool
 
-	// [New in Gloas:EIP7732] Stores post-execution-payload states keyed by beacon block root.
-	// In GLOAS, beacon block and execution payload have separate state transitions.
-	// This stores the state after ProcessExecutionPayloadEnvelope is applied.
-	executionPayloadStates sync.Map // map[common.Hash]*state.CachingBeaconState
 	// [New in Gloas:EIP7732]
 	ptcVote sync.Map // map[common.Hash][clparams.PtcSize]bool
 	// [New in Gloas:EIP7732] Indexed weight store for optimized weight calculation
@@ -302,12 +297,6 @@ func NewForkChoiceStore(
 	f.highestSeen.Store(anchorState.Slot())
 	f.time.Store(anchorState.GenesisTime() + anchorState.BeaconConfig().SecondsPerSlot*anchorState.Slot())
 
-	// [New in Gloas:EIP7732] Initialize anchor root entries with anchor state copy
-	anchorStateCopy, err := anchorState.Copy()
-	if err != nil {
-		return nil, fmt.Errorf("failed to copy anchor state for execution payload states: %w", err)
-	}
-	f.executionPayloadStates.Store(anchorRoot, anchorStateCopy)
 	f.ptcVote.Store(anchorRoot, [clparams.PtcSize]bool{})
 
 	// [New in Gloas:EIP7732] Initialize indexed weight store

cl/phase1/forkchoice/get_head.go

@@ -119,7 +119,7 @@ func (f *ForkChoiceStore) GetHead(auxilliaryState *state.CachingBeaconState) (co
 	if f.beaconCfg.GetCurrentStateVersion(currentEpoch) >= clparams.GloasVersion {
 		return f.getHeadGloas()
 	}
-	return f.getHeadPreGloas(auxilliaryState)
+	return f.getHead(auxilliaryState)
 }
 
 // getHeadGloas returns the head using GLOAS fork choice rules.
@@ -192,8 +192,8 @@ func (f *ForkChoiceStore) getHeadGloas() (common.Hash, uint64, error) {
 	}
 }
 
-// getHeadPreGloas returns the head using pre-GLOAS fork choice rules.
-func (f *ForkChoiceStore) getHeadPreGloas(auxilliaryState *state.CachingBeaconState) (common.Hash, uint64, error) {
+// getHead returns the head using pre-GLOAS fork choice rules.
+func (f *ForkChoiceStore) getHead(auxilliaryState *state.CachingBeaconState) (common.Hash, uint64, error) {
 	justifiedCheckpoint := f.justifiedCheckpoint.Load().(solid.Checkpoint)
 	var justificationState *checkpointState
 	var err error

cl/phase1/forkchoice/on_block.go

@@ -217,8 +217,15 @@ func (f *ForkChoiceStore) OnBlock(ctx context.Context, block *cltypes.SignedBeac
 	// This ensures the new block builds on the correct canonical state.
 	var parentFullState *state.CachingBeaconState
 	if isGloas && f.isParentNodeFull(block.Block) {
-		if s, ok := f.executionPayloadStates.Load(block.Block.ParentRoot); ok {
-			parentFullState = s.(*state.CachingBeaconState)
+		// Check disk for envelope existence (handles both normal operation and restart recovery)
+		if f.forkGraph.HasEnvelope(block.Block.ParentRoot) {
+			// Reconstruct the execution payload state from disk
+			parentFullState, err = f.forkGraph.GetExecutionPayloadState(block.Block.ParentRoot)
+			if err != nil {
+				log.Warn("Failed to get execution payload state for parent", "parentRoot", block.Block.ParentRoot, "err", err)
+				// Fall back to block_state via AddChainSegment's normal path
+				parentFullState = nil
+			}
 		}
 	}
 

cl/phase1/forkchoice/on_execution_payload.go

@@ -62,12 +62,12 @@ func (f *ForkChoiceStore) OnExecutionPayload(ctx context.Context, signedEnvelope
 	if err := transition.DefaultMachine.ProcessExecutionPayloadEnvelope(blockStateCopy, signedEnvelope); err != nil {
 		return fmt.Errorf("OnExecutionPayload: failed to process execution payload: %w", err)
 	}
-
-	// Store the post-execution-payload state.
-	// In GLOAS, beacon block and execution payload have separate state transitions.
-	// When the head has PayloadStatus=FULL, this state becomes the canonical state.
-	// TODO: Persistently store the execution payload states
-	f.executionPayloadStates.Store(beaconBlockRoot, blockStateCopy)
+	// Persist envelope to disk for recovery after restart.
+	// The full state can be reconstructed via GetExecutionPayloadState() which replays the envelope.
+	// HasEnvelope() checks disk for existence, replacing in-memory tracking.
+	if err := f.forkGraph.DumpEnvelopeOnDisk(beaconBlockRoot, signedEnvelope); err != nil {
+		return fmt.Errorf("OnExecutionPayload: failed to dump envelope: %w", err)
+	}
 
 	return nil
 }

cl/phase1/forkchoice/payload_vote.go

@@ -62,7 +62,7 @@ func (f *ForkChoiceStore) isPayloadTimely(root common.Hash) bool {
 
 	// If the payload is not locally available, the payload
 	// is not considered available regardless of the PTC vote
-	if _, ok := f.executionPayloadStates.Load(root); !ok {
+	if !f.forkGraph.HasEnvelope(root) {
 		return false
 	}
 
@@ -224,7 +224,8 @@ func (f *ForkChoiceStore) getNodeChildren(node ForkChoiceNode, blocks map[common
 		children := []ForkChoiceNode{
 			{Root: node.Root, PayloadStatus: cltypes.PayloadStatusEmpty},
 		}
-		if _, ok := f.executionPayloadStates.Load(node.Root); ok {
+		// Check disk for envelope existence to determine if FULL status is available
+		if f.forkGraph.HasEnvelope(node.Root) {
 			children = append(children, ForkChoiceNode{
 				Root: node.Root, PayloadStatus: cltypes.PayloadStatusFull,
 			})
@@ -259,8 +260,8 @@ func (f *ForkChoiceStore) getNodeChildren(node ForkChoiceNode, blocks map[common
 // [New in Gloas:EIP7732]
 func (f *ForkChoiceStore) validateParentPayloadPath(block *cltypes.BeaconBlock) error {
 	if f.isParentNodeFull(block) {
-		// Parent is FULL - verify execution payload state exists
-		if _, ok := f.executionPayloadStates.Load(block.ParentRoot); !ok {
+		// Parent is FULL - verify execution payload envelope exists on disk
+		if !f.forkGraph.HasEnvelope(block.ParentRoot) {
 			return errors.New("parent execution payload state not found for FULL parent")
 		}
 	} else {

cl/phase1/forkchoice/utils.go

@@ -78,7 +78,8 @@ func (f *ForkChoiceStore) onNewFinalized(newFinalized solid.Checkpoint) {
 		return true
 	})
 
-	// [New in Gloas:EIP7732] Clean up ptcVote and executionPayloadStates for finalized blocks
+	// [New in Gloas:EIP7732] Clean up ptcVote for finalized blocks
+	// Note: envelope files are cleaned up in forkGraph.Prune()
 	finalizedSlot := newFinalized.Epoch * f.beaconCfg.SlotsPerEpoch
 	f.ptcVote.Range(func(k, v any) bool {
 		root := k.(common.Hash)
@@ -87,13 +88,6 @@ func (f *ForkChoiceStore) onNewFinalized(newFinalized solid.Checkpoint) {
 		}
 		return true
 	})
-	f.executionPayloadStates.Range(func(k, v any) bool {
-		root := k.(common.Hash)
-		if header, has := f.forkGraph.GetHeader(root); !has || header.Slot <= finalizedSlot {
-			f.executionPayloadStates.Delete(k)
-		}
-		return true
-	})
 
 	slotToPrune := ((newFinalized.Epoch - 3) * f.beaconCfg.SlotsPerEpoch) - 1
 	f.forkGraph.Prune(slotToPrune)