core, params: transition tree initialization

core/blockchain_reader.go

@@ -32,6 +32,8 @@ import (
 	"github.com/ethereum/go-ethereum/event"
 	"github.com/ethereum/go-ethereum/params"
 	"github.com/ethereum/go-ethereum/rlp"
+	"github.com/ethereum/go-ethereum/trie"
+	"github.com/ethereum/go-ethereum/trie/bintrie"
 	"github.com/ethereum/go-ethereum/triedb"
 )
 
@@ -370,8 +372,17 @@ func (bc *BlockChain) TxIndexDone() bool {
 }
 
 // HasState checks if state trie is fully present in the database or not.
+// It avoids using OpenTrie which has transition-aware logic that may try
+// to open a binary tree before it exists. Instead, it directly attempts
+// to decode the root node as an MPT first, and if that fails, as a binary
+// trie.
 func (bc *BlockChain) HasState(hash common.Hash) bool {
-	_, err := bc.statedb.OpenTrie(hash)
+	// Try to open as a Merkle Patricia Trie first.
+	if _, err := trie.NewStateTrie(trie.StateTrieID(hash), bc.triedb); err == nil {
+		return true
+	}
+	// Fall back to trying as a binary trie.
+	_, err := bintrie.NewBinaryTrie(hash, bc.triedb)
 	return err == nil
 }
 

core/chain_makers.go

@@ -396,6 +396,9 @@ func GenerateChain(config *params.ChainConfig, parent *types.Block, engine conse
 			evm := vm.NewEVM(blockContext, statedb, cm.config, vm.Config{})
 			ProcessParentBlockHash(b.header.ParentHash, evm)
 		}
+		if config.IsVerkle(b.header.Number, b.header.Time) {
+			InitializeBinaryTransitionRegistry(statedb)
+		}
 
 		// Execute any user modifications to the block
 		if gen != nil {

core/overlay/state_transition.go

@@ -17,17 +17,21 @@
 package overlay
 
 import (
-	"bytes"
-	"encoding/gob"
-
 	"github.com/ethereum/go-ethereum/common"
-	"github.com/ethereum/go-ethereum/core/rawdb"
-	"github.com/ethereum/go-ethereum/ethdb"
-	"github.com/ethereum/go-ethereum/log"
 )
 
 // TransitionState is a structure that holds the progress markers of the
 // translation process.
+// TODO gballet:
+// * see if I can get rid of the pointer now that this piece
+// has been rewritten.
+// * the conversion pointers should no longer be necessary,
+// remove them when it's been confirmed.
+// * we can't keep the preimage offset in the file, since
+// some clients might decide to record their preimages and
+// skip the use of the file altogether. Therefore, they can't
+// know what the offset it, unless they keep track of how many
+// bytes have been read since the start, which is a possibility.
 type TransitionState struct {
 	CurrentAccountAddress *common.Address // addresss of the last translated account
 	CurrentSlotHash       common.Hash     // hash of the last translated storage slot
@@ -68,39 +72,3 @@ func (ts *TransitionState) Copy() *TransitionState {
 	}
 	return ret
 }
-
-// LoadTransitionState retrieves the Verkle transition state associated with
-// the given state root hash from the database.
-func LoadTransitionState(db ethdb.KeyValueReader, root common.Hash, isVerkle bool) *TransitionState {
-	var ts *TransitionState
-
-	data, _ := rawdb.ReadVerkleTransitionState(db, root)
-
-	// if a state could be read from the db, attempt to decode it
-	if len(data) > 0 {
-		var (
-			newts TransitionState
-			buf   = bytes.NewBuffer(data[:])
-			dec   = gob.NewDecoder(buf)
-		)
-		// Decode transition state
-		err := dec.Decode(&newts)
-		if err != nil {
-			log.Error("failed to decode transition state", "err", err)
-			return nil
-		}
-		ts = &newts
-	}
-
-	// Fallback that should only happen before the transition
-	if ts == nil {
-		// Initialize the first transition state, with the "ended"
-		// field set to true if the database was created
-		// as a verkle database.
-		log.Debug("no transition state found, starting fresh", "verkle", isVerkle)
-
-		// Start with a fresh state
-		ts = &TransitionState{Ended: isVerkle}
-	}
-	return ts
-}

core/state/database.go

@@ -27,6 +27,7 @@ import (
 	"github.com/ethereum/go-ethereum/core/types"
 	"github.com/ethereum/go-ethereum/crypto"
 	"github.com/ethereum/go-ethereum/ethdb"
+	"github.com/ethereum/go-ethereum/params"
 	"github.com/ethereum/go-ethereum/trie"
 	"github.com/ethereum/go-ethereum/trie/bintrie"
 	"github.com/ethereum/go-ethereum/trie/transitiontrie"
@@ -177,6 +178,76 @@ func NewDatabaseForTesting() *CachingDB {
 	return NewDatabase(triedb.NewDatabase(rawdb.NewMemoryDatabase(), nil), nil)
 }
 
+var (
+	transitionStartedKey               = common.Hash{}  // slot 0: non-zero if transition started
+	conversionProgressAddressKey       = common.Hash{1} // slot 1: current account pointer
+	conversionProgressSlotKey          = common.Hash{2} // slot 2: current slot pointer
+	conversionProgressStorageProcessed = common.Hash{3} // slot 3: storage processed flag
+	transitionEndedKey                 = common.Hash{4} // slot 4: non-zero if transition ended
+	baseRootKey                        = common.Hash{5} // slot 5: MPT base root at transition start
+)
+
+// isTransitionActive checks if the binary tree transition has been activated
+func isTransitionActive(reader StateReader) bool {
+	val, err := reader.Storage(params.BinaryTransitionRegistryAddress, transitionStartedKey)
+	if err != nil {
+		return false
+	}
+	return val != (common.Hash{})
+}
+
+// LoadTransitionState retrieves the Verkle transition state associated with
+// the given state root hash from the database.
+func LoadTransitionState(reader StateReader, root common.Hash) *overlay.TransitionState {
+	startedBytes, err := reader.Storage(params.BinaryTransitionRegistryAddress, transitionStartedKey)
+	if err != nil {
+		return nil
+	}
+	started := startedBytes != (common.Hash{})
+
+	// If not started, return nil to indicate no active transition
+	if !started {
+		return nil
+	}
+
+	endedBytes, err := reader.Storage(params.BinaryTransitionRegistryAddress, transitionEndedKey)
+	if err != nil {
+		return nil
+	}
+	ended := endedBytes != (common.Hash{})
+
+	currentAccountBytes, err := reader.Storage(params.BinaryTransitionRegistryAddress, conversionProgressAddressKey)
+	if err != nil {
+		return nil
+	}
+	currentAccount := common.BytesToAddress(currentAccountBytes[12:])
+
+	currentSlotHash, err := reader.Storage(params.BinaryTransitionRegistryAddress, conversionProgressSlotKey)
+	if err != nil {
+		return nil
+	}
+
+	storageProcessedBytes, err := reader.Storage(params.BinaryTransitionRegistryAddress, conversionProgressStorageProcessed)
+	if err != nil {
+		return nil
+	}
+	storageProcessed := storageProcessedBytes[0] == 1
+
+	baseRoot, err := reader.Storage(params.BinaryTransitionRegistryAddress, baseRootKey)
+	if err != nil {
+		return nil
+	}
+
+	return &overlay.TransitionState{
+		Started:               started,
+		Ended:                 ended,
+		CurrentAccountAddress: &currentAccount,
+		CurrentSlotHash:       currentSlotHash,
+		StorageProcessed:      storageProcessed,
+		BaseRoot:              baseRoot,
+	}
+}
+
 // StateReader returns a state reader associated with the specified state root.
 func (db *CachingDB) StateReader(stateRoot common.Hash) (StateReader, error) {
 	var readers []StateReader
@@ -190,19 +261,24 @@ func (db *CachingDB) StateReader(stateRoot common.Hash) (StateReader, error) {
 			readers = append(readers, newFlatReader(snap))
 		}
 	}
+	var ts *overlay.TransitionState
 	// Configure the state reader using the path database in path mode.
 	// This reader offers improved performance but is optional and only
 	// partially useful if the snapshot data in path database is not
 	// fully generated.
 	if db.TrieDB().Scheme() == rawdb.PathScheme {
 		reader, err := db.triedb.StateReader(stateRoot)
 		if err == nil {
-			readers = append(readers, newFlatReader(reader))
+			flatReader := newFlatReader(reader)
+			readers = append(readers, flatReader)
+			if isTransitionActive(flatReader) || db.triedb.IsVerkle() {
+				ts = LoadTransitionState(flatReader, stateRoot)
+			}
 		}
 	}
 	// Configure the trie reader, which is expected to be available as the
 	// gatekeeper unless the state is corrupted.
-	tr, err := newTrieReader(stateRoot, db.triedb)
+	tr, err := newTrieReader(stateRoot, db.triedb, ts)
 	if err != nil {
 		return nil, err
 	}
@@ -238,27 +314,51 @@ func (db *CachingDB) ReadersWithCacheStats(stateRoot common.Hash) (ReaderWithSta
 
 // OpenTrie opens the main account trie at a specific root hash.
 func (db *CachingDB) OpenTrie(root common.Hash) (Trie, error) {
-	if db.triedb.IsVerkle() {
-		ts := overlay.LoadTransitionState(db.TrieDB().Disk(), root, db.triedb.IsVerkle())
-		if ts.InTransition() {
-			panic("state tree transition isn't supported yet")
+	// Only attempt transition-aware trie opening in path scheme, since
+	// hashdb does not implement StateReader.
+	if db.TrieDB().Scheme() == rawdb.PathScheme {
+		reader, err := db.triedb.StateReader(root)
+		if err != nil {
+			return nil, err
 		}
-		if ts.Transitioned() {
-			// Use BinaryTrie instead of VerkleTrie when IsVerkle is set
-			// (IsVerkle actually means Binary Trie mode in this codebase)
-			return bintrie.NewBinaryTrie(root, db.triedb)
+		flatReader := newFlatReader(reader)
+
+		ts := LoadTransitionState(flatReader, root)
+		if isTransitionActive(flatReader) || db.triedb.IsVerkle() {
+			fmt.Printf("Opening transition-aware trie for root %s with transition state: %+v\n", root, ts)
+
+			// special case of the tree bootsrap: the root will be that of the MPT, so in that
+			// case, open an empty binary tree.
+			var bt *bintrie.BinaryTrie
+			if ts.BaseRoot == (common.Hash{}) {
+				bt, err = bintrie.NewBinaryTrie(common.Hash{}, db.triedb)
+				if err != nil {
+					return nil, fmt.Errorf("could not bootstrap the overlay tree: %w", err)
+				}
+			} else {
+				bt, err = bintrie.NewBinaryTrie(root, db.triedb)
+				if err != nil {
+					return nil, fmt.Errorf("could not open the overlay tree: %w", err)
+				}
+			}
+			if !ts.InTransition() {
+				// Transition complete, use BinaryTrie only
+				return bt, nil
+			}
+
+			base, err := trie.NewStateTrie(trie.StateTrieID(ts.BaseRoot), db.triedb)
+			if err != nil {
+				return nil, fmt.Errorf("could not create base trie in OpenTrie: %w", err)
+			}
+			return transitiontrie.NewTransitionTrie(base, bt, false), nil
 		}
 	}
-	tr, err := trie.NewStateTrie(trie.StateTrieID(root), db.triedb)
-	if err != nil {
-		return nil, err
-	}
-	return tr, nil
+	return trie.NewStateTrie(trie.StateTrieID(root), db.triedb)
 }
 
 // OpenStorageTrie opens the storage trie of an account.
 func (db *CachingDB) OpenStorageTrie(stateRoot common.Hash, address common.Address, root common.Hash, self Trie) (Trie, error) {
-	if db.triedb.IsVerkle() {
+	if self != nil && self.IsVerkle() {
 		return self, nil
 	}
 	tr, err := trie.NewStateTrie(trie.StorageTrieID(stateRoot, crypto.Keccak256Hash(address.Bytes()), root), db.triedb)

core/state/reader.go

@@ -313,24 +313,28 @@ type trieReader struct {
 
 // newTrieReader constructs a trie reader of the specific state. An error will be
 // returned if the associated trie specified by root is not existent.
-func newTrieReader(root common.Hash, db *triedb.Database) (*trieReader, error) {
+func newTrieReader(root common.Hash, db *triedb.Database, ts *overlay.TransitionState) (*trieReader, error) {
 	var (
 		tr  Trie
 		err error
 	)
-	if !db.IsVerkle() {
+	if !db.IsVerkle() && (ts == nil || !ts.InTransition()) {
 		tr, err = trie.NewStateTrie(trie.StateTrieID(root), db)
 	} else {
-		// When IsVerkle() is true, create a BinaryTrie wrapped in TransitionTrie
-		binTrie, binErr := bintrie.NewBinaryTrie(root, db)
+		var binTrie *bintrie.BinaryTrie
+		var binErr error
+		if ts.BaseRoot == (common.Hash{}) {
+			binTrie, binErr = bintrie.NewBinaryTrie(common.Hash{}, db)
+		} else {
+			binTrie, binErr = bintrie.NewBinaryTrie(root, db)
+		}
 		if binErr != nil {
 			return nil, binErr
 		}
 
 		// Based on the transition status, determine if the overlay
-		// tree needs to be created, or if a single, target tree is
+		// tree needs to be created, or if a single target tree is
 		// to be picked.
-		ts := overlay.LoadTransitionState(db.Disk(), root, true)
 		if ts.InTransition() {
 			mpt, err := trie.NewStateTrie(trie.StateTrieID(ts.BaseRoot), db)
 			if err != nil {

core/state/state_object.go

@@ -147,6 +147,9 @@ func (s *stateObject) getTrie() (Trie, error) {
 func (s *stateObject) getPrefetchedTrie() Trie {
 	// If there's nothing to meaningfully return, let the user figure it out by
 	// pulling the trie from disk.
+	if s.db.trie != nil && s.db.trie.IsVerkle() {
+		return nil
+	}
 	if (s.data.Root == types.EmptyRootHash && !s.db.db.TrieDB().IsVerkle()) || s.db.prefetcher == nil {
 		return nil
 	}

core/state/statedb.go

@@ -349,6 +349,17 @@ func (s *StateDB) GetStorageRoot(addr common.Address) common.Hash {
 	return common.Hash{}
 }
 
+// TransitionComplete checks if the EIP-7612 transition is complete.
+func (s *StateDB) InTransition() bool {
+	completeKey := common.Hash{} // slot 0 for completion flag
+	completeValue := s.GetState(params.BinaryTransitionRegistryAddress, completeKey)
+	return completeValue != (common.Hash{})
+}
+
+func (s *StateDB) isVerkle() bool {
+	return s.db.TrieDB().IsVerkle() || (s.trie != nil && s.trie.IsVerkle())
+}
+
 // TxIndex returns the current transaction index set by SetTxContext.
 func (s *StateDB) TxIndex() int {
 	return s.txIndex
@@ -823,7 +834,7 @@ func (s *StateDB) IntermediateRoot(deleteEmptyObjects bool) common.Hash {
 		start   = time.Now()
 		workers errgroup.Group
 	)
-	if s.db.TrieDB().IsVerkle() {
+	if s.isVerkle() {
 		// Whilst MPT storage tries are independent, Verkle has one single trie
 		// for all the accounts and all the storage slots merged together. The
 		// former can thus be simply parallelized, but updating the latter will
@@ -837,7 +848,7 @@ func (s *StateDB) IntermediateRoot(deleteEmptyObjects bool) common.Hash {
 		}
 		obj := s.stateObjects[addr] // closure for the task runner below
 		workers.Go(func() error {
-			if s.db.TrieDB().IsVerkle() {
+			if s.isVerkle() {
 				obj.updateTrie()
 			} else {
 				obj.updateRoot()
@@ -854,7 +865,7 @@ func (s *StateDB) IntermediateRoot(deleteEmptyObjects bool) common.Hash {
 	// If witness building is enabled, gather all the read-only accesses.
 	// Skip witness collection in Verkle mode, they will be gathered
 	// together at the end.
-	if s.witness != nil && !s.db.TrieDB().IsVerkle() {
+	if s.witness != nil && !s.isVerkle() {
 		// Pull in anything that has been accessed before destruction
 		for _, obj := range s.stateObjectsDestruct {
 			// Skip any objects that haven't touched their storage
@@ -911,7 +922,7 @@ func (s *StateDB) IntermediateRoot(deleteEmptyObjects bool) common.Hash {
 	// only a single trie is used for state hashing. Replacing a non-nil verkle tree
 	// here could result in losing uncommitted changes from storage.
 	start = time.Now()
-	if s.prefetcher != nil {
+	if s.prefetcher != nil && !s.isVerkle() {
 		if trie := s.prefetcher.trie(common.Hash{}, s.originalRoot); trie == nil {
 			log.Error("Failed to retrieve account pre-fetcher trie")
 		} else {
@@ -1130,7 +1141,7 @@ func (s *StateDB) handleDestruction(noStorageWiping bool) (map[common.Hash]*acco
 		deletes[addrHash] = op
 
 		// Short circuit if the origin storage was empty.
-		if prev.Root == types.EmptyRootHash || s.db.TrieDB().IsVerkle() {
+		if prev.Root == types.EmptyRootHash || s.isVerkle() {
 			continue
 		}
 		if noStorageWiping {