performance: cherry-pick inc-shard-def-size updates (#21033)

awskii · sudeepdino008 · web-flow · commit 2f0099ca8cc0 · 2026-05-07T12:04:37.000Z
Cherry-pick of #20930, #21025, and #21026 onto performance. --------- Co-authored-by: sudeepdino008 <sudeepdino008@gmail.com>
diff --git a/cmd/integration/commands/flags.go b/cmd/integration/commands/flags.go
@@ -48,11 +48,12 @@ var (
 
 	dbWriteMap bool
 
-	chainTipMode    bool
-	clearCommitment bool
-	resume          bool
-	noHistory       bool
-	syncCfg         = ethconfig.Defaults.Sync
+	chainTipMode                    bool
+	clearCommitment                 bool
+	resume                          bool
+	noHistory                       bool
+	erigondbDomainStepsInFrozenFile string
+	syncCfg                         = ethconfig.Defaults.Sync
 )
 
 func must(err error) {
@@ -117,7 +118,7 @@ func withYes(cmd *cobra.Command) {
 }
 
 func withSqueeze(cmd *cobra.Command) {
-	cmd.Flags().BoolVar(&squeeze, "squeeze", true, "use offset-pointers from commitment.kv to account.kv")
+	cmd.Flags().BoolVar(&squeeze, "squeeze", false, "use offset-pointers from commitment.kv to account.kv")
 }
 
 func withClearCommitment(cmd *cobra.Command) {
@@ -203,6 +204,12 @@ func withChaosMonkey(cmd *cobra.Command) {
 	cmd.Flags().BoolVar(&syncCfg.ChaosMonkey, utils.ChaosMonkeyFlag.Name, utils.ChaosMonkeyFlag.Value, utils.ChaosMonkeyFlag.Usage)
 }
 
+func withErigondbDomainStepsInFrozenFile(cmd *cobra.Command) {
+	cmd.Flags().StringVar(&erigondbDomainStepsInFrozenFile,
+		utils.ErigondbDomainStepsInFrozenFileFlag.Name, "",
+		utils.ErigondbDomainStepsInFrozenFileFlag.Usage)
+}
+
 // withStageBase applies flags common to most stage commands: config, datadir, chain, chaos monkey, heimdall, unwind.
 func withStageBase(cmd *cobra.Command) {
 	withConfig(cmd)
diff --git a/cmd/integration/commands/stages.go b/cmd/integration/commands/stages.go
@@ -24,6 +24,7 @@ import (
 	"os"
 	"runtime"
 	"slices"
+	"strconv"
 	"strings"
 	"sync"
 	"time"
@@ -36,11 +37,13 @@ import (
 	"golang.org/x/sync/semaphore"
 
 	"github.com/erigontech/erigon/cl/clparams"
+	"github.com/erigontech/erigon/cmd/utils"
 	"github.com/erigontech/erigon/common"
 	"github.com/erigontech/erigon/common/dbg"
 	"github.com/erigontech/erigon/common/dir"
 	"github.com/erigontech/erigon/common/estimate"
 	"github.com/erigontech/erigon/common/log/v3"
+	"github.com/erigontech/erigon/db/config3"
 	"github.com/erigontech/erigon/db/datadir"
 	"github.com/erigontech/erigon/db/fromdb"
 	"github.com/erigontech/erigon/db/integrity"
@@ -322,6 +325,7 @@ func init() {
 	withPruneTo(cmdStageExec)
 	withTraceFlags(cmdStageExec)
 	withChainTipMode(cmdStageExec)
+	withErigondbDomainStepsInFrozenFile(cmdStageExec)
 	rootCmd.AddCommand(cmdStageExec)
 
 	withStageBase(cmdStageExecReplay)
@@ -1063,7 +1067,28 @@ func allSnapshots(ctx context.Context, db kv.RoDB, logger log.Logger) (*freezebl
 		if erigonDBSettings, err = dbstate.ResolveErigonDBSettings(dirs, logger, false); err != nil {
 			return
 		}
-		_aggSingleton = dbstate.New(dirs).Logger(logger).WithErigonDBSettings(erigonDBSettings).MustOpen(ctx, db)
+		aggOpts := dbstate.New(dirs).Logger(logger).WithErigonDBSettings(erigonDBSettings)
+		if erigondbDomainStepsInFrozenFile != "" {
+			var v uint64
+			if strings.EqualFold(erigondbDomainStepsInFrozenFile, "inf") {
+				v = config3.UnboundedDomainMerge
+			} else {
+				parsed, perr := strconv.ParseUint(erigondbDomainStepsInFrozenFile, 10, 64)
+				if perr != nil || parsed == 0 {
+					err = fmt.Errorf("invalid --%s value %q: must be a positive integer or \"Inf\"",
+						utils.ErigondbDomainStepsInFrozenFileFlag.Name, erigondbDomainStepsInFrozenFile)
+					return
+				}
+				v = parsed
+			}
+			stepsStr := "Inf"
+			if v != config3.UnboundedDomainMerge {
+				stepsStr = fmt.Sprintf("%d", v)
+			}
+			logger.Info("domain merge cap overridden", "steps_in_frozen_file", stepsStr)
+			aggOpts = aggOpts.ErigondbDomainStepsInFrozenFile(v)
+		}
+		_aggSingleton = aggOpts.MustOpen(ctx, db)
 
 		_aggSingleton.SetProduceMod(snapCfg.ProduceE3)
 
diff --git a/cmd/utils/app/snapshots_cmd.go b/cmd/utils/app/snapshots_cmd.go
@@ -190,6 +190,7 @@ var snapshotCommand = cli.Command{
 			Usage: "create snapshots from the specified block number",
 			Flags: joinFlags([]cli.Flag{
 				&utils.DataDirFlag,
+				&utils.ErigondbDomainStepsInFrozenFileFlag,
 			}),
 		},
 		{
@@ -3199,6 +3200,27 @@ func doRetireCommand(cliCtx *cli.Context, dirs datadir.Dirs) error {
 	defer br.MadvNormal().DisableReadAhead()
 	defer agg.MadvNormal().DisableReadAhead()
 
+	if cliCtx.IsSet(utils.ErigondbDomainStepsInFrozenFileFlag.Name) {
+		s := cliCtx.String(utils.ErigondbDomainStepsInFrozenFileFlag.Name)
+		var v uint64
+		if strings.EqualFold(s, "inf") {
+			v = config3.UnboundedDomainMerge
+		} else {
+			parsed, perr := strconv.ParseUint(s, 10, 64)
+			if perr != nil || parsed == 0 {
+				return fmt.Errorf("invalid --%s value %q: must be a positive integer or \"Inf\"",
+					utils.ErigondbDomainStepsInFrozenFileFlag.Name, s)
+			}
+			v = parsed
+		}
+		stepsStr := "Inf"
+		if v != config3.UnboundedDomainMerge {
+			stepsStr = fmt.Sprintf("%d", v)
+		}
+		logger.Info("domain merge cap overridden", "steps_in_frozen_file", stepsStr)
+		agg.SetErigondbDomainStepsInFrozenFile(v)
+	}
+
 	blockSnapBuildSema := semaphore.NewWeighted(int64(runtime.NumCPU()))
 	agg.SetSnapshotBuildSema(blockSnapBuildSema)
 
diff --git a/db/integrity/commitment_integrity.go b/db/integrity/commitment_integrity.go
@@ -207,8 +207,8 @@ func checkCommitmentRootViaSd(ctx context.Context, tx kv.TemporalTx, f state.Vis
 		return nil, err
 	}
 	sd.GetCommitmentCtx().SetTrace(logger.Enabled(ctx, log.LvlTrace))
-	sd.GetCommitmentCtx().SetLimitedHistoryStateReader(tx, maxTxNum) // to use tx.Debug().GetLatestFromFiles with maxTxNum
-	latestTxNum, _, err := sd.SeekCommitment(ctx, tx)                // seek commitment again to use the new state reader instead
+	sd.GetCommitmentCtx().SetStateReader(commitmentdb.NewFilesOnlyStateReader(tx, maxTxNum))
+	latestTxNum, _, err := sd.SeekCommitment(ctx, tx) // seek commitment again to use the new state reader instead
 	if err != nil {
 		return nil, err
 	}
@@ -240,17 +240,13 @@ func checkCommitmentRootViaSd(ctx context.Context, tx kv.TemporalTx, f state.Vis
 }
 
 func checkCommitmentRootViaRecompute(ctx context.Context, tx kv.TemporalTx, sd *execctx.SharedDomains, info commitmentRootInfo, f state.VisibleFile, logger log.Logger) error {
-	trace := logger.Enabled(ctx, log.LvlTrace)
-	touchLoggingVisitor := func(k []byte) {
-		if trace {
-			logger.Trace("[integrity] CommitmentRoot", "key", common.Address(k), "blockNum", info.blockNum, "file", filepath.Base(f.Fullpath()))
-		}
-	}
-	touches, err := touchHistoricalKeys(sd, tx, kv.AccountsDomain, info.blockMinTxNum, info.txNum+1, 0, nil /* no pre-built index */, touchLoggingVisitor)
+	accountTouches, storageTouches, err := sd.TouchChangedKeysFromHistory(tx, info.blockMinTxNum, info.txNum+1)
 	if err != nil {
 		return err
 	}
-	logger.Info("[integrity] CommitmentRoot recomputing", "touches", touches, "file", filepath.Base(f.Fullpath()))
+	logger.Info("[integrity] CommitmentRoot recomputing",
+		"accountTouches", accountTouches, "storageTouches", storageTouches,
+		"file", filepath.Base(f.Fullpath()))
 	recomputedBytes, err := sd.ComputeCommitment(ctx, tx, false /* saveStateAfter */, info.blockNum, info.txNum, "", nil /* commitProgress */)
 	if err != nil {
 		return err
@@ -259,7 +255,10 @@ func checkCommitmentRootViaRecompute(ctx context.Context, tx kv.TemporalTx, sd *
 	if recomputed != info.rootHash {
 		return fmt.Errorf("%w: recomputed root does not match verified root: %s != %s", ErrIntegrity, recomputed, info.rootHash)
 	}
-	logger.Info("[integrity] CommitmentRoot recomputed matches", "root", recomputed, "touches", touches, "file", filepath.Base(f.Fullpath()))
+	logger.Info("[integrity] CommitmentRoot recomputed matches",
+		"root", recomputed,
+		"accountTouches", accountTouches, "storageTouches", storageTouches,
+		"file", filepath.Base(f.Fullpath()))
 	return nil
 }
 
diff --git a/db/state/aggregator.go b/db/state/aggregator.go
@@ -247,6 +247,14 @@ func (a *Aggregator) Dirs() datadir.Dirs        { return a.dirs }
 func (a *Aggregator) StepsInFrozenFile() uint64 { return a.stepsInFrozenFile.Load() }
 func (a *Aggregator) Logger() log.Logger        { return a.logger }
 
+// SetErigondbDomainStepsInFrozenFile applies a domain-only merge cap override at runtime.
+// Intended for one-shot tools (e.g. `erigon seg retire`) that construct the aggregator
+// via openAgg and need to override the cap after construction. Must be called before any
+// merge work begins.
+func (a *Aggregator) SetErigondbDomainStepsInFrozenFile(steps uint64) {
+	a.erigondbDomainStepsInFrozenFile = steps
+}
+
 func (a *Aggregator) ForTestReplaceKeysInValues(domain kv.Domain, v bool) {
 	a.d[domain].ReplaceKeysInValues = v
 }
diff --git a/db/state/domain.go b/db/state/domain.go
@@ -1354,9 +1354,10 @@ func (dt *DomainRoTx) getLatestFromFiles(k []byte, maxTxNum uint64) (v []byte, f
 		}
 	}
 
+	// Walk newest→oldest; skip files starting strictly after maxTxNum so a key's
+	// last write in an older .kv is still found (walkback bounded by maxTxNum).
 	for i := len(dt.files) - 1; i >= 0; i-- {
-		if maxTxNum != math.MaxUint64 && (dt.files[i].startTxNum > maxTxNum || maxTxNum > dt.files[i].endTxNum) { // (maxTxNum > dt.files[i].endTxNum || dt.files[i].startTxNum > maxTxNum) { // skip partially matched files
-			//fmt.Printf("getLatestFromFiles: skipping file %d %s, maxTxNum=%d, startTxNum=%d, endTxNum=%d\n", i, dt.files[i].src.decompressor.FileName(), maxTxNum, dt.files[i].startTxNum, dt.files[i].endTxNum)
+		if maxTxNum != math.MaxUint64 && dt.files[i].startTxNum > maxTxNum {
 			continue
 		}
 		// fmt.Printf("getLatestFromFiles: lim=%d %d %d %d %d\n", maxTxNum, dt.files[i].startTxNum, dt.files[i].endTxNum, dt.files[i].startTxNum/dt.stepSize, dt.files[i].endTxNum/dt.stepSize)
diff --git a/db/state/domain_committed.go b/db/state/domain_committed.go
@@ -35,12 +35,17 @@ import (
 	"github.com/erigontech/erigon/execution/commitment/commitmentdb"
 )
 
-const minStepsForReferencing = 2
+const minStepsForReferencing = 99999
 
 // ValuesPlainKeyReferencingThresholdReached checks if the range from..to is large enough to use plain key referencing
 // Used for commitment branches - to store references to account and storage keys as shortened keys (file offsets)
 func ValuesPlainKeyReferencingThresholdReached(stepSize, from, to uint64) bool {
-	return (to-from)/stepSize >= minStepsForReferencing
+	return ((to-from)/stepSize)%minStepsForReferencing == 0
+}
+
+// Values can use key referencing in this range?
+func MayContainValuesPlainKeyReferencing(stepSize, from, to uint64) bool {
+	return ((to - from) / stepSize) > minStepsForReferencing
 }
 
 // replaceShortenedKeysInBranch expands shortened key references (file offsets) in branch data back to full keys
diff --git a/db/state/squeeze.go b/db/state/squeeze.go
@@ -895,8 +895,7 @@ func RebuildCommitmentFiles(ctx context.Context, rwDb kv.TemporalRwDB, txNumsRea
 		stepsInShard := uint64(shardTo - shardFrom)
 		keysPerStep := totalKeys / stepsInShard // how many keys in just one step?
 
-		// shardStepsSize := kv.Step(2)
-		shardStepsSize := kv.Step(min(uint64(math.Pow(2, math.Log2(float64(stepsInShard)))), 16))
+		shardStepsSize := kv.Step(min(uint64(math.Pow(2, math.Log2(float64(stepsInShard)))), 64))
 		if uint64(shardStepsSize) != stepsInShard { // processing shard in several smaller steps
 			shardTo = shardFrom + shardStepsSize // if shard is quite big, we will process it in several steps
 		}
@@ -975,7 +974,7 @@ func RebuildCommitmentFiles(ctx context.Context, rwDb kv.TemporalRwDB, txNumsRea
 
 			domains.SetTxNum(lastTxnumInShard - 1)
 			currentTxNum := lastTxnumInShard - 1
-			domains.GetCommitmentCtx().SetLimitedHistoryStateReader(rwTx, lastTxnumInShard) // this helps to read state from correct file during commitment
+			domains.GetCommitmentCtx().SetStateReader(commitmentdb.NewFilesOnlyStateReader(rwTx, lastTxnumInShard-1))
 			if concurrent {
 				domains.EnableParaTrieDB(rwDb)
 			}
@@ -1044,14 +1043,14 @@ func RebuildCommitmentFiles(ctx context.Context, rwDb kv.TemporalRwDB, txNumsRea
 
 	acRo.Close()
 
-	if !squeeze && !statecfg.Schema.CommitmentDomain.ReplaceKeysInValues {
+	if !squeeze {
 		return latestRoot, nil
 	}
 	logger.Info("[squeeze] starting")
 	a.recalcVisibleFiles()
 
 	logger.Info(fmt.Sprintf("[squeeze] latest root %x", latestRoot))
-	a.ForTestReplaceKeysInValues(kv.CommitmentDomain, true)
+	a.ForTestReplaceKeysInValues(kv.CommitmentDomain, squeeze)
 
 	actx := a.BeginFilesRo()
 	defer actx.Close()
diff --git a/db/state/statecfg/state_schema.go b/db/state/statecfg/state_schema.go
@@ -73,7 +73,7 @@ func Configure(Schema SchemaGen, a AggSetters, dirs datadir.Dirs, salt *uint32,
 	return nil
 }
 
-const AggregatorSqueezeCommitmentValues = true
+const AggregatorSqueezeCommitmentValues = false
 const MaxNonFuriousDirtySpacePerTx = 64 * datasize.MB
 
 var dbgCommBtIndex = dbg.EnvBool("AGG_COMMITMENT_BT", false)
diff --git a/execution/commitment/commitmentdb/commitment_context.go b/execution/commitment/commitmentdb/commitment_context.go
@@ -141,11 +141,6 @@ func (sdc *SharedDomainsCommitmentContext) SetCustomHistoryStateReader(stateRead
 	sdc.SetStateReader(stateReader)
 }
 
-// SetLimitedHistoryStateReader sets the state reader to read *limited* (i.e. *without-recent-files*) historical state at specified txNum.
-func (sdc *SharedDomainsCommitmentContext) SetLimitedHistoryStateReader(roTx kv.TemporalTx, limitReadAsOfTxNum uint64) {
-	sdc.SetStateReader(NewLimitedHistoryStateReader(roTx, sdc.sharedDomains, limitReadAsOfTxNum))
-}
-
 func (sdc *SharedDomainsCommitmentContext) SetTrace(trace bool) {
 	sdc.trace = trace
 	sdc.patriciaTrie.SetTrace(trace)
diff --git a/execution/commitment/commitmentdb/reader.go b/execution/commitment/commitmentdb/reader.go
@@ -80,57 +80,39 @@ func (r *HistoryStateReader) Clone(tx kv.TemporalTx) StateReader {
 	return NewHistoryStateReader(tx, r.limitReadAsOfTxNum)
 }
 
-// LimitedHistoryStateReader reads from *limited* (i.e. *without-recent-files*) state at specified txNum, otherwise from *latest*.
-// `limitReadAsOfTxNum` here is used for unusual operation: "hide recent .kv files and read the latest state from files".
-type LimitedHistoryStateReader struct {
-	HistoryStateReader
-	sharedDomains sd
-	getter        kv.TemporalGetter
+// FilesOnlyStateReader reads from .kv files only, capped at limitTxNum.
+// On miss (key not present in any frozen .kv file ≤ limitTxNum), returns nil
+// without any fallback. This is the right semantic for integrity checks and
+// commitment rebuild that validate "what does the .kv snapshot at this
+// boundary actually contain?": no consultation of history index, no
+// consultation of current DB state, no consultation of .kv files past the
+// boundary.
+type FilesOnlyStateReader struct {
+	roTx       kv.TemporalTx
+	limitTxNum uint64
 }
 
-func NewLimitedHistoryStateReader(roTx kv.TemporalTx, sd sd, limitReadAsOfTxNum uint64) *LimitedHistoryStateReader {
-	return &LimitedHistoryStateReader{
-		HistoryStateReader: HistoryStateReader{
-			roTx:               roTx,
-			limitReadAsOfTxNum: limitReadAsOfTxNum,
-		},
-		sharedDomains: sd,
-		getter:        sd.AsGetter(roTx),
-	}
+func NewFilesOnlyStateReader(roTx kv.TemporalTx, limitTxNum uint64) *FilesOnlyStateReader {
+	return &FilesOnlyStateReader{roTx: roTx, limitTxNum: limitTxNum}
 }
 
-func (r *LimitedHistoryStateReader) WithHistory() bool {
-	return false
-}
+func (r *FilesOnlyStateReader) WithHistory() bool { return false }
+
+func (r *FilesOnlyStateReader) CheckDataAvailable(kv.Domain, kv.Step) error { return nil }
 
-// Reason why we have `kv.TemporalDebugTx.GetLatestFromFiles' call here: `state.RebuildCommitmentFiles` can build commitment.kv from account.kv.
-// Example: we have account.0-16.kv and account.16-18.kv, let's generate commitment.0-16.kv => it means we need to make account.16-18.kv invisible
-// and then read "latest state" like there is no account.16-18.kv
-func (r *LimitedHistoryStateReader) Read(d kv.Domain, plainKey []byte, stepSize uint64) (enc []byte, step kv.Step, err error) {
-	var ok bool
-	var endTxNum uint64
-	// reading from domain files this way will dereference domain key correctly,
-	// GetAsOf itself does not dereference keys in commitment domain values
-	enc, ok, _, endTxNum, err = r.roTx.Debug().GetLatestFromFiles(d, plainKey, r.limitReadAsOfTxNum)
+func (r *FilesOnlyStateReader) Read(d kv.Domain, plainKey []byte, stepSize uint64) (enc []byte, step kv.Step, err error) {
+	enc, ok, _, endTxNum, err := r.roTx.Debug().GetLatestFromFiles(d, plainKey, r.limitTxNum)
 	if err != nil {
-		return nil, 0, fmt.Errorf("LimitedHistoryStateReader(GetLatestFromFiles) %q: (limitTxNum=%d): %w", d, r.limitReadAsOfTxNum, err)
+		return nil, 0, fmt.Errorf("FilesOnlyStateReader %q (limitTxNum=%d): %w", d, r.limitTxNum, err)
 	}
 	if !ok {
-		enc = nil
-	} else {
-		step = kv.Step(endTxNum / stepSize)
+		return nil, 0, nil
 	}
-	if enc == nil {
-		enc, step, err = r.getter.GetLatest(d, plainKey)
-		if err != nil {
-			return nil, 0, fmt.Errorf("LimitedHistoryStateReader(GetLatest) %q: %w", d, err)
-		}
-	}
-	return enc, step, nil
+	return enc, kv.Step(endTxNum / stepSize), nil
 }
 
-func (r *LimitedHistoryStateReader) Clone(tx kv.TemporalTx) StateReader {
-	return NewLimitedHistoryStateReader(tx, r.sharedDomains, r.limitReadAsOfTxNum)
+func (r *FilesOnlyStateReader) Clone(tx kv.TemporalTx) StateReader {
+	return NewFilesOnlyStateReader(tx, r.limitTxNum)
 }
 
 // SplitStateReader implements commitmentdb.StateReader using (potentially) different state readers for commitment

Original file line number	Diff line number	Diff line change
`@@ -1354,9 +1354,10 @@ func (dt *DomainRoTx) getLatestFromFiles(k []byte, maxTxNum uint64) (v []byte, f`
`1354`	`1354`	`}`
`1355`	`1355`	`}`
`1356`	`1356`
	`1357`	`+ // Walk newest→oldest; skip files starting strictly after maxTxNum so a key's`
	`1358`	`+ // last write in an older .kv is still found (walkback bounded by maxTxNum).`
`1357`	`1359`	`for i := len(dt.files) - 1; i >= 0; i-- {`
`1358`		`- if maxTxNum != math.MaxUint64 && (dt.files[i].startTxNum > maxTxNum \|\| maxTxNum > dt.files[i].endTxNum) { // (maxTxNum > dt.files[i].endTxNum \|\| dt.files[i].startTxNum > maxTxNum) { // skip partially matched files`
`1359`		`- //fmt.Printf("getLatestFromFiles: skipping file %d %s, maxTxNum=%d, startTxNum=%d, endTxNum=%d\n", i, dt.files[i].src.decompressor.FileName(), maxTxNum, dt.files[i].startTxNum, dt.files[i].endTxNum)`
	`1360`	`+ if maxTxNum != math.MaxUint64 && dt.files[i].startTxNum > maxTxNum {`
`1360`	`1361`	`continue`
`1361`	`1362`	`}`
`1362`	`1363`	`// fmt.Printf("getLatestFromFiles: lim=%d %d %d %d %d\n", maxTxNum, dt.files[i].startTxNum, dt.files[i].endTxNum, dt.files[i].startTxNum/dt.stepSize, dt.files[i].endTxNum/dt.stepSize)`
Original file line number	Diff line number	Diff line change
`@@ -73,7 +73,7 @@ func Configure(Schema SchemaGen, a AggSetters, dirs datadir.Dirs, salt *uint32,`
`73`	`73`	`return nil`
`74`	`74`	`}`
`75`	`75`
`76`		`-const AggregatorSqueezeCommitmentValues = true`
	`76`	`+const AggregatorSqueezeCommitmentValues = false`
`77`	`77`	`const MaxNonFuriousDirtySpacePerTx = 64 * datasize.MB`
`78`	`78`
`79`	`79`	`var dbgCommBtIndex = dbg.EnvBool("AGG_COMMITMENT_BT", false)`