db/state, db/integrity: FilesOnlyStateReader for commitment rebuild and CommitmentRoot (#21026)

## LimitedHistoryStateReader -> FilesOnlyStateReader

`CommitmentRoot` has a subcheck `checkCommitmentRootViaRecompute`: it
touches/replays content of the final block in commitment.kv file and
does `ComputeCommitment`. The resulting root hash should remain
unchanged.

`RebuildCommitment`: it creates shard of size 16 and ultimately merged. 

Both need contrainted/limited query of state data (i.e. only return
value for key K from before step X)


both suffer from 2 problems today:
- `getLatestFromFiles(maxTxNum)`: it effectively searches only a single
kv file (the one that contains maxTxNum). The files "left and right" of
it are ignored.
- `LimitedHistoryStateReader`: if `getLatestFromFiles(maxTxNum)` returns
nil, it fallbacks to GetLatest.

This creates problem like:
- commitment prefix is queried, it's not there in current step range.
It's there in some previous file, so it'd return nil (because of
`getLatestFromFiles` logic)
- say we're building 128-192 range, we look for a storage slot key which
became nil in this range; `LimitedHistoryStateReader` would fallback on
GetLatest in this case.

FilesOnlyStateReader: it's essentially LimitedHistoryStateReader,
without the fallback on `GetLatest`.

## getLatestFromFiles walkback

the problem is mentioned above. The fix simply walks back from "maxTxNum
containing snapshot" to the first snapshot.
The change only impacts when `maxTxNum != uint64.max` which is used in
`FilesOnlyStateReader`, `checkCommitmentRootViaFileData` and `commitment
print` command.

## checkCommitmentRootViaRecompute doing TouchKey only on accounts
domain.
- `checkCommitmentRootViaRecompute` — switch from account-domain-only
`touchHistoricalKeys` to `sd.TouchChangedKeysFromHistory` (accounts +
storage; code is covered transitively via `account.codeHash`). This is
the same helper used by `rpc/rpchelper/commitment.go` and receipts
generation.

Author: sudeepdino008

db/integrity/commitment_integrity.go

@@ -207,8 +207,8 @@ func checkCommitmentRootViaSd(ctx context.Context, tx kv.TemporalTx, f state.Vis
 		return nil, err
 	}
 	sd.GetCommitmentCtx().SetTrace(logger.Enabled(ctx, log.LvlTrace))
-	sd.GetCommitmentCtx().SetLimitedHistoryStateReader(tx, maxTxNum) // to use tx.Debug().GetLatestFromFiles with maxTxNum
-	latestTxNum, _, err := sd.SeekCommitment(ctx, tx)                // seek commitment again to use the new state reader instead
+	sd.GetCommitmentCtx().SetStateReader(commitmentdb.NewFilesOnlyStateReader(tx, maxTxNum))
+	latestTxNum, _, err := sd.SeekCommitment(ctx, tx) // seek commitment again to use the new state reader instead
 	if err != nil {
 		return nil, err
 	}
@@ -240,17 +240,13 @@ func checkCommitmentRootViaSd(ctx context.Context, tx kv.TemporalTx, f state.Vis
 }
 
 func checkCommitmentRootViaRecompute(ctx context.Context, tx kv.TemporalTx, sd *execctx.SharedDomains, info commitmentRootInfo, f state.VisibleFile, logger log.Logger) error {
-	trace := logger.Enabled(ctx, log.LvlTrace)
-	touchLoggingVisitor := func(k []byte) {
-		if trace {
-			logger.Trace("[integrity] CommitmentRoot", "key", common.Address(k), "blockNum", info.blockNum, "file", filepath.Base(f.Fullpath()))
-		}
-	}
-	touches, err := touchHistoricalKeys(sd, tx, kv.AccountsDomain, info.blockMinTxNum, info.txNum+1, 0, nil /* no pre-built index */, touchLoggingVisitor)
+	accountTouches, storageTouches, err := sd.TouchChangedKeysFromHistory(tx, info.blockMinTxNum, info.txNum+1)
 	if err != nil {
 		return err
 	}
-	logger.Info("[integrity] CommitmentRoot recomputing", "touches", touches, "file", filepath.Base(f.Fullpath()))
+	logger.Info("[integrity] CommitmentRoot recomputing",
+		"accountTouches", accountTouches, "storageTouches", storageTouches,
+		"file", filepath.Base(f.Fullpath()))
 	recomputedBytes, err := sd.ComputeCommitment(ctx, tx, false /* saveStateAfter */, info.blockNum, info.txNum, "", nil /* commitProgress */)
 	if err != nil {
 		return err
@@ -259,7 +255,10 @@ func checkCommitmentRootViaRecompute(ctx context.Context, tx kv.TemporalTx, sd *
 	if recomputed != info.rootHash {
 		return fmt.Errorf("%w: recomputed root does not match verified root: %s != %s", ErrIntegrity, recomputed, info.rootHash)
 	}
-	logger.Info("[integrity] CommitmentRoot recomputed matches", "root", recomputed, "touches", touches, "file", filepath.Base(f.Fullpath()))
+	logger.Info("[integrity] CommitmentRoot recomputed matches",
+		"root", recomputed,
+		"accountTouches", accountTouches, "storageTouches", storageTouches,
+		"file", filepath.Base(f.Fullpath()))
 	return nil
 }
 

db/state/domain.go

@@ -1333,9 +1333,10 @@ func (dt *DomainRoTx) getLatestFromFiles(k []byte, maxTxNum uint64) (v []byte, f
 		}
 	}
 
+	// Walk newest→oldest; skip files starting strictly after maxTxNum so a key's
+	// last write in an older .kv is still found (walkback bounded by maxTxNum).
 	for i := len(dt.files) - 1; i >= 0; i-- {
-		if maxTxNum != math.MaxUint64 && (dt.files[i].startTxNum > maxTxNum || maxTxNum > dt.files[i].endTxNum) { // (maxTxNum > dt.files[i].endTxNum || dt.files[i].startTxNum > maxTxNum) { // skip partially matched files
-			//fmt.Printf("getLatestFromFiles: skipping file %d %s, maxTxNum=%d, startTxNum=%d, endTxNum=%d\n", i, dt.files[i].src.decompressor.FileName(), maxTxNum, dt.files[i].startTxNum, dt.files[i].endTxNum)
+		if maxTxNum != math.MaxUint64 && dt.files[i].startTxNum > maxTxNum {
 			continue
 		}
 		// fmt.Printf("getLatestFromFiles: lim=%d %d %d %d %d\n", maxTxNum, dt.files[i].startTxNum, dt.files[i].endTxNum, dt.files[i].startTxNum/dt.stepSize, dt.files[i].endTxNum/dt.stepSize)

db/state/squeeze.go

@@ -987,7 +987,7 @@ func RebuildCommitmentFiles(ctx context.Context, rwDb kv.TemporalRwDB, txNumsRea
 
 			domains.SetTxNum(lastTxnumInShard - 1)
 			currentTxNum := lastTxnumInShard - 1
-			domains.GetCommitmentCtx().SetLimitedHistoryStateReader(rwTx, lastTxnumInShard) // this helps to read state from correct file during commitment
+			domains.GetCommitmentCtx().SetStateReader(commitmentdb.NewFilesOnlyStateReader(rwTx, lastTxnumInShard-1))
 			if concurrent {
 				domains.EnableParaTrieDB(rwDb)
 			}

execution/commitment/commitmentdb/commitment_context.go

@@ -141,11 +141,6 @@ func (sdc *SharedDomainsCommitmentContext) SetCustomHistoryStateReader(stateRead
 	sdc.SetStateReader(stateReader)
 }
 
-// SetLimitedHistoryStateReader sets the state reader to read *limited* (i.e. *without-recent-files*) historical state at specified txNum.
-func (sdc *SharedDomainsCommitmentContext) SetLimitedHistoryStateReader(roTx kv.TemporalTx, limitReadAsOfTxNum uint64) {
-	sdc.SetStateReader(NewLimitedHistoryStateReader(roTx, sdc.sharedDomains, limitReadAsOfTxNum))
-}
-
 func (sdc *SharedDomainsCommitmentContext) SetTrace(trace bool) {
 	sdc.trace = trace
 	sdc.patriciaTrie.SetTrace(trace)

execution/commitment/commitmentdb/reader.go

@@ -80,57 +80,39 @@ func (r *HistoryStateReader) Clone(tx kv.TemporalTx) StateReader {
 	return NewHistoryStateReader(tx, r.limitReadAsOfTxNum)
 }
 
-// LimitedHistoryStateReader reads from *limited* (i.e. *without-recent-files*) state at specified txNum, otherwise from *latest*.
-// `limitReadAsOfTxNum` here is used for unusual operation: "hide recent .kv files and read the latest state from files".
-type LimitedHistoryStateReader struct {
-	HistoryStateReader
-	sharedDomains sd
-	getter        kv.TemporalGetter
+// FilesOnlyStateReader reads from .kv files only, capped at limitTxNum.
+// On miss (key not present in any frozen .kv file ≤ limitTxNum), returns nil
+// without any fallback. This is the right semantic for integrity checks and
+// commitment rebuild that validate "what does the .kv snapshot at this
+// boundary actually contain?": no consultation of history index, no
+// consultation of current DB state, no consultation of .kv files past the
+// boundary.
+type FilesOnlyStateReader struct {
+	roTx       kv.TemporalTx
+	limitTxNum uint64
 }
 
-func NewLimitedHistoryStateReader(roTx kv.TemporalTx, sd sd, limitReadAsOfTxNum uint64) *LimitedHistoryStateReader {
-	return &LimitedHistoryStateReader{
-		HistoryStateReader: HistoryStateReader{
-			roTx:               roTx,
-			limitReadAsOfTxNum: limitReadAsOfTxNum,
-		},
-		sharedDomains: sd,
-		getter:        sd.AsGetter(roTx),
-	}
+func NewFilesOnlyStateReader(roTx kv.TemporalTx, limitTxNum uint64) *FilesOnlyStateReader {
+	return &FilesOnlyStateReader{roTx: roTx, limitTxNum: limitTxNum}
 }
 
-func (r *LimitedHistoryStateReader) WithHistory() bool {
-	return false
-}
+func (r *FilesOnlyStateReader) WithHistory() bool { return false }
+
+func (r *FilesOnlyStateReader) CheckDataAvailable(kv.Domain, kv.Step) error { return nil }
 
-// Reason why we have `kv.TemporalDebugTx.GetLatestFromFiles' call here: `state.RebuildCommitmentFiles` can build commitment.kv from account.kv.
-// Example: we have account.0-16.kv and account.16-18.kv, let's generate commitment.0-16.kv => it means we need to make account.16-18.kv invisible
-// and then read "latest state" like there is no account.16-18.kv
-func (r *LimitedHistoryStateReader) Read(d kv.Domain, plainKey []byte, stepSize uint64) (enc []byte, step kv.Step, err error) {
-	var ok bool
-	var endTxNum uint64
-	// reading from domain files this way will dereference domain key correctly,
-	// GetAsOf itself does not dereference keys in commitment domain values
-	enc, ok, _, endTxNum, err = r.roTx.Debug().GetLatestFromFiles(d, plainKey, r.limitReadAsOfTxNum)
+func (r *FilesOnlyStateReader) Read(d kv.Domain, plainKey []byte, stepSize uint64) (enc []byte, step kv.Step, err error) {
+	enc, ok, _, endTxNum, err := r.roTx.Debug().GetLatestFromFiles(d, plainKey, r.limitTxNum)
 	if err != nil {
-		return nil, 0, fmt.Errorf("LimitedHistoryStateReader(GetLatestFromFiles) %q: (limitTxNum=%d): %w", d, r.limitReadAsOfTxNum, err)
+		return nil, 0, fmt.Errorf("FilesOnlyStateReader %q (limitTxNum=%d): %w", d, r.limitTxNum, err)
 	}
 	if !ok {
-		enc = nil
-	} else {
-		step = kv.Step(endTxNum / stepSize)
+		return nil, 0, nil
 	}
-	if enc == nil {
-		enc, step, err = r.getter.GetLatest(d, plainKey)
-		if err != nil {
-			return nil, 0, fmt.Errorf("LimitedHistoryStateReader(GetLatest) %q: %w", d, err)
-		}
-	}
-	return enc, step, nil
+	return enc, kv.Step(endTxNum / stepSize), nil
 }
 
-func (r *LimitedHistoryStateReader) Clone(tx kv.TemporalTx) StateReader {
-	return NewLimitedHistoryStateReader(tx, r.sharedDomains, r.limitReadAsOfTxNum)
+func (r *FilesOnlyStateReader) Clone(tx kv.TemporalTx) StateReader {
+	return NewFilesOnlyStateReader(tx, r.limitTxNum)
 }
 
 // SplitStateReader implements commitmentdb.StateReader using (potentially) different state readers for commitment