db/state: fix spurious accessor rebuild for post-merge subset files

After a merge, the old sub-range .ef files stay on disk while held open
by active readers. OpenFolder/scanDirtyFiles re-adds them to dirtyFiles.
BuildMissedAccessors then found these re-added items had no .efi accessor
(already deleted as part of merge cleanup) and rebuilt them, producing
orphaned .efi files at paths that dir.trackRemovedFiles had been told were
deleted, triggering repeated "Removed file unexpectedly exists" warnings.

Fix: skip items in fileItemsWithMissedAccessors that are proper subsets
of another item already in dirtyFiles — their accessor should not be
rebuilt because the superset file supersedes them.

Add ERIGON_AGG_TRACE_FILE_LIFE logging to observe when the condition
fires: Debug by default; set the env var to empty to escalate all to
Warn, or to a substring to filter by filename base.

Also add dbg.EnvStringLookup to distinguish "not set" from "set to
empty string".

Fixes #19797

Author: anacrolix

common/dbg/dbg_env.go

@@ -59,6 +59,12 @@ func EnvString(envVarName string, defaultVal string) string {
 	return defaultVal
 }
 
+// EnvStringLookup returns the value of the env var and whether it was set.
+// Unlike EnvString, it distinguishes between "not set" and "set to empty string".
+func EnvStringLookup(envVarName string) (string, bool) {
+	return envLookup(envVarName)
+}
+
 func EnvStrings(envVarName string, sep string, defaultVal []string) []string {
 	if v, _ := envLookup(envVarName); v != "" {
 		return strings.Split(v, sep)

db/state/dirty_files.go

@@ -806,8 +806,42 @@ func readDirNames(dir string) dirListing {
 
 // fileItemsWithMissedAccessors returns list of files with missed accessors
 // here "accessors" are generated dynamically by `accessorsFor`
-func fileItemsWithMissedAccessors(dirtyFiles []*FilesItem, aggregationStep uint64, accessorsFor func(fromStep, toStep kv.Step) []string) (l []*FilesItem) {
-	for _, item := range dirtyFiles {
+func fileItemsWithMissedAccessors(
+	dirtyFiles []*FilesItem,
+	aggregationStep uint64,
+	accessorsFor func(fromStep, toStep kv.Step) []string,
+	filenameBase string,
+	logger log.Logger,
+) (l []*FilesItem) {
+	for i, item := range dirtyFiles {
+		// Skip items that are proper subsets of other dirty files.
+		// After a merge, old sub-range data files stay on disk while held open by
+		// active readers; OpenFolder/scanDirtyFiles re-adds them to dirtyFiles.
+		// Rebuilding their accessors creates orphaned .efi files and triggers false
+		// "Removed file unexpectedly exists" warnings.
+		var coveringItem *FilesItem
+		for j, other := range dirtyFiles {
+			if i != j && item.isProperSubsetOf(other) {
+				coveringItem = other
+				break
+			}
+		}
+		if coveringItem != nil {
+			if traceFileLifeSet && (traceFileLife == "" || strings.Contains(filenameBase, traceFileLife)) {
+				logger.Warn("[agg.dbg] fileItemsWithMissedAccessors: skipping subset item",
+					"f", filenameBase,
+					"item", fmt.Sprintf("%d-%d", item.startTxNum/aggregationStep, item.endTxNum/aggregationStep),
+					"coveredBy", fmt.Sprintf("%d-%d", coveringItem.startTxNum/aggregationStep, coveringItem.endTxNum/aggregationStep),
+				)
+			} else {
+				logger.Debug("[agg.dbg] fileItemsWithMissedAccessors: skipping subset item",
+					"f", filenameBase,
+					"item", fmt.Sprintf("%d-%d", item.startTxNum/aggregationStep, item.endTxNum/aggregationStep),
+					"coveredBy", fmt.Sprintf("%d-%d", coveringItem.startTxNum/aggregationStep, coveringItem.endTxNum/aggregationStep),
+				)
+			}
+			continue
+		}
 		for _, fName := range accessorsFor(item.StepRange(aggregationStep)) {
 			exists, err := dir.FileExist(fName)
 			if err != nil {

db/state/dirty_files_test.go

@@ -10,6 +10,7 @@ import (
 	btree2 "github.com/tidwall/btree"
 
 	"github.com/erigontech/erigon/common/dir"
+	"github.com/erigontech/erigon/common/log/v3"
 	"github.com/erigontech/erigon/db/kv"
 )
 
@@ -93,7 +94,7 @@ func TestFileItemWithMissedAccessor(t *testing.T) {
 		defer dir.RemoveFile(fname)
 	}
 
-	fileItems := fileItemsWithMissedAccessors(btree.Items(), aggStep, accessorFor)
+	fileItems := fileItemsWithMissedAccessors(btree.Items(), aggStep, accessorFor, "", log.New())
 	require.Len(t, fileItems, 1)
 	require.Equal(t, f3, fileItems[0])
 }

db/state/domain.go

@@ -56,8 +56,8 @@ import (
 )
 
 var (
-	asserts          = dbg.EnvBool("AGG_ASSERTS", false)
-	traceFileLife    = dbg.EnvString("AGG_TRACE_FILE_LIFE", "")
+	asserts              = dbg.EnvBool("AGG_ASSERTS", false)
+	traceFileLife, traceFileLifeSet = dbg.EnvStringLookup("AGG_TRACE_FILE_LIFE")
 	traceGetAsOf     = dbg.EnvString("AGG_TRACE_GET_AS_OF", "")
 	tracePutWithPrev = dbg.EnvString("AGG_TRACE_PUT_WITH_PREV", "")
 )
@@ -1129,7 +1129,7 @@ func (d *Domain) missedBtreeAccessors(source []*FilesItem, dl dirListing) (l []*
 			panic(err)
 		}
 		return []string{btF, exF}
-	})
+	}, d.FilenameBase, d.logger)
 }
 
 func (d *Domain) MissedMapAccessors() (l []*FilesItem) {
@@ -1146,7 +1146,7 @@ func (d *Domain) missedMapAccessors(source []*FilesItem, dl dirListing) (l []*Fi
 			panic(err)
 		}
 		return []string{fPath}
-	})
+	}, d.FilenameBase, d.logger)
 	//return fileItemsWithMissedAccessors(source, d.stepSize, func(fromStep, toStep uint64) []string {
 	//	var files []string
 	//	if d.Accessors.Has(AccessorHashMap) {

db/state/history.go

@@ -208,7 +208,7 @@ func (h *History) missedMapAccessors(source []*FilesItem, dl dirListing) (l []*F
 			panic(err)
 		}
 		return []string{fPath}
-	})
+	}, h.FilenameBase, h.logger)
 }
 
 func (h *History) buildVi(ctx context.Context, item *FilesItem, ps *background.ProgressSet) (err error) {

db/state/inverted_index.go

@@ -258,7 +258,7 @@ func (ii *InvertedIndex) missedMapAccessors(source []*FilesItem, dl dirListing)
 			panic(err)
 		}
 		return []string{fPath}
-	})
+	}, ii.FilenameBase, ii.logger)
 }
 
 func (ii *InvertedIndex) buildEfAccessor(ctx context.Context, item *FilesItem, ps *background.ProgressSet) (err error) {

db/state/snap_repo.go

@@ -181,7 +181,7 @@ func (f *SnapshotRepo) DirtyFilesWithNoBtreeAccessors() (l []*FilesItem) {
 		fname, _ := p.BtIdxFile(v, from, to)
 		existenceFile, _ := p.ExistenceFile(v, from, to)
 		return []string{fname, existenceFile}
-	})
+	}, f.schema.DataTag(), f.logger)
 }
 
 func (f *SnapshotRepo) DirtyFilesWithNoHashAccessors() (l []*FilesItem) {
@@ -199,7 +199,7 @@ func (f *SnapshotRepo) DirtyFilesWithNoHashAccessors() (l []*FilesItem) {
 			files[i], _ = p.AccessorIdxFile(v, RootNum(fromStep.ToTxNum(ss)), RootNum(toStep.ToTxNum(ss)), i)
 		}
 		return files
-	})
+	}, f.schema.DataTag(), f.logger)
 }
 
 func (f *SnapshotRepo) EndRootNum() RootNum {
@@ -331,23 +331,23 @@ func (f *SnapshotRepo) FilesWithMissedAccessors() *MissedFilesMap {
 			fileItemsWithMissedAccessors(f.dirtyFiles.Items(), f.stepSize, func(fromStep, toStep kv.Step) []string {
 				file, _ := f.schema.BtIdxFile(version.V1_0, RootNum(fromStep*kv.Step(f.stepSize)), RootNum(toStep*kv.Step(f.stepSize)))
 				return []string{file}
-			})
+			}, f.schema.DataTag(), f.logger)
 	}
 
 	if f.accessors.Has(statecfg.AccessorHashMap) {
 		mf[statecfg.AccessorHashMap] =
 			fileItemsWithMissedAccessors(f.dirtyFiles.Items(), f.stepSize, func(fromStep, toStep kv.Step) []string {
 				file, _ := f.schema.AccessorIdxFile(version.V1_0, RootNum(fromStep*kv.Step(f.stepSize)), RootNum(toStep*kv.Step(f.stepSize)), 0)
 				return []string{file}
-			})
+			}, f.schema.DataTag(), f.logger)
 	}
 
 	if f.accessors.Has(statecfg.AccessorExistence) {
 		mf[statecfg.AccessorExistence] =
 			fileItemsWithMissedAccessors(f.dirtyFiles.Items(), f.stepSize, func(fromStep, toStep kv.Step) []string {
 				file, _ := f.schema.ExistenceFile(version.V1_0, RootNum(fromStep*kv.Step(f.stepSize)), RootNum(toStep*kv.Step(f.stepSize)))
 				return []string{file}
-			})
+			}, f.schema.DataTag(), f.logger)
 	}
 
 	return (*MissedFilesMap)(&mf)