cache directory entries to avoid expensive filepath.Glob in index-related ops

agents.md

@@ -51,3 +51,5 @@ Erigon is a high-performance Ethereum execution client with embedded consensus l
 ## Conventions
 
 Commit messages: prefix with package(s) modified, e.g., `eth, rpc: make trace configs optional`
+
+**Important**: Always run `make lint` after making code changes and before committing. Fix any linter errors before proceeding.

db/snapshotsync/freezeblocks/caplin_snapshots.go

@@ -161,6 +161,17 @@ func (s *CaplinSnapshots) OpenList(fileNames []string, optimistic bool) error {
 	defer s.dirtyLock.Unlock()
 
 	s.closeWhatNotInList(fileNames)
+
+	// Get idx files for efficient index file lookups
+	idxFiles, err := snaptype.IdxFiles(s.dir)
+	if err != nil {
+		return fmt.Errorf("read idx files %s: %w", s.dir, err)
+	}
+	dirEntries := make([]string, 0, len(idxFiles))
+	for _, f := range idxFiles {
+		dirEntries = append(dirEntries, f.Name())
+	}
+
 	var segmentsMax uint64
 	var segmentsMaxSet bool
 Loop:
@@ -215,7 +226,7 @@ Loop:
 				// then make segment available even if index open may fail
 				s.dirty[snaptype.BeaconBlocks.Enum()].Set(sn)
 			}
-			if err := sn.OpenIdxIfNeed(s.dir, optimistic); err != nil {
+			if err := sn.OpenIdxIfNeed(s.dir, optimistic, dirEntries); err != nil {
 				return err
 			}
 			// Only bob sidecars count for progression
@@ -271,7 +282,7 @@ Loop:
 				// then make segment available even if index open may fail
 				s.dirty[snaptype.BlobSidecars.Enum()].Set(sn)
 			}
-			if err := sn.OpenIdxIfNeed(s.dir, optimistic); err != nil {
+			if err := sn.OpenIdxIfNeed(s.dir, optimistic, dirEntries); err != nil {
 				return err
 			}
 		}

db/snapshotsync/merger.go

@@ -129,7 +129,7 @@ func (m *Merger) mergeSubSegment(
 		if err = buildIdx(ctx, sn, indexBuilder, m.chainConfig, m.tmpDir, p, m.lvl, m.logger); err != nil {
 			return
 		}
-		err = newDirtySegment.openIdx(snapDir)
+		err = newDirtySegment.openIdx(snapDir, nil)
 		if err != nil {
 			return
 		}

db/snapshotsync/snapshots.go

@@ -434,13 +434,13 @@ func (s *DirtySegment) closeAndRemoveFiles() {
 	}
 }
 
-func (s *DirtySegment) OpenIdxIfNeed(dir string, optimistic bool) (err error) {
+func (s *DirtySegment) OpenIdxIfNeed(dir string, optimistic bool, dirEntries []string) (err error) {
 	if len(s.Type().IdxFileNames(s.from, s.to)) == 0 {
 		return nil
 	}
 
 	if s.refcount.Load() == 0 {
-		err = s.openIdx(dir)
+		err = s.openIdx(dir, dirEntries)
 
 		if err != nil {
 			if !errors.Is(err, os.ErrNotExist) {
@@ -456,7 +456,7 @@ func (s *DirtySegment) OpenIdxIfNeed(dir string, optimistic bool) (err error) {
 	return nil
 }
 
-func (s *DirtySegment) openIdx(dir string) (err error) {
+func (s *DirtySegment) openIdx(dir string, dirEntries []string) (err error) {
 	if s.Decompressor == nil {
 		return nil
 	}
@@ -469,11 +469,18 @@ func (s *DirtySegment) openIdx(dir string) (err error) {
 		if s.indexes[i] != nil {
 			continue
 		}
-		fPathMask, err := version.ReplaceVersionWithMask(filepath.Join(dir, fileName))
+		fPathMask, err := version.ReplaceVersionWithMask(fileName)
 		if err != nil {
 			return fmt.Errorf("[open index] can't replace with mask in file %s: %w", fileName, err)
 		}
-		fPath, _, ok, err := version.FindFilesWithVersionsByPattern(fPathMask)
+
+		var fPath string
+		var ok bool
+		if dirEntries != nil {
+			fPath, _, ok, err = version.MatchVersionedFile(fPathMask, dirEntries, dir)
+		} else {
+			fPath, _, ok, err = version.FindFilesWithVersionsByPattern(filepath.Join(dir, fPathMask))
+		}
 		if err != nil {
 			return fmt.Errorf("%w, fileName: %s", err, fileName)
 		}
@@ -1082,6 +1089,21 @@ func (s *RoSnapshots) openSegments(fileNames []string, open bool, optimistic boo
 
 	snConfig, _ := snapcfg.KnownCfg(s.cfg.ChainName)
 
+	// Read full directory listing once for efficient index file lookups
+	var dirEntries []string
+	if open {
+		entries, err := os.ReadDir(s.dir)
+		if err != nil && !os.IsNotExist(err) {
+			return fmt.Errorf("read dir %s: %w", s.dir, err)
+		}
+		dirEntries = make([]string, 0, len(entries))
+		for _, e := range entries {
+			if !e.IsDir() {
+				dirEntries = append(dirEntries, e.Name())
+			}
+		}
+	}
+
 	for _, fName := range fileNames {
 		f, isState, ok := snaptype.ParseFileName(s.dir, fName)
 		if !ok || isState || snaptype.IsTorrentPartial(f.Ext) {
@@ -1142,7 +1164,7 @@ func (s *RoSnapshots) openSegments(fileNames []string, open bool, optimistic boo
 
 		if open {
 			wg.Go(func() error {
-				if err := sn.OpenIdxIfNeed(s.dir, optimistic); err != nil {
+				if err := sn.OpenIdxIfNeed(s.dir, optimistic, dirEntries); err != nil {
 					return err
 				}
 				return nil

db/state/aggregator.go

@@ -331,7 +331,6 @@ func (a *Aggregator) OpenFolder() error {
 	return nil
 }
 
-// TODO: convert this func to `map` or struct instead of 4 return params
 func scanDirs(dirs datadir.Dirs) (r *ScanDirsResult, err error) {
 	r = &ScanDirsResult{}
 	r.iiFiles, err = filesFromDir(dirs.SnapIdx)
@@ -346,13 +345,18 @@ func scanDirs(dirs datadir.Dirs) (r *ScanDirsResult, err error) {
 	if err != nil {
 		return
 	}
+	r.accessorFiles, err = filesFromDir(dirs.SnapAccessors)
+	if err != nil {
+		return
+	}
 	return r, nil
 }
 
 type ScanDirsResult struct {
-	domainFiles  []string
-	historyFiles []string
-	iiFiles      []string
+	domainFiles   []string
+	historyFiles  []string
+	iiFiles       []string
+	accessorFiles []string
 }
 
 func (a *Aggregator) openFolder() error {

db/state/dirty_files.go

@@ -320,15 +320,15 @@ func deleteMergeFile(dirtyFiles *btree2.BTreeG[*FilesItem], outs []*FilesItem, f
 	}
 }
 
-func (d *Domain) openDirtyFiles() (err error) {
+func (d *Domain) openDirtyFiles(dirEntries []string) (err error) {
 	invalidFileItems := make([]*FilesItem, 0)
 	invalidFileItemsLock := sync.Mutex{}
 	d.dirtyFiles.Walk(func(items []*FilesItem) bool {
 		for _, item := range items {
 			fromStep, toStep := item.StepRange(d.stepSize)
 			if item.decompressor == nil {
-				fPathMask := d.kvFilePathMask(fromStep, toStep)
-				fPath, fileVer, ok, err := version.FindFilesWithVersionsByPattern(fPathMask)
+				fNameMask := d.kvFileNameMask(fromStep, toStep)
+				fPath, fileVer, ok, err := version.MatchVersionedFile(fNameMask, dirEntries, d.dirs.SnapDomain)
 				if err != nil {
 					_, fName := filepath.Split(fPath)
 					d.logger.Debug("[agg] Domain.openDirtyFiles: FileExist err", "f", fName, "err", err)
@@ -338,7 +338,7 @@ func (d *Domain) openDirtyFiles() (err error) {
 					continue
 				}
 				if !ok {
-					_, fName := filepath.Split(fPath)
+					fName := fNameMask
 					d.logger.Debug("[agg] Domain.openDirtyFiles: file does not exists", "f", fName)
 					invalidFileItemsLock.Lock()
 					invalidFileItems = append(invalidFileItems, item)
@@ -367,8 +367,8 @@ func (d *Domain) openDirtyFiles() (err error) {
 			}
 
 			if item.index == nil && d.Accessors.Has(statecfg.AccessorHashMap) {
-				fPathMask := d.kviAccessorFilePathMask(fromStep, toStep)
-				fPath, fileVer, ok, err := version.FindFilesWithVersionsByPattern(fPathMask)
+				fNameMask := d.kviAccessorFileNameMask(fromStep, toStep)
+				fPath, fileVer, ok, err := version.MatchVersionedFile(fNameMask, dirEntries, d.dirs.SnapDomain)
 				if err != nil {
 					_, fName := filepath.Split(fPath)
 					d.logger.Warn("[agg] Domain.openDirtyFiles", "err", err, "f", fName)
@@ -386,8 +386,8 @@ func (d *Domain) openDirtyFiles() (err error) {
 				}
 			}
 			if item.bindex == nil && d.Accessors.Has(statecfg.AccessorBTree) {
-				fPathMask := d.kvBtAccessorFilePathMask(fromStep, toStep)
-				fPath, fileVer, ok, err := version.FindFilesWithVersionsByPattern(fPathMask)
+				fNameMask := d.kvBtAccessorFileNameMask(fromStep, toStep)
+				fPath, fileVer, ok, err := version.MatchVersionedFile(fNameMask, dirEntries, d.dirs.SnapDomain)
 				if err != nil {
 					_, fName := filepath.Split(fPath)
 					d.logger.Warn("[agg] Domain.openDirtyFiles", "err", err, "f", fName)
@@ -405,8 +405,8 @@ func (d *Domain) openDirtyFiles() (err error) {
 				}
 			}
 			if item.existence == nil && d.Accessors.Has(statecfg.AccessorExistence) {
-				fPathMask := d.kvExistenceIdxFilePathMask(fromStep, toStep)
-				fPath, fileVer, ok, err := version.FindFilesWithVersionsByPattern(fPathMask)
+				fNameMask := d.kvExistenceIdxFileNameMask(fromStep, toStep)
+				fPath, fileVer, ok, err := version.MatchVersionedFile(fNameMask, dirEntries, d.dirs.SnapDomain)
 				if err != nil {
 					_, fName := filepath.Split(fPath)
 					d.logger.Warn("[agg] Domain.openDirtyFiles", "err", err, "f", fName)
@@ -435,15 +435,15 @@ func (d *Domain) openDirtyFiles() (err error) {
 	return nil
 }
 
-func (h *History) openDirtyFiles() error {
+func (h *History) openDirtyFiles(dataEntries, accessorEntries []string) error {
 	invalidFilesMu := sync.Mutex{}
 	invalidFileItems := make([]*FilesItem, 0)
 	h.dirtyFiles.Walk(func(items []*FilesItem) bool {
 		for _, item := range items {
 			fromStep, toStep := item.StepRange(h.stepSize)
 			if item.decompressor == nil {
-				fPathMask := h.vFilePathMask(fromStep, toStep)
-				fPath, fileVer, ok, err := version.FindFilesWithVersionsByPattern(fPathMask)
+				fNameMask := h.vFileNameMask(fromStep, toStep)
+				fPath, fileVer, ok, err := version.MatchVersionedFile(fNameMask, dataEntries, h.dirs.SnapHistory)
 				if err != nil {
 					_, fName := filepath.Split(fPath)
 					h.logger.Debug("[agg] History.openDirtyFiles: FileExist", "f", fName, "err", err)
@@ -453,7 +453,7 @@ func (h *History) openDirtyFiles() error {
 					continue
 				}
 				if !ok {
-					_, fName := filepath.Split(fPath)
+					fName := fNameMask
 					h.logger.Debug("[agg] History.openDirtyFiles: file does not exists", "f", fName)
 					invalidFilesMu.Lock()
 					invalidFileItems = append(invalidFileItems, item)
@@ -494,8 +494,8 @@ func (h *History) openDirtyFiles() error {
 			}
 
 			if item.index == nil {
-				fPathMask := h.vAccessorFilePathMask(fromStep, toStep)
-				fPath, fileVer, ok, err := version.FindFilesWithVersionsByPattern(fPathMask)
+				fNameMask := h.vAccessorFileNameMask(fromStep, toStep)
+				fPath, fileVer, ok, err := version.MatchVersionedFile(fNameMask, accessorEntries, h.dirs.SnapAccessors)
 				if err != nil {
 					_, fName := filepath.Split(fPath)
 					h.logger.Warn("[agg] History.openDirtyFiles", "err", err, "f", fName)
@@ -523,26 +523,26 @@ func (h *History) openDirtyFiles() error {
 	return nil
 }
 
-func (ii *InvertedIndex) openDirtyFiles() error {
+func (ii *InvertedIndex) openDirtyFiles(dataEntries, accessorEntries []string) error {
 	var invalidFileItems []*FilesItem
 	invalidFileItemsLock := sync.Mutex{}
 	ii.dirtyFiles.Walk(func(items []*FilesItem) bool {
 		for _, item := range items {
 			fromStep, toStep := item.StepRange(ii.stepSize)
 			if item.decompressor == nil {
-				fPathPattern := ii.efFilePathMask(fromStep, toStep)
-				fPath, fileVer, ok, err := version.FindFilesWithVersionsByPattern(fPathPattern)
+				fNameMask := ii.efFileNameMask(fromStep, toStep)
+				fPath, fileVer, ok, err := version.MatchVersionedFile(fNameMask, dataEntries, ii.dirs.SnapIdx)
 				if err != nil {
 					_, fName := filepath.Split(fPath)
-					ii.logger.Debug("[agg] InvertedIndex.openDirtyFiles: FindFilesWithVersionsByPattern error", "f", fName, "err", err)
+					ii.logger.Debug("[agg] InvertedIndex.openDirtyFiles: MatchVersionedFile error", "f", fName, "err", err)
 					invalidFileItemsLock.Lock()
 					invalidFileItems = append(invalidFileItems, item)
 					invalidFileItemsLock.Unlock()
 					continue
 				}
 
 				if !ok {
-					_, fName := filepath.Split(fPath)
+					fName := fNameMask
 					ii.logger.Debug("[agg] InvertedIndex.openDirtyFiles: file does not exists", "f", fName)
 					invalidFileItemsLock.Lock()
 					invalidFileItems = append(invalidFileItems, item)
@@ -571,8 +571,8 @@ func (ii *InvertedIndex) openDirtyFiles() error {
 			}
 
 			if item.index == nil {
-				fPathPattern := ii.efAccessorFilePathMask(fromStep, toStep)
-				fPath, fileVer, ok, err := version.FindFilesWithVersionsByPattern(fPathPattern)
+				fNameMask := ii.efAccessorFileNameMask(fromStep, toStep)
+				fPath, fileVer, ok, err := version.MatchVersionedFile(fNameMask, accessorEntries, ii.dirs.SnapAccessors)
 				if err != nil {
 					_, fName := filepath.Split(fPath)
 					ii.logger.Warn("[agg] InvertedIndex.openDirtyFiles", "err", err, "f", fName)

db/state/domain.go

@@ -166,6 +166,19 @@ func (d *Domain) kvBtAccessorFilePathMask(fromStep, toStep kv.Step) string {
 	return filepath.Join(d.dirs.SnapDomain, fmt.Sprintf("*-%s.%d-%d.bt", d.FilenameBase, fromStep, toStep))
 }
 
+func (d *Domain) kvFileNameMask(fromStep, toStep kv.Step) string {
+	return fmt.Sprintf("*-%s.%d-%d.kv", d.FilenameBase, fromStep, toStep)
+}
+func (d *Domain) kviAccessorFileNameMask(fromStep, toStep kv.Step) string {
+	return fmt.Sprintf("*-%s.%d-%d.kvi", d.FilenameBase, fromStep, toStep)
+}
+func (d *Domain) kvExistenceIdxFileNameMask(fromStep, toStep kv.Step) string {
+	return fmt.Sprintf("*-%s.%d-%d.kvei", d.FilenameBase, fromStep, toStep)
+}
+func (d *Domain) kvBtAccessorFileNameMask(fromStep, toStep kv.Step) string {
+	return fmt.Sprintf("*-%s.%d-%d.bt", d.FilenameBase, fromStep, toStep)
+}
+
 // maxStepInDB - return the latest available step in db (at-least 1 value in such step)
 func (d *Domain) maxStepInDB(tx kv.Tx) (lstInDb kv.Step) {
 	lstIdx, _ := kv.LastKey(tx, d.History.KeysTable)
@@ -214,14 +227,14 @@ func (dt *DomainRoTx) NewWriter() *DomainBufferedWriter { return dt.newWriter(dt
 // It's ok if some files was open earlier.
 // If some file already open: noop.
 // If some file already open but not in provided list: close and remove from `files` field.
-func (d *Domain) OpenList(idxFiles, histFiles, domainFiles []string) error {
-	if err := d.History.openList(idxFiles, histFiles); err != nil {
+func (d *Domain) OpenList(scanResult ScanDirsResult) error {
+	if err := d.History.openList(scanResult.iiFiles, scanResult.historyFiles, scanResult.accessorFiles); err != nil {
 		return err
 	}
 
-	d.closeWhatNotInList(domainFiles)
-	d.scanDirtyFiles(domainFiles)
-	if err := d.openDirtyFiles(); err != nil {
+	d.closeWhatNotInList(scanResult.domainFiles)
+	d.scanDirtyFiles(scanResult.domainFiles)
+	if err := d.openDirtyFiles(scanResult.domainFiles); err != nil {
 		return fmt.Errorf("Domain(%s).openList: %w", d.FilenameBase, err)
 	}
 	d.protectFromHistoryFilesAheadOfDomainFiles()
@@ -239,11 +252,7 @@ func (d *Domain) openFolder(r *ScanDirsResult) error {
 	if d.Disable {
 		return nil
 	}
-
-	if err := d.OpenList(r.iiFiles, r.historyFiles, r.domainFiles); err != nil {
-		return err
-	}
-	return nil
+	return d.OpenList(*r)
 }
 
 func (d *Domain) closeFilesAfterStep(lowerBound kv.Step) {

db/state/history.go

@@ -120,7 +120,10 @@ func (h *History) vFilePathMask(fromStep, toStep kv.Step) string {
 	return filepath.Join(h.dirs.SnapHistory, h.vFileNameMask(fromStep, toStep))
 }
 func (h *History) vAccessorFilePathMask(fromStep, toStep kv.Step) string {
-	return filepath.Join(h.dirs.SnapAccessors, fmt.Sprintf("*-%s.%d-%d.vi", h.FilenameBase, fromStep, toStep))
+	return filepath.Join(h.dirs.SnapAccessors, h.vAccessorFileNameMask(fromStep, toStep))
+}
+func (h *History) vAccessorFileNameMask(fromStep, toStep kv.Step) string {
+	return fmt.Sprintf("*-%s.%d-%d.vi", h.FilenameBase, fromStep, toStep)
 }
 
 func (h *History) openHashMapAccessor(fPath string) (*recsplit.Index, error) {
@@ -135,21 +138,21 @@ func (h *History) openHashMapAccessor(fPath string) (*recsplit.Index, error) {
 // It's ok if some files was open earlier.
 // If some file already open: noop.
 // If some file already open but not in provided list: close and remove from `files` field.
-func (h *History) openList(idxFiles, histNames []string) error {
-	if err := h.InvertedIndex.openList(idxFiles); err != nil {
+func (h *History) openList(idxFiles, histNames, accessorFiles []string) error {
+	if err := h.InvertedIndex.openList(idxFiles, accessorFiles); err != nil {
 		return err
 	}
 
 	h.closeWhatNotInList(histNames)
 	h.scanDirtyFiles(histNames)
-	if err := h.openDirtyFiles(); err != nil {
+	if err := h.openDirtyFiles(histNames, accessorFiles); err != nil {
 		return fmt.Errorf("History(%s).openList: %w", h.FilenameBase, err)
 	}
 	return nil
 }
 
 func (h *History) openFolder(scanDirsRes *ScanDirsResult) error {
-	return h.openList(scanDirsRes.iiFiles, scanDirsRes.historyFiles)
+	return h.openList(scanDirsRes.iiFiles, scanDirsRes.historyFiles, scanDirsRes.accessorFiles)
 }
 
 func (h *History) scanDirtyFiles(fileNames []string) {