[3.4] seg: show dict size and mem in seg ls output (cherry-pick) (#20792)

Cherry-pick of #20790 to release/3.4.

Author: AskAlexSharov

db/seg/decompress.go

@@ -509,8 +509,40 @@ func (d *Decompressor) DataHandle() unsafe.Pointer {
 }
 func (d *Decompressor) SerializedDictSize() uint64      { return d.serializedDictSize }
 func (d *Decompressor) SerializedLenSize() uint64       { return d.lenDictSize }
+func (d *Decompressor) SerializedTotalDictSize() uint64 { return d.serializedDictSize + d.lenDictSize }
 func (d *Decompressor) DictWords() int                  { return d.dictWords }
 func (d *Decompressor) DictLens() int                   { return d.dictLens }
+
+// DictMemSize returns the in-memory size of the decoded Huffman table structures
+// (arena-allocated codeword/table/slot slabs). Pattern bytes are subslices of the
+// mmap'd file data and are not included.
+func (d *Decompressor) DictMemSize() uint64 {
+	var total uint64
+	if d.patArena != nil {
+		total += uint64(cap(d.patArena.codewords)) * uint64(unsafe.Sizeof(codeword{}))
+		total += uint64(cap(d.patArena.tables)) * uint64(unsafe.Sizeof(patternTable{}))
+		total += uint64(cap(d.patArena.slots)) * uint64(unsafe.Sizeof((*codeword)(nil)))
+	}
+	if d.posArena != nil {
+		total += uint64(cap(d.posArena.tables)) * uint64(unsafe.Sizeof(posTable{}))
+		total += uint64(cap(d.posArena.entriesArr)) * uint64(unsafe.Sizeof(posEntry{}))
+		total += uint64(cap(d.posArena.ptrsArr)) * uint64(unsafe.Sizeof((*posTable)(nil)))
+	}
+	return total
+}
+
+// Stats accumulates snapshot segment stats for summary logging.
+type Stats struct {
+	Words   uint64
+	Dict    uint64
+	DictMem uint64
+}
+
+func (s *Stats) Add(d *Decompressor) {
+	s.Words += uint64(d.Count())
+	s.Dict += d.SerializedTotalDictSize()
+	s.DictMem += d.DictMemSize()
+}
 func (d *Decompressor) CompressedPageValuesCount() int  { return int(d.compPageValuesCount) }
 func (d *Decompressor) CompressionFormatVersion() uint8 { return d.version }
 

db/snapshotsync/caplin_state_snapshots.go

@@ -215,13 +215,17 @@ func (s *CaplinStateSnapshots) LS() {
 	view := s.View()
 	defer view.Close()
 
+	var stats seg.Stats
 	for _, roTx := range view.roTxs {
 		if roTx != nil {
-			for _, seg := range roTx.Segments {
-				s.logger.Info("[agg] ", "f", seg.src.filePath, "words", seg.src.Decompressor.Count())
+			for _, sn := range roTx.Segments {
+				d := sn.src.Decompressor
+				s.logger.Info("[agg] ", "f", d.FileName(), "words", d.Count(), "dictOnDisk", common.ByteCount(d.SerializedTotalDictSize()), "dictMem", common.ByteCount(d.DictMemSize()))
+				stats.Add(d)
 			}
 		}
 	}
+	s.logger.Info("[agg] total", "words", stats.Words, "dictOnDisk", common.ByteCount(stats.Dict), "dictMem", common.ByteCount(stats.DictMem))
 }
 
 func (s *CaplinStateSnapshots) SegFileNames(from, to uint64) []string {

db/snapshotsync/freezeblocks/caplin_snapshots.go

@@ -109,16 +109,22 @@ func (s *CaplinSnapshots) LS() {
 	view := s.View()
 	defer view.Close()
 
+	var stats seg.Stats
+	lsSeg := func(d *seg.Decompressor) {
+		log.Info("[agg] ", "f", d.FileName(), "words", d.Count(), "dictOnDisk", common.ByteCount(d.SerializedTotalDictSize()), "dictMem", common.ByteCount(d.DictMemSize()))
+		stats.Add(d)
+	}
 	if view.BeaconBlockRotx != nil {
-		for _, seg := range view.BeaconBlockRotx.Segments {
-			log.Info("[agg] ", "f", seg.Src().Decompressor.FileName(), "words", seg.Src().Decompressor.Count())
+		for _, sn := range view.BeaconBlockRotx.Segments {
+			lsSeg(sn.Src().Decompressor)
 		}
 	}
 	if view.BlobSidecarRotx != nil {
-		for _, seg := range view.BlobSidecarRotx.Segments {
-			log.Info("[agg] ", "f", seg.Src().Decompressor.FileName(), "words", seg.Src().Decompressor.Count())
+		for _, sn := range view.BlobSidecarRotx.Segments {
+			lsSeg(sn.Src().Decompressor)
 		}
 	}
+	log.Info("[agg] total", "words", stats.Words, "dictOnDisk", common.ByteCount(stats.Dict), "dictMem", common.ByteCount(stats.DictMem))
 }
 
 func (s *CaplinSnapshots) SegFileNames(from, to uint64) []string {

db/snapshotsync/snapshots.go

@@ -967,14 +967,18 @@ func (s *RoSnapshots) Ls() {
 	view := s.View()
 	defer view.Close()
 
+	var stats seg.Stats
 	for _, t := range s.enums {
-		for _, seg := range s.visible[t] {
-			if seg.src == nil || seg.src.Decompressor == nil {
+		for _, sn := range s.visible[t] {
+			if sn.src == nil || sn.src.Decompressor == nil {
 				continue
 			}
-			log.Info("[snapshots] ", "f", seg.src.Decompressor.FileName(), "count", seg.src.Decompressor.Count())
+			d := sn.src.Decompressor
+			log.Info("[snapshots] ", "f", d.FileName(), "words", d.Count(), "dictOnDisk", common.ByteCount(d.SerializedTotalDictSize()), "dictMem", common.ByteCount(d.DictMemSize()))
+			stats.Add(d)
 		}
 	}
+	log.Info("[snapshots] total", "words", stats.Words, "dictOnDisk", common.ByteCount(stats.Dict), "dictMem", common.ByteCount(stats.DictMem))
 }
 
 func (s *RoSnapshots) Files() (list []string) {

db/state/aggregator.go

@@ -50,6 +50,7 @@ import (
 	"github.com/erigontech/erigon/db/kv/order"
 	"github.com/erigontech/erigon/db/kv/rawdbv3"
 	"github.com/erigontech/erigon/db/kv/stream"
+	"github.com/erigontech/erigon/db/seg"
 	"github.com/erigontech/erigon/db/state/changeset"
 	"github.com/erigontech/erigon/db/state/statecfg"
 	"github.com/erigontech/erigon/db/version"
@@ -669,13 +670,15 @@ func (a *Aggregator) Files() []string {
 	return ac.AllFiles().Fullpaths()
 }
 func (a *Aggregator) LS() {
+	var stats seg.Stats
 	doLS := func(dirtyFiles *btree.BTreeG[*FilesItem]) {
 		dirtyFiles.Walk(func(items []*FilesItem) bool {
 			for _, item := range items {
 				if item.decompressor == nil {
 					continue
 				}
-				a.logger.Info("[agg] ", "f", item.decompressor.FileName(), "words", item.decompressor.Count())
+				a.logger.Info("[agg] ", "f", item.decompressor.FileName(), "words", item.decompressor.Count(), "dictOnDisk", common.ByteCount(item.decompressor.SerializedTotalDictSize()), "dictMem", common.ByteCount(item.decompressor.DictMemSize()))
+				stats.Add(item.decompressor)
 			}
 			return true
 		})
@@ -691,6 +694,7 @@ func (a *Aggregator) LS() {
 	for _, d := range a.iis {
 		doLS(d.dirtyFiles)
 	}
+	a.logger.Info("[agg] total", "words", stats.Words, "dictOnDisk", common.ByteCount(stats.Dict), "dictMem", common.ByteCount(stats.DictMem))
 }
 
 func (a *Aggregator) WaitForBuildAndMerge(ctx context.Context) chan struct{} {