feat(migration): statehistory migration

migration/state/newstate/internal/common/committer.go

@@ -0,0 +1,58 @@
+package common
+
+import (
+	"github.com/NethermindEth/juno/db"
+	"github.com/NethermindEth/juno/migration/pipeline"
+	"github.com/NethermindEth/juno/migration/semaphore"
+	"github.com/NethermindEth/juno/utils/log"
+	"go.uber.org/zap"
+)
+
+type Committer struct {
+	logger         log.StructuredLogger
+	counter        Counter
+	batchSemaphore semaphore.ResourceSemaphore[db.Batch]
+	phaseName      string
+}
+
+var _ pipeline.State[Task, struct{}] = (*Committer)(nil)
+
+func NewCommitter(
+	logger log.StructuredLogger,
+	batchSemaphore semaphore.ResourceSemaphore[db.Batch],
+	phaseName string,
+) *Committer {
+	return &Committer{
+		logger:         logger,
+		counter:        NewCounter(logger, TimeLogRate, phaseName),
+		batchSemaphore: batchSemaphore,
+		phaseName:      phaseName,
+	}
+}
+
+func (c *Committer) Run(_ int, t Task, _ chan<- struct{}) error {
+	defer c.batchSemaphore.Put()
+
+	fields := make([]zap.Field, 0, 4)
+	if c.phaseName != "" {
+		fields = append(fields, zap.String("phase", c.phaseName))
+	}
+	fields = append(fields,
+		zap.Int("completedAddrs", t.CompletedAddrs),
+		zap.Int("entryCount", t.EntryCount),
+		zap.Int("batchSize", t.Batch.Size()),
+	)
+	c.logger.Debug("writing batch", fields...)
+
+	byteSize := uint64(t.Batch.Size())
+	if err := t.Batch.Write(); err != nil {
+		return err
+	}
+
+	c.counter.Log(byteSize, t.CompletedAddrs, t.EntryCount)
+	return nil
+}
+
+func (c *Committer) Done(int, chan<- struct{}) error {
+	return nil
+}

migration/state/newstate/internal/common/constants.go

@@ -0,0 +1,14 @@
+package common
+
+import (
+	"time"
+
+	"github.com/NethermindEth/juno/db"
+)
+
+const (
+	BatchByteSize       = 128 * db.Megabyte
+	TargetBatchByteSize = 96 * db.Megabyte
+	IngestorCount       = 4
+	TimeLogRate         = 5 * time.Second
+)

migration/state/newstate/internal/common/counter.go

@@ -0,0 +1,67 @@
+package common
+
+import (
+	"math"
+	"time"
+
+	"github.com/NethermindEth/juno/db"
+	"github.com/NethermindEth/juno/utils/log"
+	"go.uber.org/zap"
+)
+
+type Counter struct {
+	logger         log.StructuredLogger
+	timeLogRate    time.Duration
+	phaseName      string
+	start          time.Time
+	size           uint64
+	completedAddrs uint64
+	entryCount     uint64
+}
+
+func NewCounter(logger log.StructuredLogger, timeLogRate time.Duration, phaseName string) Counter {
+	if zl, ok := logger.(*log.ZapLogger); ok {
+		logger = zl.WithOptions(zap.AddCallerSkip(1))
+	}
+	return Counter{
+		logger:      logger,
+		timeLogRate: timeLogRate,
+		phaseName:   phaseName,
+		start:       time.Now(),
+	}
+}
+
+func (c *Counter) Log(byteSize uint64, completedAddrs, entryCount int) {
+	c.size += byteSize
+	c.completedAddrs += uint64(completedAddrs)
+	c.entryCount += uint64(entryCount)
+
+	const cent = 100
+
+	now := time.Now()
+	elapsed := now.Sub(c.start).Seconds()
+	if elapsed <= c.timeLogRate.Seconds() {
+		return
+	}
+
+	mbs := float64(c.size) / float64(db.Megabyte)
+	fields := make([]zap.Field, 0, 8)
+	if c.phaseName != "" {
+		fields = append(fields, zap.String("phase", c.phaseName))
+	}
+	fields = append(fields,
+		zap.Float64("MB", math.Round(mbs*cent)/cent),
+		zap.Float64("MB/s", math.Round(mbs/elapsed*cent)/cent),
+		zap.Uint64("completedContracts", c.completedAddrs),
+		zap.Float64("completedContracts/s", float64(c.completedAddrs)/elapsed),
+		zap.Uint64("entries", c.entryCount),
+		zap.Float64("entries/s", float64(c.entryCount)/elapsed),
+		zap.Float64("time", elapsed),
+	)
+	c.logger.Info("write speed", fields...)
+
+	c.start = now
+	c.size = 0
+	c.completedAddrs = 0
+	c.entryCount = 0
+}