Skip to content

Commit a78c658

Browse files
eastorskieastorski
authored
Cherry-pick: Fixed milestone pruning and fixed BorEvents snapshots (#15914)
1 parent afbc003 commit a78c658

File tree

7 files changed

+51
-43
lines changed

7 files changed

+51
-43
lines changed

‎erigon-lib/downloader/snaptype/type.go‎

Lines changed: 14 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,7 @@ import (
2929
"sync"
3030

3131
"github.com/erigontech/erigon-lib/chain"
32+
"github.com/erigontech/erigon-lib/common"
3233
"github.com/erigontech/erigon-lib/common/background"
3334
"github.com/erigontech/erigon-lib/common/dbg"
3435
"github.com/erigontech/erigon-lib/common/dir"
@@ -72,14 +73,18 @@ type Versions struct {
7273

7374
type FirstKeyGetter func(ctx context.Context) uint64
7475

76+
type BlockHashResolver interface {
77+
CanonicalHash(ctx context.Context, tx kv.Getter, blockHeight uint64) (h common.Hash, ok bool, err error)
78+
}
79+
7580
type RangeExtractor interface {
76-
Extract(ctx context.Context, blockFrom, blockTo uint64, firstKey FirstKeyGetter, db kv.RoDB, chainConfig *chain.Config, collect func([]byte) error, workers int, lvl log.Lvl, logger log.Logger) (uint64, error)
81+
Extract(ctx context.Context, blockFrom, blockTo uint64, firstKey FirstKeyGetter, db kv.RoDB, chainConfig *chain.Config, collect func([]byte) error, workers int, lvl log.Lvl, logger log.Logger, hashResolver BlockHashResolver) (uint64, error)
7782
}
7883

79-
type RangeExtractorFunc func(ctx context.Context, blockFrom, blockTo uint64, firstKey FirstKeyGetter, db kv.RoDB, chainConfig *chain.Config, collect func([]byte) error, workers int, lvl log.Lvl, logger log.Logger) (uint64, error)
84+
type RangeExtractorFunc func(ctx context.Context, blockFrom, blockTo uint64, firstKey FirstKeyGetter, db kv.RoDB, chainConfig *chain.Config, collect func([]byte) error, workers int, lvl log.Lvl, logger log.Logger, hashResolver BlockHashResolver) (uint64, error)
8085

81-
func (f RangeExtractorFunc) Extract(ctx context.Context, blockFrom, blockTo uint64, firstKey FirstKeyGetter, db kv.RoDB, chainConfig *chain.Config, collect func([]byte) error, workers int, lvl log.Lvl, logger log.Logger) (uint64, error) {
82-
return f(ctx, blockFrom, blockTo, firstKey, db, chainConfig, collect, workers, lvl, logger)
86+
func (f RangeExtractorFunc) Extract(ctx context.Context, blockFrom, blockTo uint64, firstKey FirstKeyGetter, db kv.RoDB, chainConfig *chain.Config, collect func([]byte) error, workers int, lvl log.Lvl, logger log.Logger, hashResolver BlockHashResolver) (uint64, error) {
87+
return f(ctx, blockFrom, blockTo, firstKey, db, chainConfig, collect, workers, lvl, logger, hashResolver)
8388
}
8489

8590
type IndexBuilder interface {
@@ -199,7 +204,7 @@ type Type interface {
199204
Indexes() []Index
200205
HasIndexFiles(info FileInfo, logger log.Logger) bool
201206
BuildIndexes(ctx context.Context, info FileInfo, indexBuilder IndexBuilder, chainConfig *chain.Config, tmpDir string, p *background.Progress, lvl log.Lvl, logger log.Logger) error
202-
ExtractRange(ctx context.Context, info FileInfo, rangeExtractor RangeExtractor, indexBuilder IndexBuilder, firstKeyGetter FirstKeyGetter, db kv.RoDB, chainConfig *chain.Config, tmpDir string, workers int, lvl log.Lvl, logger log.Logger) (uint64, error)
207+
ExtractRange(ctx context.Context, info FileInfo, rangeExtractor RangeExtractor, indexBuilder IndexBuilder, firstKeyGetter FirstKeyGetter, db kv.RoDB, chainConfig *chain.Config, tmpDir string, workers int, lvl log.Lvl, logger log.Logger, hashResolver BlockHashResolver) (uint64, error)
203208

204209
RangeExtractor() RangeExtractor
205210
}
@@ -269,11 +274,11 @@ func (s snapType) FileInfo(dir string, from uint64, to uint64) FileInfo {
269274
return f
270275
}
271276

272-
func (s snapType) ExtractRange(ctx context.Context, info FileInfo, rangeExtractor RangeExtractor, indexBuilder IndexBuilder, firstKeyGetter FirstKeyGetter, db kv.RoDB, chainConfig *chain.Config, tmpDir string, workers int, lvl log.Lvl, logger log.Logger) (uint64, error) {
277+
func (s snapType) ExtractRange(ctx context.Context, info FileInfo, rangeExtractor RangeExtractor, indexBuilder IndexBuilder, firstKeyGetter FirstKeyGetter, db kv.RoDB, chainConfig *chain.Config, tmpDir string, workers int, lvl log.Lvl, logger log.Logger, hashResolver BlockHashResolver) (uint64, error) {
273278
if rangeExtractor == nil {
274279
rangeExtractor = s.rangeExtractor
275280
}
276-
return ExtractRange(ctx, info, rangeExtractor, indexBuilder, firstKeyGetter, db, chainConfig, tmpDir, workers, lvl, logger)
281+
return ExtractRange(ctx, info, rangeExtractor, indexBuilder, firstKeyGetter, db, chainConfig, tmpDir, workers, lvl, logger, hashResolver)
277282
}
278283

279284
func (s snapType) Indexes() []Index {
@@ -554,7 +559,7 @@ func BuildIndexWithSnapName(ctx context.Context, info FileInfo, cfg recsplit.Rec
554559
}
555560
}
556561

557-
func ExtractRange(ctx context.Context, f FileInfo, extractor RangeExtractor, indexBuilder IndexBuilder, firstKey FirstKeyGetter, chainDB kv.RoDB, chainConfig *chain.Config, tmpDir string, workers int, lvl log.Lvl, logger log.Logger) (uint64, error) {
562+
func ExtractRange(ctx context.Context, f FileInfo, extractor RangeExtractor, indexBuilder IndexBuilder, firstKey FirstKeyGetter, chainDB kv.RoDB, chainConfig *chain.Config, tmpDir string, workers int, lvl log.Lvl, logger log.Logger, hashResolver BlockHashResolver) (uint64, error) {
558563
var lastKeyValue uint64
559564

560565
sn, err := seg.NewCompressor(ctx, "Snapshot "+f.Type.Name(), f.Path, tmpDir, seg.DefaultCfg, lvl, logger)
@@ -566,7 +571,7 @@ func ExtractRange(ctx context.Context, f FileInfo, extractor RangeExtractor, ind
566571

567572
lastKeyValue, err = extractor.Extract(ctx, f.From, f.To, firstKey, chainDB, chainConfig, func(v []byte) error {
568573
return sn.AddWord(v)
569-
}, workers, lvl, logger)
574+
}, workers, lvl, logger, hashResolver)
570575

571576
if err != nil {
572577
return lastKeyValue, fmt.Errorf("ExtractRange: %w", err)

‎eth/backend.go‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -615,6 +615,15 @@ func New(ctx context.Context, stack *node.Node, config *ethconfig.Config, logger
615615
EventFetcher: heimdallClient,
616616
})
617617

618+
if err := heimdallStore.Milestones().Prepare(ctx); err != nil {
619+
return nil, err
620+
}
621+
622+
_, err := heimdallStore.Milestones().DeleteFromBlockNum(ctx, 0)
623+
if err != nil {
624+
return nil, err
625+
}
626+
618627
heimdallService = heimdall.NewService(heimdall.ServiceConfig{
619628
Store: heimdallStore,
620629
BorConfig: borConfig,

‎polygon/bor/bordb/prune.go‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -121,13 +121,13 @@ func UnwindHeimdall(ctx context.Context, heimdallStore heimdall.Store, bridgeSto
121121
}
122122
}
123123

124-
if heimdall.CheckpointsEnabled() && !unwindCfg.KeepCheckpoints {
124+
if heimdall.WaypointsEnabled() && !unwindCfg.KeepCheckpoints {
125125
if err := UnwindCheckpoints(ctx, heimdallStore, tx, unwindPoint); err != nil {
126126
return err
127127
}
128128
}
129129

130-
if heimdall.MilestonesEnabled() && !unwindCfg.KeepMilestones {
130+
if heimdall.WaypointsEnabled() && !unwindCfg.KeepMilestones {
131131
if err := UnwindMilestones(ctx, heimdallStore, tx, unwindPoint); err != nil {
132132
return err
133133
}
@@ -220,7 +220,7 @@ func PruneHeimdall(ctx context.Context, heimdallStore heimdall.Store, bridgeStor
220220
return deleted, err
221221
}
222222

223-
if heimdall.CheckpointsEnabled() {
223+
if heimdall.WaypointsEnabled() {
224224
checkpointStore := heimdallStore.Checkpoints()
225225

226226
if tx != nil {
@@ -240,7 +240,7 @@ func PruneHeimdall(ctx context.Context, heimdallStore heimdall.Store, bridgeStor
240240
}
241241
}
242242

243-
if heimdall.MilestonesEnabled() {
243+
if heimdall.WaypointsEnabled() {
244244
milestoneStore := heimdallStore.Milestones()
245245

246246
if tx != nil {

‎polygon/heimdall/entity_store.go‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -473,7 +473,7 @@ func (s txEntityStore[TEntity]) DeleteFromBlockNum(ctx context.Context, unwindPo
473473
}
474474

475475
defer cursor.Close()
476-
lastEntityToKeep, ok, err := s.EntityIdFromBlockNum(ctx, unwindPoint)
476+
firstEntityToDelete, ok, err := s.EntityIdFromBlockNum(ctx, unwindPoint+1)
477477
if err != nil {
478478
return 0, err
479479
}
@@ -482,7 +482,7 @@ func (s txEntityStore[TEntity]) DeleteFromBlockNum(ctx context.Context, unwindPo
482482
return 0, nil
483483
}
484484

485-
var entityKey = entityStoreKey(lastEntityToKeep + 1)
485+
var entityKey = entityStoreKey(firstEntityToDelete)
486486
var k []byte
487487
var deleted int
488488
for k, _, err = cursor.Seek(entityKey[:]); err == nil && k != nil; k, _, err = cursor.Next() {

‎polygon/heimdall/snapshot_store.go‎

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,7 @@ func NewSnapshotStore(base Store, snapshots *RoSnapshots) *SnapshotStore {
2323
return &SnapshotStore{
2424
Store: base,
2525
checkpoints: NewCheckpointSnapshotStore(base.Checkpoints(), snapshots),
26-
milestones: NewMilestoneSnapshotStore(base.Milestones(), snapshots),
26+
milestones: base.Milestones(),
2727
spans: NewSpanSnapshotStore(base.Spans(), snapshots),
2828
spanBlockProducerSelections: base.SpanBlockProducerSelections(),
2929
}
@@ -87,9 +87,9 @@ func (s *SpanSnapshotStore) WithTx(tx kv.Tx) EntityStore[*Span] {
8787

8888
func (s *SpanSnapshotStore) RangeExtractor() snaptype.RangeExtractor {
8989
return snaptype.RangeExtractorFunc(
90-
func(ctx context.Context, blockFrom, blockTo uint64, firstKey snaptype.FirstKeyGetter, db kv.RoDB, chainConfig *chain.Config, collect func([]byte) error, workers int, lvl log.Lvl, logger log.Logger) (uint64, error) {
90+
func(ctx context.Context, blockFrom, blockTo uint64, firstKey snaptype.FirstKeyGetter, db kv.RoDB, chainConfig *chain.Config, collect func([]byte) error, workers int, lvl log.Lvl, logger log.Logger, hashResolver snaptype.BlockHashResolver) (uint64, error) {
9191
return s.SnapType().RangeExtractor().Extract(ctx, blockFrom, blockTo, firstKey,
92-
s.EntityStore.(*mdbxEntityStore[*Span]).db.RoDB(), chainConfig, collect, workers, lvl, logger)
92+
s.EntityStore.(*mdbxEntityStore[*Span]).db.RoDB(), chainConfig, collect, workers, lvl, logger, hashResolver)
9393
})
9494
}
9595

@@ -225,9 +225,9 @@ func (s *MilestoneSnapshotStore) WithTx(tx kv.Tx) EntityStore[*Milestone] {
225225

226226
func (s *MilestoneSnapshotStore) RangeExtractor() snaptype.RangeExtractor {
227227
return snaptype.RangeExtractorFunc(
228-
func(ctx context.Context, blockFrom, blockTo uint64, firstKey snaptype.FirstKeyGetter, db kv.RoDB, chainConfig *chain.Config, collect func([]byte) error, workers int, lvl log.Lvl, logger log.Logger) (uint64, error) {
228+
func(ctx context.Context, blockFrom, blockTo uint64, firstKey snaptype.FirstKeyGetter, db kv.RoDB, chainConfig *chain.Config, collect func([]byte) error, workers int, lvl log.Lvl, logger log.Logger, hashResolver snaptype.BlockHashResolver) (uint64, error) {
229229
return s.SnapType().RangeExtractor().Extract(ctx, blockFrom, blockTo, firstKey,
230-
s.EntityStore.(*mdbxEntityStore[*Milestone]).db.RoDB(), chainConfig, collect, workers, lvl, logger)
230+
s.EntityStore.(*mdbxEntityStore[*Milestone]).db.RoDB(), chainConfig, collect, workers, lvl, logger, hashResolver)
231231
})
232232
}
233233

@@ -344,9 +344,9 @@ func NewCheckpointSnapshotStore(base EntityStore[*Checkpoint], snapshots *RoSnap
344344

345345
func (s *CheckpointSnapshotStore) RangeExtractor() snaptype.RangeExtractor {
346346
return snaptype.RangeExtractorFunc(
347-
func(ctx context.Context, blockFrom, blockTo uint64, firstKey snaptype.FirstKeyGetter, db kv.RoDB, chainConfig *chain.Config, collect func([]byte) error, workers int, lvl log.Lvl, logger log.Logger) (uint64, error) {
347+
func(ctx context.Context, blockFrom, blockTo uint64, firstKey snaptype.FirstKeyGetter, db kv.RoDB, chainConfig *chain.Config, collect func([]byte) error, workers int, lvl log.Lvl, logger log.Logger, hashResolver snaptype.BlockHashResolver) (uint64, error) {
348348
return s.SnapType().RangeExtractor().Extract(ctx, blockFrom, blockTo, firstKey,
349-
s.EntityStore.(*mdbxEntityStore[*Checkpoint]).db.RoDB(), chainConfig, collect, workers, lvl, logger)
349+
s.EntityStore.(*mdbxEntityStore[*Checkpoint]).db.RoDB(), chainConfig, collect, workers, lvl, logger, hashResolver)
350350
})
351351
}
352352

‎polygon/heimdall/types.go‎

Lines changed: 14 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -40,7 +40,6 @@ import (
4040
"github.com/erigontech/erigon-lib/log/v3"
4141
"github.com/erigontech/erigon-lib/recsplit"
4242
"github.com/erigontech/erigon-lib/seg"
43-
"github.com/erigontech/erigon/core/rawdb"
4443
coresnaptype "github.com/erigontech/erigon/core/snaptype"
4544
bortypes "github.com/erigontech/erigon/polygon/bor/types"
4645
)
@@ -89,7 +88,7 @@ type EventRangeExtractor struct {
8988
EventsDb func() kv.RoDB
9089
}
9190

92-
func (e EventRangeExtractor) Extract(ctx context.Context, blockFrom, blockTo uint64, firstEventId snaptype.FirstKeyGetter, chainDb kv.RoDB, chainConfig *chain.Config, collect func([]byte) error, workers int, lvl log.Lvl, logger log.Logger) (uint64, error) {
91+
func (e EventRangeExtractor) Extract(ctx context.Context, blockFrom, blockTo uint64, firstEventId snaptype.FirstKeyGetter, chainDb kv.RoDB, chainConfig *chain.Config, collect func([]byte) error, workers int, lvl log.Lvl, logger log.Logger, hashResolver snaptype.BlockHashResolver) (uint64, error) {
9392
logEvery := time.NewTicker(20 * time.Second)
9493
defer logEvery.Stop()
9594

@@ -114,9 +113,14 @@ func (e EventRangeExtractor) Extract(ctx context.Context, blockFrom, blockTo uin
114113
if err := kv.BigChunks(eventsDb, kv.BorEventNums, from, func(tx kv.Tx, blockNumBytes, eventIdBytes []byte) (bool, error) {
115114
endEventId := binary.BigEndian.Uint64(eventIdBytes) + 1
116115
blockNum := binary.BigEndian.Uint64(blockNumBytes)
117-
blockHash, e := rawdb.ReadCanonicalHash(chainTx, blockNum)
118-
if e != nil {
119-
return false, e
116+
117+
blockHash, ok, err := hashResolver.CanonicalHash(ctx, chainTx, blockNum)
118+
if err != nil {
119+
return false, err
120+
}
121+
122+
if !ok {
123+
return false, fmt.Errorf("failed to read canonical hash for the block number %d", blockNum)
120124
}
121125

122126
if blockNum >= blockTo {
@@ -256,7 +260,7 @@ var (
256260
MinSupported: 1,
257261
},
258262
snaptype.RangeExtractorFunc(
259-
func(ctx context.Context, blockFrom, blockTo uint64, firstKeyGetter snaptype.FirstKeyGetter, db kv.RoDB, _ *chain.Config, collect func([]byte) error, workers int, lvl log.Lvl, logger log.Logger) (uint64, error) {
263+
func(ctx context.Context, blockFrom, blockTo uint64, firstKeyGetter snaptype.FirstKeyGetter, db kv.RoDB, _ *chain.Config, collect func([]byte) error, workers int, lvl log.Lvl, logger log.Logger, hashResolver snaptype.BlockHashResolver) (uint64, error) {
260264
spanFrom := uint64(SpanIdAt(blockFrom))
261265
spanTo := uint64(SpanIdAt(blockTo))
262266
return extractValueRange(ctx, kv.BorSpans, spanFrom, spanTo, db, collect, workers, lvl, logger)
@@ -285,7 +289,7 @@ var (
285289
MinSupported: 1,
286290
},
287291
snaptype.RangeExtractorFunc(
288-
func(ctx context.Context, blockFrom, blockTo uint64, firstKeyGetter snaptype.FirstKeyGetter, db kv.RoDB, _ *chain.Config, collect func([]byte) error, workers int, lvl log.Lvl, logger log.Logger) (uint64, error) {
292+
func(ctx context.Context, blockFrom, blockTo uint64, firstKeyGetter snaptype.FirstKeyGetter, db kv.RoDB, _ *chain.Config, collect func([]byte) error, workers int, lvl log.Lvl, logger log.Logger, hashResolver snaptype.BlockHashResolver) (uint64, error) {
289293
var checkpointTo, checkpointFrom CheckpointId
290294

291295
err := db.View(ctx, func(tx kv.Tx) (err error) {
@@ -351,7 +355,7 @@ var (
351355
MinSupported: 1,
352356
},
353357
snaptype.RangeExtractorFunc(
354-
func(ctx context.Context, blockFrom, blockTo uint64, firstKeyGetter snaptype.FirstKeyGetter, db kv.RoDB, _ *chain.Config, collect func([]byte) error, workers int, lvl log.Lvl, logger log.Logger) (uint64, error) {
358+
func(ctx context.Context, blockFrom, blockTo uint64, firstKeyGetter snaptype.FirstKeyGetter, db kv.RoDB, _ *chain.Config, collect func([]byte) error, workers int, lvl log.Lvl, logger log.Logger, hashResolver snaptype.BlockHashResolver) (uint64, error) {
355359
var milestoneFrom, milestoneTo MilestoneId
356360

357361
err := db.View(ctx, func(tx kv.Tx) (err error) {
@@ -419,13 +423,13 @@ func RecordWayPoints(value bool) {
419423

420424
func SnapshotTypes() []snaptype.Type {
421425
if recordWaypoints {
422-
return []snaptype.Type{Events, Spans, Checkpoints, Milestones}
426+
return []snaptype.Type{Events, Spans, Checkpoints}
423427
}
424428

425429
return []snaptype.Type{Events, Spans}
426430
}
427431

428-
func CheckpointsEnabled() bool {
432+
func WaypointsEnabled() bool {
429433
for _, snapType := range SnapshotTypes() {
430434
if snapType.Enum() == Checkpoints.Enum() {
431435
return true
@@ -435,16 +439,6 @@ func CheckpointsEnabled() bool {
435439
return false
436440
}
437441

438-
func MilestonesEnabled() bool {
439-
for _, snapType := range SnapshotTypes() {
440-
if snapType.Enum() == Milestones.Enum() {
441-
return true
442-
}
443-
}
444-
445-
return false
446-
}
447-
448442
// This function extracts values in interval [valueFrom, valueTo)
449443
func extractValueRange(ctx context.Context, table string, valueFrom, valueTo uint64, db kv.RoDB, collect func([]byte) error, workers int, lvl log.Lvl, logger log.Logger) (uint64, error) {
450444
logEvery := time.NewTicker(20 * time.Second)

‎turbo/snapshotsync/freezeblocks/bor_snapshots.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -83,7 +83,7 @@ func (br *BlockRetire) retireBorBlocks(ctx context.Context, minBlockNum uint64,
8383

8484
for i := blockFrom; i < blockTo; i = chooseSegmentEnd(i, blockTo, snap.Enum(), chainConfig) {
8585
end := chooseSegmentEnd(i, blockTo, snap.Enum(), chainConfig)
86-
if _, err := snap.ExtractRange(ctx, snap.FileInfo(snapshots.Dir(), i, end), rangeExtractor, indexBuilder, firstKeyGetter, db, chainConfig, tmpDir, workers, lvl, logger); err != nil {
86+
if _, err := snap.ExtractRange(ctx, snap.FileInfo(snapshots.Dir(), i, end), rangeExtractor, indexBuilder, firstKeyGetter, db, chainConfig, tmpDir, workers, lvl, logger, br.blockReader); err != nil {
8787
return ok, fmt.Errorf("ExtractRange: %d-%d: %w", i, end, err)
8888
}
8989
}

0 commit comments

Comments
 (0)