fixup! feat(migrations): add backfill for botched epoch_start, balance_start, balance_min

Author: invis-bitfly

backend/pkg/commons/types/config.go

@@ -209,7 +209,7 @@ type Config struct {
 		TransferAtOnce         int64 `yaml:"transferAtOnce" env:"TRANSFER_AT_ONCE, default=2"`                 // how much data to transfer in a single transfer
 		RollingsInParallel     int64 `yaml:"rollingsAtOnce" env:"ROLLINGS_AT_ONCE, default=3"`                 // how many rollings to do at once
 		RollingPartsInParallel int64 `yaml:"rollingsInParallel" env:"ROLLINGS_IN_PARALLEL, default=3"`         // how man parts of a single rolling to do at once
-		BackfillAtOnce         int64 `yaml:"backfillAtOnce" env:"BACKFILL_AT_ONCE, default=64"`                // how much data to backfill in a single backfill
+		BackfillAtOnce         int64 `yaml:"backfillAtOnce" env:"BACKFILL_AT_ONCE, default=4"`                 // how much data to backfill in a single backfill
 		BackfillInParallel     int64 `yaml:"backfillInParallel" env:"BACKFILL_IN_PARALLEL, default=2"`         // how many backfills to do at once
 	} `yaml:"dashboardExporter" env:", prefix=DASHBOARD_EXPORTER_"`
 	Pprof struct {

backend/pkg/exporter/db/db.go

@@ -1301,10 +1301,10 @@ func doGenericBackfill(epochs []BackfillMetadata, backfillType BackfillType, col
 	abortCtx, cancel := context.WithTimeout(context.Background(), 3*time.Minute)
 	defer cancel()
 	ctx := ch.Context(abortCtx, ch.WithSettings(ch.Settings{
-		"insert_deduplication_token":    epochs[0].BackfillBatchId.String(),
-		"insert_deduplicate":            true,
-		"select_sequential_consistency": 1,
-		"use_skip_indexes_if_final":     1, // this is only safe because our index is over a column from the primary key
+		"insert_deduplication_token": epochs[0].BackfillBatchId.String(),
+		"insert_deduplicate":         true,
+		"use_skip_indexes_if_final":  1, // this is only safe because our index is over a column from the primary key
+		"max_threads":                2,
 	}))
 
 	var epoch_timestamp []time.Time
@@ -1362,7 +1362,10 @@ func BackfillMissingMissedRewards(epochs []BackfillMetadata) error {
 }
 
 func BackfillBotchedEpochStart(epochs []BackfillMetadata) error {
-	return doGenericBackfill(epochs, BackfillTypeBotchedEpochStart, nil, nil, BackfillBotchedEpochStartSink)
+	return doGenericBackfill(epochs, BackfillTypeBotchedEpochStart,
+		[]exp.AliasedExpression{
+			goqu.I("balance_end").As("balance_end"),
+		}, nil, BackfillBotchedEpochStartSink)
 }
 
 func GetIncompleteInsertEpochs() ([]EpochMetadata, error) { // no limit because it should never grow too large

backend/pkg/exporter/modules/dashboard_data_backfills.go

@@ -23,6 +23,7 @@ func (d *dashboardData) roiBackfillTask() {
 		edb.BackfillTypeEBLookup,
 		edb.BackfillTypeElectraForkEpochEvents,
 		edb.BackfillTypeMissingMissedRewards,
+		edb.BackfillTypeBotchedEpochStart,
 	}
 	for _, backfillType := range jobs {
 		// fork for every backfill we have to do