misc(exporter): add retries when migrating legacy balances

Author: peterbitfly

backend/cmd/misc/commands/migrate_legacy_balances.go

@@ -3,6 +3,7 @@ package commands
 import (
 	"flag"
 	"fmt"
+	"time"
 
 	"github.com/gobitfly/beaconchain/cmd/misc/misctypes"
 	"github.com/gobitfly/beaconchain/pkg/commons/db"
@@ -42,25 +43,39 @@ func (s *MigrateLegacyBalancesCommand) Run(clClient *rpc.LighthouseClient) error
 	log.Infof("command: migrate-legacy-balances start-epoch=%d end-epoch=%d", s.Config.StartEpoch, s.Config.EndEpoch)
 
 	for epoch := s.Config.StartEpoch; epoch <= s.Config.EndEpoch; epoch++ {
-		state, err := clClient.GetValidatorState(epoch)
-		if err != nil {
-			log.Error(err, fmt.Sprintf("error getting balances for epoch %d: %v", epoch, err), 0)
-			continue
-		}
+		for attempt := 1; attempt <= 5; attempt++ {
+			state, err := clClient.GetValidatorState(epoch)
+			if err != nil {
+				log.Error(err, fmt.Sprintf("error getting balances for epoch %d (attempt %d/5): %v", epoch, attempt, err), 0)
+				if attempt == 5 {
+					return fmt.Errorf("failed to get balances for epoch %d after %d attempts: %w", epoch, attempt, err)
+				}
+				time.Sleep(5 * time.Second)
+				continue
+			}
 
-		validators := make([]*types.Validator, 0, len(state.Data))
-		for _, validator := range state.Data {
-			validators = append(validators, &types.Validator{
-				Index:            validator.Index,
-				Balance:          validator.Balance,
-				EffectiveBalance: validator.Validator.EffectiveBalance,
-			})
-		}
+			validators := make([]*types.Validator, 0, len(state.Data))
+			for _, validator := range state.Data {
+				validators = append(validators, &types.Validator{
+					Index:            validator.Index,
+					Balance:          validator.Balance,
+					EffectiveBalance: validator.Validator.EffectiveBalance,
+				})
+			}
+
+			log.Infof("writing epoch %d to clickhouse with %d validators", epoch, len(state.Data))
+			err = db.SaveLegacyValidatorBalancesToClickhouse(epoch, validators)
+			if err != nil {
+				log.Error(err, fmt.Sprintf("error saving legacy balances for epoch %d (attempt %d/5): %v", epoch, attempt, err), 0)
+				if attempt == 5 {
+					return fmt.Errorf("error saving legacy balances for epoch %d after %d attempts: %w", epoch, attempt, err)
+				}
+				time.Sleep(5 * time.Second)
+				continue
+			}
 
-		log.Infof("writing epoch %d to clickhouse with %d validators", epoch, len(state.Data))
-		err = db.SaveLegacyValidatorBalancesToClickhouse(epoch, validators)
-		if err != nil {
-			return fmt.Errorf("error saving legacy balances for epoch %d: %w", epoch, err)
+			// success for this epoch
+			break
 		}
 	}
 	return nil