Merge branch 'ryan/simplify-start-index' into 'main'

simplify start index calculation

See merge request flarenetwork/FSP/flare-system-c-chain-indexer!71

Author: ryanc-flare

README.md

@@ -48,8 +48,8 @@ database = "flare_ftso_indexer"
 username = "root"
 password = "root"
 log_queries = false
-drop_table_at_start = true
-history_drop = 604800 # Enable deleting the transactions and logs in DB that are older (timestamp of the block) than history_drop (in seconds); set 0 to turn off; defaults to 7 days for Flare/Songbird and 2 days for Coston*
+drop_table_at_start = false  # set to true to drop existing tables at the start of the indexing - useful to force re-indexing
+history_drop = 604800  # Enable deleting the transactions and logs in DB that are older (timestamp of the block) than history_drop (in seconds); set 0 to turn off; defaults to 7 days for Flare/Songbird and 2 days for Coston*
 
 [logger]
 level = "INFO"
@@ -66,6 +66,21 @@ backoff_max_elapsed_time_seconds = 300 # optional, defaults to 300s = 5 minutes.
 timeout_milis = 1000  # optional, defaults to 1000ms = 1s. Try increasing if you see timeout errors often.
 ```
 
+If the C chain indexer has been previously run and there is existing data in the database,
+subsequent runs will resume indexing from after the last indexed block. Only when starting with an
+empty database will the indexer have to decide where to start. Normally this will be based on the
+history drop configuration parameter - if the history drop is 14 days for example then the indexer
+will start indexing from the block that was mined 14 days ago. If the history drop is disabled (set
+to 0) then the indexer will start from the configured `start_index` block or from block 0 if not
+set.
+
+In case the indexer has been previously run and you need more historical data, you can increase
+the history drop parameter or disable history drop and set the start_index to the desired
+starting block. You can manually delete all existing data from the database in order to trigger
+re-indexing from the new starting block. You can also set the `drop_table_at_start` parameter to
+true to have the indexer drop existing tables at startup and force re-indexing - though remember to
+set it back to false afterwards to avoid losing data on subsequent runs.
+
 ### Database
 
 In `database/docker` we provide a simple database. Navigate to the folder and run

database/history_drop.go

@@ -87,18 +87,19 @@ func dropHistoryIteration(
 
 // GetStartBlock returns the block number to start indexing from based on the history drop parameter.
 func GetStartBlock(
-	ctx context.Context, db *gorm.DB, historyDropIntervalSeconds uint64, client *chain.Client, configuredStartBlock uint64,
+	ctx context.Context, historyDropIntervalSeconds uint64, client *chain.Client, configuredStartBlock uint64,
 ) (uint64, error) {
 	lastBlockTime, lastBlockNumber, err := getBlockTimestamp(ctx, nil, client)
 	if err != nil {
 		return 0, errors.Wrap(err, "Failed to get the latest block")
 	}
 
-	db = db.WithContext(ctx)
 	deleteStartTime := lastBlockTime - historyDropIntervalSeconds
 
-	return getNearestBlockByTimestamp(
-		ctx, deleteStartTime, db, client, configuredStartBlock, lastBlockNumber,
+	// This function is only ever called when starting with an empty DB state
+	// so we can skip the DB check and jump straight to the chain search.
+	return getNearestBlockByTimestampFromChain(
+		ctx, deleteStartTime, client, configuredStartBlock, lastBlockNumber,
 	)
 }
 

main.go

@@ -87,29 +87,60 @@ func run(ctx context.Context) error {
 		}
 	}
 
-	if historyDrop > 0 {
-		firstBlockNumber, err := boff.Retry(
-			ctx,
-			func() (uint64, error) {
-				return database.GetStartBlock(
-					ctx, db, historyDrop, ethClient, cfg.Indexer.StartIndex,
-				)
-			},
-			"GetStartBlock",
-		)
-		if err != nil {
-			return errors.Wrap(err, "GetStartBlock error")
-		}
+	startIndex, err := getStartIndex(ctx, db, ethClient, cfg, historyDrop)
+	if err != nil {
+		return errors.Wrap(err, "getStartIndex error")
+	}
 
-		logger.Info("firstBlockNumber = %d", firstBlockNumber)
+	cfg.Indexer.StartIndex = startIndex
 
-		if firstBlockNumber > cfg.Indexer.StartIndex {
-			logger.Info("Setting new startIndex due to history drop: %d", firstBlockNumber)
-			cfg.Indexer.StartIndex = firstBlockNumber
-		}
+	return runIndexer(ctx, cfg, db, ethClient, historyDrop)
+}
+
+func getStartIndex(
+	ctx context.Context,
+	db *gorm.DB,
+	ethClient *chain.Client,
+	cfg *config.Config,
+	historyDrop uint64,
+) (uint64, error) {
+	var latestIndexedBlock database.Block
+	err := db.Last(&database.Block{}).Select("number").Scan(&latestIndexedBlock).Error
+
+	// If a latest indexed block is found, return the next block number
+	if err == nil {
+		logger.Info("Starting after latest indexed block from DB: %d", latestIndexedBlock.Number)
+		return latestIndexedBlock.Number + 1, nil
 	}
 
-	return runIndexer(ctx, cfg, db, ethClient, historyDrop)
+	// In case of an unexpected error, return it
+	if !errors.Is(err, gorm.ErrRecordNotFound) {
+		return 0, errors.Wrap(err, "DB query error")
+	}
+
+	// No blocks are indexed yet
+	// If history drop is disabled, return the configured start index
+	if historyDrop == 0 {
+		logger.Info("No indexed blocks found in DB, starting from configured start index: %d", cfg.Indexer.StartIndex)
+		return cfg.Indexer.StartIndex, nil
+	}
+
+	// History drop is enabled so calculate start index based on it.
+	firstBlockNumber, err := boff.Retry(
+		ctx,
+		func() (uint64, error) {
+			return database.GetStartBlock(
+				ctx, historyDrop, ethClient, cfg.Indexer.StartIndex,
+			)
+		},
+		"GetStartBlock",
+	)
+	if err != nil {
+		return 0, errors.Wrap(err, "GetStartBlock error")
+	}
+
+	logger.Info("No indexed blocks found in DB, starting from calculated start index based on history drop: %d", firstBlockNumber)
+	return firstBlockNumber, nil
 }
 
 func runIndexer(