fix: make core components use config json file with constant default values

Author: kuco23

.env.template

@@ -10,7 +10,6 @@ CHAIN=songbird
 # rpc config for the chain ${CHAIN}
 RPC_URL=https://songbird-api.flare.network/ext/C/rpc
 RPC_API_KEY=
-LOG_QUERY_BATCH_SIZE=29
 
 # exposed port for the running api
 API_PORT=3000

README.md

@@ -33,7 +33,6 @@ If new events want to be added to tracking without stopping the indexer, you sho
 Additional optional dev settings:
 - **CONFIG_PATH**: should be set to a json file containing a list of indexed event names, see `./configs/everything.json`;
 - **ADDRESSES_JSON**: path to file with the FAsset deployment contracts (when indexing non-official test deployments);
-- **LOG_QUERY_BATCH_SIZE**: the size of the log batch requested by indexer to the rpc node (should be strictly less than 30);
 - **MIN_BLOCK_NUMBER**: the block number that the indexer will start with. If empty, it is set to the block at which the FAsset contracts were deployed. Other values can lead to errors due to event gaps.
 
 To run the indexer, run `yarn run-indexer`.

compose.yaml

@@ -26,7 +26,6 @@ services:
       RPC_URL: ${RPC_URL}
       RPC_API_KEY: ${RPC_API_KEY:-}
       CONFIG_PATH: /usr/src/app/config.json
-      LOG_QUERY_BATCH_SIZE: ${LOG_QUERY_BATCH_SIZE:-}
       REINDEX_TYPE: ${REINDEX_TYPE:-}
       REINDEX_DIFF: ${REINDEX_DIFF:-}
       REINDEX_NAME: ${REINDEX_NAME:-}

packages/fasset-indexer-core/.env.template

@@ -5,7 +5,6 @@ RPC_URL=
 RPC_API_KEY=
 ADDRESSES_JSON=
 MIN_BLOCK_NUMBER=
-LOG_QUERY_BATCH_SIZE=
 
 # database
 DB_TYPE=sqlite|postgres

packages/fasset-indexer-core/src/config/config.ts

@@ -4,6 +4,13 @@ import { SqliteDriver } from "@mikro-orm/sqlite"
 import { PostgreSqlDriver } from "@mikro-orm/postgresql"
 import { getContractInfo } from "./contracts"
 import { SchemaUpdate } from "../orm/interface"
+import {
+  EVENT_NAMES,
+  DEFAULT_EVM_BLOCK_HEIGHT_OFFSET,
+  DEFAULT_EVM_LOG_FETCH_BLOCK_BATCH_SIZE,
+  DEFAULT_EVM_LOG_FETCH_SLEEP_MS,
+  DEFAULT_EVM_STATE_UPDATE_SLEEP_MS
+} from "./constants"
 import type { ContractInfo, ConfigJson, ReindexConfig } from "./interface"
 
 
@@ -84,20 +91,32 @@ export class ConfigLoader {
     return this.isNull(minBlock) ? undefined : parseInt(minBlock!)
   }
 
-  get logQueryBatchSize(): number {
-    const size = process.env.LOG_QUERY_BATCH_SIZE
-    return this.isNull(size) ? 28 : parseInt(size!)
+  get evmLogFetchBlockBatchSize(): number {
+    const def = DEFAULT_EVM_LOG_FETCH_BLOCK_BATCH_SIZE
+    const val = this.json?.events.logFetchBlockBatchSize
+    return val == null ? def : val
   }
 
-  get json(): ConfigJson | undefined {
-    if (this._configJson == null) {
-      const configPath = process.env.CONFIG_PATH
-      if (configPath != null) {
-        const file = readFileSync(configPath)
-        this._configJson = JSON.parse(file.toString())
-      }
-    }
-    return this._configJson
+  get evmLogFetchCycleSleepMs(): number {
+    const def = DEFAULT_EVM_LOG_FETCH_SLEEP_MS
+    const val = this.json?.events.logFetchCycleSleepMs
+    return val == null ? def : val
+  }
+
+  get evmLogFetchBlockHeightOffset(): number {
+    const def = DEFAULT_EVM_BLOCK_HEIGHT_OFFSET
+    const val = this.json?.events.logFetchBlockHeightOffset
+    return val == null ? def : val
+  }
+
+  get stateFetchCycleSleepMs(): number {
+    const def = DEFAULT_EVM_STATE_UPDATE_SLEEP_MS
+    const val = this.json?.watchdog.cycleSleepMs
+    return val == null ? def : val
+  }
+
+  get indexEvents(): string[] {
+    return this.json?.indexEvents ?? EVENT_NAMES
   }
 
   get reindexing(): ReindexConfig | null {
@@ -111,6 +130,17 @@ export class ConfigLoader {
     return { type, diff: diff.split(','), name }
   }
 
+  protected get json(): ConfigJson | undefined {
+    if (this._configJson == null) {
+      const configPath = process.env.CONFIG_PATH
+      if (configPath != null) {
+        const file = readFileSync(configPath)
+        this._configJson = JSON.parse(file.toString())
+      }
+    }
+    return this._configJson
+  }
+
   protected get dbType(): string {
     return this.required('DB_TYPE')
   }

packages/fasset-indexer-core/src/config/constants.ts

@@ -2,9 +2,10 @@
 export const SLEEP_AFTER_ERROR_MS = 3000
 
 // evm event scrape config
-export const EVM_LOG_FETCH_SLEEP_MS = 10 * 1000 // collect logs every 10 seconds
-export const EVM_STATE_UPDATE_SLEEP_MS = 60 * 1000 // collect state every one minute
-export const EVM_BLOCK_HEIGHT_OFFSET = 10 // log collection offset from the current block height
+export const DEFAULT_EVM_LOG_FETCH_SLEEP_MS = 10 * 1000 // collect logs every 10 seconds
+export const DEFAULT_EVM_STATE_UPDATE_SLEEP_MS = 60 * 1000 // collect state every one minute
+export const DEFAULT_EVM_BLOCK_HEIGHT_OFFSET = 10 // log collection offset from the current block height
+export const DEFAULT_EVM_LOG_FETCH_BLOCK_BATCH_SIZE = 29
 
 // db settings
 export const MIN_DATABASE_POOL_CONNECTIONS = 2

packages/fasset-indexer-core/src/indexer/indexer.ts

@@ -5,7 +5,6 @@ import { EventScraper } from './eventlib/event-scraper'
 import { logger } from '../logger'
 import {
   FIRST_UNHANDLED_EVENT_BLOCK_DB_KEY,
-  EVM_BLOCK_HEIGHT_OFFSET,
   MIN_EVM_BLOCK_NUMBER_DB_KEY
 } from '../config/constants'
 import type { Log } from 'ethers'
@@ -38,8 +37,9 @@ export class EventIndexer {
     if (endBlock === undefined || endBlock > lastBlockToHandle) {
       endBlock = lastBlockToHandle
     }
-    for (let i = startBlock; i <= endBlock; i += this.context.config.logQueryBatchSize + 1) {
-      const endLoopBlock = Math.min(endBlock, i + this.context.config.logQueryBatchSize)
+    const batchSize = this.context.config.evmLogFetchBlockBatchSize
+    for (let i = startBlock; i <= endBlock; i += batchSize + 1) {
+      const endLoopBlock = Math.min(endBlock, i + batchSize)
       const logs = await this.eventScraper.getLogs(i, endLoopBlock)
       await this.storeLogs(logs)
       await this.setFirstUnhandledBlock(endLoopBlock + 1)
@@ -49,8 +49,7 @@ export class EventIndexer {
 
   async lastBlockToHandle(): Promise<number> {
     const blockHeight = await this.context.provider.getBlockNumber()
-    const offset = this.context.config.json?.events.logFetchBlockHeightOffset ?? EVM_BLOCK_HEIGHT_OFFSET
-    return blockHeight - offset
+    return blockHeight - this.context.config.evmLogFetchBlockHeightOffset
   }
 
   async firstUnhandledBlock(startBlock?: number): Promise<number> {

packages/fasset-indexer-core/src/indexer/watchdog.ts

@@ -4,7 +4,6 @@ import { AgentVault } from "../orm/entities/agent"
 import { FtsoPrice } from "../orm/entities/state/price"
 import { isUntrackedAgentVault, updateAgentVaultInfo } from "./utils"
 import { Context } from "../context/context"
-import { EVM_STATE_UPDATE_SLEEP_MS } from "../config/constants"
 import { logger } from "../logger"
 import type { EntityManager } from "@mikro-orm/knex"
 import type { FAssetType } from "../shared"
@@ -24,7 +23,7 @@ export class EvmStateWatchdog {
       } catch (e: any) {
         logger.error(`error in top-level Flare watchdog: ${e}`)
       }
-      await sleep(this.context.config.json?.watchdog.cycleSleepMs ?? EVM_STATE_UPDATE_SLEEP_MS)
+      await sleep(this.context.config.stateFetchCycleSleepMs)
     }
   }
 

packages/fasset-indexer-core/src/run/run-indexer.ts

@@ -7,7 +7,6 @@ import { IndexerRunner } from "../indexer/runner"
 import { EventIndexerParallelBackPopulation } from "../indexer/reindexing/indexer-parallel-back-population"
 import { EventIndexerParallelRacePopulation } from "../indexer/reindexing/indexer-parallel-race-population"
 import { migrateCollateralPoolEvents } from "../scripts/migrate-collateral-pool-events"
-import { EVENT_NAMES, EVM_LOG_FETCH_SLEEP_MS } from "../config/constants"
 import { logger } from "../logger"
 
 
@@ -17,7 +16,7 @@ async function runIndexer() {
 
   // define indexer by configured type
   let indexer: EventIndexer | EventIndexerParallelBackPopulation | EventIndexerParallelRacePopulation
-  const allEvents = config.json?.indexEvents ?? EVENT_NAMES
+  const allEvents = config.indexEvents
   if (config.reindexing == null) {
     indexer = new EventIndexer(context, allEvents)
   } else if (config.reindexing.type == 'back') {
@@ -44,10 +43,9 @@ async function runIndexer() {
 
   logger.info(`starting ${context.chain} event indexer...`)
   const runner = new IndexerRunner(indexer, 'native')
-  const sleepms = config.json?.events.logFetchCycleSleepMs ?? EVM_LOG_FETCH_SLEEP_MS
   await Promise.all([
     migrateCollateralPoolEvents(context.orm.em.fork()),
-    runner.run(sleepms, undefined, config.reindexing != null)
+    runner.run(config.evmLogFetchCycleSleepMs, undefined, config.reindexing != null)
   ])
 }