fix: remove db access per validation for fee data (#328)

## High Level Overview of Change

<!--
Please include a summary/list of the changes.
If too broad, please consider splitting into multiple PRs.
If a relevant Asana task, please link it here.
-->
The validator table in database is being queried every time there's a
new validation to retrieve fee data, which caused this error:
`KnexTimeoutError: Knex: Timeout acquiring a connection. The pool is
probably full. Are you missing a .transacting(trx) call?`
We can instead cache this validator-network data and update it every
hour to reduce number of queries.


### Type of Change

<!--
Please check relevant options, delete irrelevant ones.
-->

- [x] Bug fix (non-breaking change which fixes an issue)
- [ ] New feature (non-breaking change which adds functionality)
- [ ] Breaking change (fix or feature that would cause existing
functionality to not work as expected)
- [ ] Refactor (non-breaking change that only restructures code)
- [ ] Tests (You added tests for code that already exists, or your new
feature included in this PR)
- [ ] Documentation Updates
- [ ] Release

---------

Co-authored-by: Raj Patel <[email protected]>

Author: pdp2121

src/connection-manager/agreement.ts

@@ -17,7 +17,9 @@ import {
   ValidationRaw,
   ValidatorKeys,
   Ballot,
+  Chain,
 } from '../shared/types'
+import { getLists, overlaps } from '../shared/utils'
 import logger from '../shared/utils/logger'
 
 import chains from './chains'
@@ -120,6 +122,30 @@ function isPreceedingFlagLedger(ledger_index: string): boolean {
   return parseInt(ledger_index, 10) % 256 === 255
 }
 
+/**
+ * Finds network name from chain id.
+ *
+ * @param chain - A chain object.
+ * @returns String.
+ */
+async function getNetworkNameFromChainId(chain: Chain): Promise<string> {
+  let id = chain.id
+  const lists = await getLists().catch((err) => {
+    log.error('Error getting validator lists', err)
+    return undefined
+  })
+
+  if (lists != null) {
+    Object.entries(lists).forEach(([network, set]) => {
+      if (overlaps(chain.validators, set)) {
+        id = network
+      }
+    })
+  }
+
+  return id
+}
+
 /**
  *
  */
@@ -153,6 +179,14 @@ class Agreement {
     for (const chain of agreementChains) {
       const ledger_hashes = chain.ledgers
 
+      const networkName = await getNetworkNameFromChainId(chain)
+
+      log.info(
+        `Agreement: ${chain.id}:${networkName}:${Array.from(
+          chain.validators,
+        ).join(',')}`,
+      )
+
       for (const signing_key of chain.validators) {
         promises.push(
           this.calculateValidatorAgreement(

src/connection-manager/connections.ts

@@ -27,6 +27,7 @@ const ports = [443, 80, 6005, 6006, 51233, 51234]
 const protocols = ['wss://', 'ws://']
 const connections: Map<string, WebSocket> = new Map()
 const networkFee: Map<string, FeeVote> = new Map()
+const validationNetworkDb: Map<string, string> = new Map()
 const CM_INTERVAL = 60 * 60 * 1000
 const WS_TIMEOUT = 10000
 const REPORTING_INTERVAL = 15 * 60 * 1000
@@ -60,12 +61,6 @@ async function setHandlers(
   isInitialNode = false,
   retryCount = 0,
 ): Promise<void> {
-  log.info(
-    `Initiated Websocket connection for: ${ws_url} on ${
-      networks ?? 'unknown network'
-    }`,
-  )
-
   const ledger_hashes: string[] = []
   return new Promise(function setHandlersPromise(resolve, _reject) {
     ws.on('open', () => {
@@ -119,6 +114,7 @@ async function setHandlers(
           networks,
           networkFee,
           ws,
+          validationNetworkDb,
         )
       }
     })
@@ -230,13 +226,23 @@ async function findConnection(node: WsNode): Promise<void> {
   return Promise.resolve()
 }
 
+async function getValidationNetworkDb(): Promise<void> {
+  const validatorNetwork: Array<{ signing_key: string; networks: string }> =
+    await query('validators').select('signing_key', 'networks')
+  for (const entry of validatorNetwork) {
+    validationNetworkDb.set(entry.signing_key, entry.networks)
+  }
+}
+
 /**
  * Creates connections to nodes found in the database.
  *
  * @returns A promise that resolves to void once all possible connections have been created.
  */
 async function createConnections(): Promise<void> {
   log.info('Finding Connections...')
+  validationNetworkDb.clear()
+  await getValidationNetworkDb()
   const tenMinutesAgo = new Date()
   tenMinutesAgo.setMinutes(tenMinutesAgo.getMinutes() - 10)
 

src/connection-manager/wsHandling.ts

@@ -9,18 +9,13 @@ import {
 import { AMENDMENTS_ID } from 'xrpl/dist/npm/models/ledger'
 import { LedgerResponseExpanded } from 'xrpl/dist/npm/models/methods/ledger'
 
-import {
-  query,
-  saveAmendmentStatus,
-  saveAmendmentsStatus,
-} from '../shared/database'
+import { saveAmendmentStatus, saveAmendmentsStatus } from '../shared/database'
 import {
   NETWORKS_HOSTS,
   deleteAmendmentStatus,
 } from '../shared/database/amendments'
 import {
   AmendmentStatus,
-  DatabaseValidator,
   FeeVote,
   StreamLedger,
   StreamManifest,
@@ -106,6 +101,7 @@ function isFlagLedgerPlusOne(ledger_index: number): boolean {
  * @param networks - The networks of subscribed node.
  * @param network_fee - The map of default fee for the network to be used in case the validator does not vote for a new fee.
  * @param ws - The WebSocket message received from.
+ * @param validationNetworkDb -- A map of validator signing_keys to their corresponding networks.
  * @returns Void.
  */
 // eslint-disable-next-line max-params -- Disabled for this function.
@@ -115,22 +111,17 @@ export async function handleWsMessageSubscribeTypes(
   networks: string | undefined,
   network_fee: Map<string, FeeVote>,
   ws: WebSocket,
+  validationNetworkDb: Map<string, string>,
 ): Promise<void> {
   if (data.type === 'validationReceived') {
     const validationData = data as ValidationRaw
     if (ledger_hashes.includes(validationData.ledger_hash)) {
       validationData.networks = networks
     }
 
-    // Get network of the validation if ledger_hash is not in cache.
-    const validationNetworkDb: DatabaseValidator | undefined = await query(
-      'validators',
-    )
-      .select('*')
-      .where('signing_key', validationData.validation_public_key)
-      .first()
     const validationNetwork =
-      validationNetworkDb?.networks ?? validationData.networks
+      validationNetworkDb.get(validationData.validation_public_key) ??
+      validationData.networks
 
     // Get the fee for the network to be used in case the validator does not vote for a new fee.
     if (validationNetwork) {