From 98877d0676fc092b5ed73f905d47df7e2a7e1ad1 Mon Sep 17 00:00:00 2001 From: Phu Pham Date: Thu, 3 Apr 2025 23:59:48 +1100 Subject: [PATCH 1/7] remove db access per validation for fee data --- src/connection-manager/connections.ts | 52 ++++++++++++++++----------- src/connection-manager/wsHandling.ts | 12 +++---- 2 files changed, 36 insertions(+), 28 deletions(-) diff --git a/src/connection-manager/connections.ts b/src/connection-manager/connections.ts index 7ed4e571..326d1adb 100644 --- a/src/connection-manager/connections.ts +++ b/src/connection-manager/connections.ts @@ -27,6 +27,7 @@ const ports = [443, 80, 6005, 6006, 51233, 51234] const protocols = ['wss://', 'ws://'] const connections: Map = new Map() const networkFee: Map = new Map() +const validationNetworkDb: Map = new Map() const CM_INTERVAL = 60 * 60 * 1000 const WS_TIMEOUT = 10000 const REPORTING_INTERVAL = 15 * 60 * 1000 @@ -60,11 +61,11 @@ async function setHandlers( isInitialNode = false, retryCount = 0, ): Promise { - log.info( - `Initiated Websocket connection for: ${ws_url} on ${ - networks ?? 'unknown network' - }`, - ) + // log.info( + // `Initiated Websocket connection for: ${ws_url} on ${ + // networks ?? 'unknown network' + // }`, + // ) const ledger_hashes: string[] = [] return new Promise(function setHandlersPromise(resolve, _reject) { @@ -119,24 +120,25 @@ async function setHandlers( networks, networkFee, ws, + validationNetworkDb, ) } }) ws.on('close', async (code, reason) => { - log.error( - `Websocket closed for ${ws.url} on ${ - networks ?? 'unknown network' - } with code ${code} and reason ${reason.toString('utf-8')}.`, - ) + // log.error( + // `Websocket closed for ${ws.url} on ${ + // networks ?? 'unknown network' + // } with code ${code} and reason ${reason.toString('utf-8')}.`, + // ) const delay = BASE_RETRY_DELAY * 2 ** retryCount if (CLOSING_CODES.includes(code) && delay <= MAX_RETRY_DELAY) { - log.info( - `Reconnecting to ${ws.url} on ${ - networks ?? 'unknown network' - } after ${delay}ms...`, - ) + // log.info( + // `Reconnecting to ${ws.url} on ${ + // networks ?? 'unknown network' + // } after ${delay}ms...`, + // ) // Clean up the old Websocket connection connections.delete(ws.url) ws.terminate() @@ -167,11 +169,11 @@ async function setHandlers( resolve() }) ws.on('error', (err) => { - log.error( - `Websocket connection error for ${ws.url} on ${ - networks ?? 'unknown network' - } - ${err.message}`, - ) + // log.error( + // `Websocket connection error for ${ws.url} on ${ + // networks ?? 'unknown network' + // } - ${err.message}`, + // ) if (connections.get(ws.url)?.url === ws.url) { connections.delete(ws.url) @@ -230,6 +232,14 @@ async function findConnection(node: WsNode): Promise { return Promise.resolve() } +async function getValidationNetworkDb(): Promise { + const validatorNetwork: Array<{ signing_key: string; networks: string }> = + await query('validators').select('signing_key', 'networks') + for (const entry of validatorNetwork) { + validationNetworkDb.set(entry.signing_key, entry.networks) + } +} + /** * Creates connections to nodes found in the database. * @@ -237,6 +247,8 @@ async function findConnection(node: WsNode): Promise { */ async function createConnections(): Promise { log.info('Finding Connections...') + validationNetworkDb.clear() + await getValidationNetworkDb() const tenMinutesAgo = new Date() tenMinutesAgo.setMinutes(tenMinutesAgo.getMinutes() - 10) diff --git a/src/connection-manager/wsHandling.ts b/src/connection-manager/wsHandling.ts index 86fb4506..682d9a90 100644 --- a/src/connection-manager/wsHandling.ts +++ b/src/connection-manager/wsHandling.ts @@ -106,6 +106,7 @@ function isFlagLedgerPlusOne(ledger_index: number): boolean { * @param networks - The networks of subscribed node. * @param network_fee - The map of default fee for the network to be used in case the validator does not vote for a new fee. * @param ws - The WebSocket message received from. + * @param validationNetworkDb -- The validation network map to fetch fee data. * @returns Void. */ // eslint-disable-next-line max-params -- Disabled for this function. @@ -115,6 +116,7 @@ export async function handleWsMessageSubscribeTypes( networks: string | undefined, network_fee: Map, ws: WebSocket, + validationNetworkDb: Map, ): Promise { if (data.type === 'validationReceived') { const validationData = data as ValidationRaw @@ -122,15 +124,9 @@ export async function handleWsMessageSubscribeTypes( validationData.networks = networks } - // Get network of the validation if ledger_hash is not in cache. - const validationNetworkDb: DatabaseValidator | undefined = await query( - 'validators', - ) - .select('*') - .where('signing_key', validationData.validation_public_key) - .first() const validationNetwork = - validationNetworkDb?.networks ?? validationData.networks + validationNetworkDb.get(validationData.validation_public_key) ?? + validationData.networks // Get the fee for the network to be used in case the validator does not vote for a new fee. if (validationNetwork) { From be59d1ea999504a8828a766156f87768e2bdd296 Mon Sep 17 00:00:00 2001 From: Phu Pham Date: Fri, 4 Apr 2025 00:17:29 +1100 Subject: [PATCH 2/7] fix lint --- src/connection-manager/connections.ts | 4 ++-- src/connection-manager/wsHandling.ts | 7 +------ 2 files changed, 3 insertions(+), 8 deletions(-) diff --git a/src/connection-manager/connections.ts b/src/connection-manager/connections.ts index 326d1adb..d7a6d98b 100644 --- a/src/connection-manager/connections.ts +++ b/src/connection-manager/connections.ts @@ -124,7 +124,7 @@ async function setHandlers( ) } }) - ws.on('close', async (code, reason) => { + ws.on('close', async (code) => { // log.error( // `Websocket closed for ${ws.url} on ${ // networks ?? 'unknown network' @@ -168,7 +168,7 @@ async function setHandlers( ws.terminate() resolve() }) - ws.on('error', (err) => { + ws.on('error', () => { // log.error( // `Websocket connection error for ${ws.url} on ${ // networks ?? 'unknown network' diff --git a/src/connection-manager/wsHandling.ts b/src/connection-manager/wsHandling.ts index 682d9a90..6a0db3dd 100644 --- a/src/connection-manager/wsHandling.ts +++ b/src/connection-manager/wsHandling.ts @@ -9,18 +9,13 @@ import { import { AMENDMENTS_ID } from 'xrpl/dist/npm/models/ledger' import { LedgerResponseExpanded } from 'xrpl/dist/npm/models/methods/ledger' -import { - query, - saveAmendmentStatus, - saveAmendmentsStatus, -} from '../shared/database' +import { saveAmendmentStatus, saveAmendmentsStatus } from '../shared/database' import { NETWORKS_HOSTS, deleteAmendmentStatus, } from '../shared/database/amendments' import { AmendmentStatus, - DatabaseValidator, FeeVote, StreamLedger, StreamManifest, From 5e248f7930099a01f151bd79943918960d43adb5 Mon Sep 17 00:00:00 2001 From: Raj Patel Date: Thu, 3 Apr 2025 09:58:35 -0400 Subject: [PATCH 3/7] print chains included in hourly agreement --- src/connection-manager/agreement.ts | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/connection-manager/agreement.ts b/src/connection-manager/agreement.ts index d4c2f65c..c6475173 100644 --- a/src/connection-manager/agreement.ts +++ b/src/connection-manager/agreement.ts @@ -153,6 +153,10 @@ class Agreement { for (const chain of agreementChains) { const ledger_hashes = chain.ledgers + log.info( + `Agreement: ${chain.id}:${Array.from(chain.validators).join(',')}`, + ) + for (const signing_key of chain.validators) { promises.push( this.calculateValidatorAgreement( From f7706020b7397fd46864ca53ba3317e3178fba78 Mon Sep 17 00:00:00 2001 From: Raj Patel Date: Thu, 3 Apr 2025 10:28:48 -0400 Subject: [PATCH 4/7] add network name to logs --- src/connection-manager/agreement.ts | 34 +++++++++++++++++++++++++++-- 1 file changed, 32 insertions(+), 2 deletions(-) diff --git a/src/connection-manager/agreement.ts b/src/connection-manager/agreement.ts index c6475173..c9cab3b3 100644 --- a/src/connection-manager/agreement.ts +++ b/src/connection-manager/agreement.ts @@ -17,14 +17,16 @@ import { ValidationRaw, ValidatorKeys, Ballot, + Chain, } from '../shared/types' +import { getLists, overlaps } from '../shared/utils' import logger from '../shared/utils/logger' import chains from './chains' const log = logger({ name: 'agreement' }) -const AGREEMENT_INTERVAL = 60 * 60 * 1000 +const AGREEMENT_INTERVAL = 0.5 * 60 * 1000 const PURGE_INTERVAL = 10 * 60 * 1000 /** @@ -120,6 +122,30 @@ function isPreceedingFlagLedger(ledger_index: string): boolean { return parseInt(ledger_index, 10) % 256 === 255 } +/** + * Finds network name from chain id + * + * @param chain - A chain object. + * @returns string. + */ +async function getNetworkNameFromChainId(chain: Chain): Promise { + let id = chain.id + const lists = await getLists().catch((err) => { + log.error('Error getting validator lists', err) + return undefined + }) + + if (lists != null) { + Object.entries(lists).forEach(([network, set]) => { + if (overlaps(chain.validators, set)) { + id = network + } + }) + } + + return id +} + /** * */ @@ -153,8 +179,12 @@ class Agreement { for (const chain of agreementChains) { const ledger_hashes = chain.ledgers + const networkName = await getNetworkNameFromChainId(chain) + log.info( - `Agreement: ${chain.id}:${Array.from(chain.validators).join(',')}`, + `Agreement: ${chain.id}:${networkName}:${Array.from( + chain.validators, + ).join(',')}`, ) for (const signing_key of chain.validators) { From 9aa3c13fc7dccc9382d757721923cef0a66771d4 Mon Sep 17 00:00:00 2001 From: Raj Patel Date: Thu, 3 Apr 2025 10:29:45 -0400 Subject: [PATCH 5/7] revert AGREEMENT_INTERVAL --- src/connection-manager/agreement.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/connection-manager/agreement.ts b/src/connection-manager/agreement.ts index c9cab3b3..670ad079 100644 --- a/src/connection-manager/agreement.ts +++ b/src/connection-manager/agreement.ts @@ -26,7 +26,7 @@ import chains from './chains' const log = logger({ name: 'agreement' }) -const AGREEMENT_INTERVAL = 0.5 * 60 * 1000 +const AGREEMENT_INTERVAL = 60 * 60 * 1000 const PURGE_INTERVAL = 10 * 60 * 1000 /** From f85a9b16db7b4ce4f6cee023c8a5665e5c617a9a Mon Sep 17 00:00:00 2001 From: Raj Patel Date: Thu, 3 Apr 2025 10:34:10 -0400 Subject: [PATCH 6/7] fix comment --- src/connection-manager/agreement.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/connection-manager/agreement.ts b/src/connection-manager/agreement.ts index 670ad079..38277acb 100644 --- a/src/connection-manager/agreement.ts +++ b/src/connection-manager/agreement.ts @@ -123,10 +123,10 @@ function isPreceedingFlagLedger(ledger_index: string): boolean { } /** - * Finds network name from chain id + * Finds network name from chain id. * * @param chain - A chain object. - * @returns string. + * @returns String. */ async function getNetworkNameFromChainId(chain: Chain): Promise { let id = chain.id From 4ffb62ba1d845db9152b40359a1e7068b1a2f4df Mon Sep 17 00:00:00 2001 From: Raj Patel Date: Thu, 3 Apr 2025 17:47:31 -0400 Subject: [PATCH 7/7] add logs --- src/connection-manager/connections.ts | 40 ++++++++++++--------------- src/connection-manager/wsHandling.ts | 2 +- 2 files changed, 18 insertions(+), 24 deletions(-) diff --git a/src/connection-manager/connections.ts b/src/connection-manager/connections.ts index d7a6d98b..ec283b8d 100644 --- a/src/connection-manager/connections.ts +++ b/src/connection-manager/connections.ts @@ -61,12 +61,6 @@ async function setHandlers( isInitialNode = false, retryCount = 0, ): Promise { - // log.info( - // `Initiated Websocket connection for: ${ws_url} on ${ - // networks ?? 'unknown network' - // }`, - // ) - const ledger_hashes: string[] = [] return new Promise(function setHandlersPromise(resolve, _reject) { ws.on('open', () => { @@ -124,21 +118,21 @@ async function setHandlers( ) } }) - ws.on('close', async (code) => { - // log.error( - // `Websocket closed for ${ws.url} on ${ - // networks ?? 'unknown network' - // } with code ${code} and reason ${reason.toString('utf-8')}.`, - // ) + ws.on('close', async (code, reason) => { + log.error( + `Websocket closed for ${ws.url} on ${ + networks ?? 'unknown network' + } with code ${code} and reason ${reason.toString('utf-8')}.`, + ) const delay = BASE_RETRY_DELAY * 2 ** retryCount if (CLOSING_CODES.includes(code) && delay <= MAX_RETRY_DELAY) { - // log.info( - // `Reconnecting to ${ws.url} on ${ - // networks ?? 'unknown network' - // } after ${delay}ms...`, - // ) + log.info( + `Reconnecting to ${ws.url} on ${ + networks ?? 'unknown network' + } after ${delay}ms...`, + ) // Clean up the old Websocket connection connections.delete(ws.url) ws.terminate() @@ -168,12 +162,12 @@ async function setHandlers( ws.terminate() resolve() }) - ws.on('error', () => { - // log.error( - // `Websocket connection error for ${ws.url} on ${ - // networks ?? 'unknown network' - // } - ${err.message}`, - // ) + ws.on('error', (err) => { + log.error( + `Websocket connection error for ${ws.url} on ${ + networks ?? 'unknown network' + } - ${err.message}`, + ) if (connections.get(ws.url)?.url === ws.url) { connections.delete(ws.url) diff --git a/src/connection-manager/wsHandling.ts b/src/connection-manager/wsHandling.ts index 6a0db3dd..31c0066e 100644 --- a/src/connection-manager/wsHandling.ts +++ b/src/connection-manager/wsHandling.ts @@ -101,7 +101,7 @@ function isFlagLedgerPlusOne(ledger_index: number): boolean { * @param networks - The networks of subscribed node. * @param network_fee - The map of default fee for the network to be used in case the validator does not vote for a new fee. * @param ws - The WebSocket message received from. - * @param validationNetworkDb -- The validation network map to fetch fee data. + * @param validationNetworkDb -- A map of validator signing_keys to their corresponding networks. * @returns Void. */ // eslint-disable-next-line max-params -- Disabled for this function.