feat: Use network_id in the Validation subscripton stream (#396)

## High Level Overview of Change

Use `network_id` value from the Validation subscription stream. This PR
obviates the need for complex `chains` assignment logic in the
`chains.updateChains` method.

<!--
Please include a summary/list of the changes.
If too broad, please consider splitting into multiple PRs.
If a relevant Asana task, please link it here.
-->

### Context of Change

<!--
Please include the context of a change.
If a bug fix, when was the bug introduced? What was the behavior?
If a new feature, why was this architecture chosen? What were the
alternatives?
If a refactor, how is this better than the previous implementation?

If there is a design document for this feature, please link it here.
-->


<!--
Please check relevant options, delete irrelevant ones.
-->

- [ ] Bug fix (non-breaking change which fixes an issue)
- [ ] New feature (non-breaking change which adds functionality)
- [ ] Breaking change (fix or feature that would cause existing
functionality to not work as expected)
- [x] Refactor (non-breaking change that only restructures code)
- [ ] Tests (You added tests for code that already exists, or your new
feature included in this PR)
- [ ] Documentation Updates
- [ ] Release

## Before / After

There are no changes to the externally visible APIs.

<!--
If just refactoring / back-end changes, this can be just an in-English
description of the change at a technical level.
If a UI change, screenshots should be included.
-->

## Test Plan

Existing unit tests pass with this refactor. Additional unit tests have
also been added to test the logic of chains.

<!--
Please describe the tests that you ran to verify your changes and
provide instructions so that others can reproduce.
-->

<!--
## Future Tasks
For future tasks related to PR.
-->

Author: ckeshava

src/api/routes/v1/get-network.ts

@@ -190,12 +190,14 @@ async function getNetworkFromPublicKey(
  * @param url - The URL endpoint of the node.
  * @param unl - The UNL of the node.
  * @param port - The peer port of the node.
+ * @param network_id - The network ID which uniquely identifies the XRPL network.
  * @returns The ID of the new network.
  */
 async function addNode(
   url: string,
   unl: string | null,
   port: number,
+  network_id: number,
 ): Promise<string> {
   const newNetwork = (maxNetwork + 1).toString()
   maxNetwork += 1
@@ -205,6 +207,7 @@ async function addNode(
     entry: url,
     port,
     unls: unl ? [unl] : [],
+    network_id,
   }
   await query('networks').insert({
     ...network,
@@ -248,7 +251,7 @@ export default async function getNetworkOrAdd(
     }
 
     // check if node public key is already recorded
-    const { public_key } = crawl.this_node
+    const { public_key, network_id } = crawl.this_node
     const publicKeyNetwork = await getNetworkFromPublicKey(public_key)
     if (publicKeyNetwork != null) {
       return res.status(200).send({
@@ -257,7 +260,7 @@ export default async function getNetworkOrAdd(
       })
     }
     // add node to networks list
-    const newNetwork = await addNode(entryUrl, node_unl, port)
+    const newNetwork = await addNode(entryUrl, node_unl, port, network_id)
 
     return res.status(200).send({
       result: 'success',

src/connection-manager/agreement.ts

@@ -1,3 +1,4 @@
+/* eslint-disable max-lines -- This file contains useful logging statements and complex logic. */
 import {
   getAgreementScores,
   saveHourlyAgreement,
@@ -18,6 +19,7 @@ import {
   ValidatorKeys,
   Ballot,
   Chain,
+  LedgerHashIndex,
 } from '../shared/types'
 import { getLists, overlaps } from '../shared/utils'
 import logger from '../shared/utils/logger'
@@ -98,18 +100,48 @@ async function updateAgreementScores(
 async function updateDailyAgreement(
   validator_keys: ValidatorKeys,
 ): Promise<void> {
-  const start = new Date()
-  start.setHours(0, 0, 0, 0)
-  const end = new Date()
-  end.setHours(23, 59, 59, 999)
-
-  const agreement = await getAgreementScores(validator_keys, start, end)
+  interface DailyAgreementDate {
+    start: Date
+    end: Date
+  }
+  const datesForDailyAgreementComputation: DailyAgreementDate[] = []
+  const todayStart = new Date()
+  todayStart.setHours(0, 0, 0, 0)
+  const todayEnd = new Date()
+  todayEnd.setHours(23, 59, 59, 999)
+
+  datesForDailyAgreementComputation.push({ start: todayStart, end: todayEnd })
+
+  // if this method is invoked over the last hour of a given day, save the agreement scores for the previous day.
+  // Without this additional logic, the agreement scores for the last hour (over VHSs' 24 hour time period) will not be saved.
+  // Background Context: VHS retroactively measures agreement scores for the validations received over the previous hour.
+  if (new Date().getHours() === 0) {
+    const yesterdayStart = new Date()
+    yesterdayStart.setDate(yesterdayStart.getDate() - 1)
+    yesterdayStart.setHours(0, 0, 0, 0)
+
+    const yesterdayEnd = new Date()
+    yesterdayEnd.setHours(23, 59, 59, 999)
+
+    datesForDailyAgreementComputation.push({
+      start: yesterdayStart,
+      end: yesterdayEnd,
+    })
+    // Note: saveDailyAgreement method is idempotent. There is no harm in invoking it for today and previous day.
+  }
 
-  await saveDailyAgreement({
-    main_key: validator_keys.master_key ?? validator_keys.signing_key,
-    day: start,
-    agreement,
-  })
+  for (const date of datesForDailyAgreementComputation) {
+    const agreement = await getAgreementScores(
+      validator_keys,
+      date.start,
+      date.end,
+    )
+    await saveDailyAgreement({
+      main_key: validator_keys.master_key ?? validator_keys.signing_key,
+      day: date.start,
+      agreement,
+    })
+  }
 }
 
 /**
@@ -129,7 +161,7 @@ function isPreceedingFlagLedger(ledger_index: string): boolean {
  * @returns String.
  */
 async function getNetworkNameFromChainId(chain: Chain): Promise<string> {
-  let id = chain.id
+  let networkID: number | string = chain.network_id
   const lists = await getLists().catch((err) => {
     log.error('Error getting validator lists', err)
     return undefined
@@ -138,12 +170,12 @@ async function getNetworkNameFromChainId(chain: Chain): Promise<string> {
   if (lists != null) {
     Object.entries(lists).forEach(([network, set]) => {
       if (overlaps(chain.validators, set)) {
-        id = network
+        networkID = network
       }
     })
   }
 
-  return id
+  return networkID.toString()
 }
 
 /**
@@ -182,7 +214,7 @@ class Agreement {
       const networkName = await getNetworkNameFromChainId(chain)
 
       log.info(
-        `Agreement: ${chain.id}:${networkName}:${Array.from(
+        `Agreement: ${chain.network_id}:${networkName}:${Array.from(
           chain.validators,
         ).join(',')}`,
       )
@@ -254,7 +286,7 @@ class Agreement {
         validator.server_version = serverVersion
       }
 
-      chains.updateLedgers(validation)
+      await chains.updateLedgers(validation)
       await saveValidator(validator)
     }
   }
@@ -268,7 +300,7 @@ class Agreement {
    */
   private async calculateValidatorAgreement(
     signing_key: string,
-    ledger_hashes: Set<string>,
+    ledger_hashes: Set<LedgerHashIndex>,
     incomplete: boolean,
   ): Promise<void> {
     const master_key = await signingToMaster(signing_key)
@@ -289,19 +321,30 @@ class Agreement {
    *
    * @param validator_keys - Signing keys of validations for one validator.
    * @param validations - Set of ledger_hashes validated by signing_key.
-   * @param ledgers - Set of ledger_hashes validated by network.
+   * @param ledgerHashIndexMap - Set of pairs of (ledger_hash, ledger_index) ledgers validated by the network.
    * @param incomplete - Is this agreement score incomplete.
    * @returns Void.
    */
   private async calculateHourlyAgreement(
     validator_keys: ValidatorKeys,
     validations: Map<string, number>,
-    ledgers: Set<string>,
+    ledgerHashIndexMap: Set<LedgerHashIndex>,
     incomplete: boolean,
   ): Promise<void> {
+    // obtain ledger_hashes validated by the network, strip out the ledger_index info for agreement calculation purposes
+    const ledgers = new Set<string>()
+    for (const value of ledgerHashIndexMap) {
+      ledgers.add(value.ledger_hash)
+    }
     const missed = setDifference(ledgers, validations)
     const validated = setIntersection(ledgers, validations)
 
+    log.trace(
+      `Tracking information from validator with Master-Key: ${validator_keys.master_key ?? ''}, Signing-Key: ${validator_keys.signing_key}`,
+    )
+    log.trace(`Missed ledgers: ${JSON.stringify(Array.from(missed))}`)
+    log.trace(`Validated ledgers: ${JSON.stringify(Array.from(validated))}`)
+
     const agreement: AgreementScore = {
       validated: validated.size,
       missed: missed.size,
@@ -312,6 +355,9 @@ class Agreement {
       start: this.reported_at,
       agreement,
     })
+    log.info(
+      `Saving hourly agreement for validator with Master-Key: ${validator_keys.master_key ?? ''}, Signing-Key: ${validator_keys.signing_key} indexed by time: ${this.reported_at.toISOString()}; Current time: ${new Date().toISOString()}; Agreement: ${JSON.stringify(agreement)}`,
+    )
 
     await update1HourValidatorAgreement(validator_keys, agreement)
     await updateAgreementScores(validator_keys)
@@ -361,3 +407,4 @@ function getAgreementInstance(): Agreement {
 }
 
 export default getAgreementInstance()
+/* eslint-enable max-lines */