Expose WebSocket connection count as prometheus metrics and some refactors

.env.example

@@ -17,6 +17,7 @@ DB_HOST=
 DB_USER=
 DB_DATABASE=
 DB_PASSWORD=
+ACQUIRE_CONNECTION_TIMEOUT=
 
 # Mainnet entry point for crawler (without https://).
 MAINNET_P2P_ENTRY=

ARCHITECTURE.md

@@ -12,6 +12,7 @@ There are 3 folders in `src`, corresponding to the 3 processes that the VHS runs
 * `/`: Information about the endpoints.
 * `v1`
   * `/health`: A health check for the VHS. Returns the number of nodes that it is connected to.
+  * `/metrics/:network`: A health check for the VHS. Returns the number of nodes that it is connected to for a particular network in prometheus exposition format.
   * `/networks`: Returns the list of all networks on VHS database.
   * `/network/validator_reports`: Returns scores for the nodes that it has crawled in the last day.
   * `/network/topology`: Returns information about all the nodes that the crawler has crawled in the last hour.
@@ -42,8 +43,6 @@ This table keeps track of the nodes in the network, which it finds via crawling
 | `complete_shards`     |The [history shards](https://xrpl.org/history-sharding.html) the node keeps track of.|
 | `ip`                 |The IP address of the node.                                                          |
 | `port`               |The peer port of the node.                                                           |
-| `ws_url`             |The WS URL of the node. Optional.                                                    |
-| `connected`          |This appears to be false for every node.                                             |
 | `networks`           |The network(s) that the node belongs to.                                             |
 | `type`               |Whether the TCP connection to the peer is incoming or outgoing.                      |
 | `uptime`             |The uptime of the node.                                                              |
@@ -186,5 +185,18 @@ This table keeps track of the validators on the networks.
 | `agreement_24hour`   |Data about the reliability of the validator over the last 24 hours.|
 | `agreement_30day`    |Data about the reliability of the validator over the 30 days.      |
 
+
+### `connection_health`
+
+This table keeps track of the WebSocket connection status for all networks.
+
+| Key                  | Definition                                                        |
+|----------------------|-------------------------------------------------------------------|
+| `ws_url`             |The connection websocket url.                                      |
+| `public_key `        |The public key of the node.                                        |
+| `network`            |The network that the node belongs to.                              |
+| `connected`          |Boolean denoting websocket connection status.                      |
+| `status_update_time` |Time when the connected column was updated.                        |
+
 *Partial validations are not meant to vote for any particular ledger. A partial validation indicates that the validator is still online but not keeping up with consensus.
 **A chain is a group of validators validating the same set of ledgers. `main`, `test`, and `dev` represent the validated versions of mainnet, testnet, and devnet respectively. Validators on a fork/validating an alternate version of the ledger will have a different value, usually of the form `chain.[num]`.

src/api/routes/v1/health.ts

@@ -13,11 +13,37 @@ export default async function handleHealth(
   res: Response,
 ): Promise<void> {
   try {
-    const count = (await query('crawls')
-      .countDistinct('ip')
+    const count = (await query('connection_health')
+      .count('ws_url')
       .where('connected', '=', true)) as Array<{ [key: string]: number }>
     res.status(200).send(count[0])
   } catch {
     res.send({ result: 'error', message: 'internal error' })
   }
 }
+
+/**
+ * Handles monitoring metrics requests.
+ *
+ * @param req - HTTP request object.
+ * @param res - Response containing number of connected nodes in Prometheus exposition format.
+ */
+export async function handleWebSocketHealthMetrics(
+  req: Request,
+  res: Response,
+): Promise<void> {
+  try {
+    const { network } = req.params
+    const result = (await query('connection_health')
+      .count('ws_url')
+      .where('network', '=', network)
+      .andWhere('connected', '=', true)) as Array<{ [key: string]: number }>
+
+    const metrics = `connected_nodes{network="${network}"} ${result[0].count}`
+    res.set('Content-Type', 'text/plain')
+    res.status(200)
+    res.send(metrics)
+  } catch {
+    res.send({ result: 'error', message: 'internal error' })
+  }
+}

src/api/routes/v1/index.ts

@@ -8,7 +8,7 @@ import {
 } from './amendments'
 import handleDailyScores from './daily-report'
 import getNetworkOrAdd from './get-network'
-import handleHealth from './health'
+import handleHealth, { handleWebSocketHealthMetrics } from './health'
 import handleValidatorManifest from './manifests'
 import handleNetworks from './networks'
 import { handleNode, handleNodes, handleTopology } from './nodes'
@@ -18,6 +18,7 @@ import handleValidatorReport from './validator-report'
 const api = createRouter()
 
 api.use('/health', handleHealth)
+api.use('/metrics/:network', handleWebSocketHealthMetrics)
 api.use('/network/validator_reports', handleDailyScores)
 api.use('/network/amendment/info/:param', handleAmendmentInfo)
 api.use('/network/amendments/info', handleAmendmentsInfo)

src/api/routes/v1/info.ts

@@ -82,6 +82,17 @@ const info = {
       example:
         'https://data.xrpl.org/v1/network/amendments/vote/{network}/{identifier}',
     },
+    {
+      action: 'Get total number of connected rippled nodes.',
+      route: '/v1/health',
+      example: 'https://data.xrpl.org/v1/health',
+    },
+    {
+      action:
+        'Get total number of connected rippled nodes for a particular network in prometheus exposition format.',
+      route: '/v1/metrics/{network}',
+      example: 'https://data.xrpl.org/v1/metrics/{network}',
+    },
   ],
 }