feat(health): expose whether the tip we serve is still the chain's tip

The explorer reads everything from one faircoind. When that node stops
following the chain the RPC keeps answering perfectly, so every endpoint keeps
returning 200 with a frozen height and nothing looks wrong — which is how the
node sat 80+ blocks behind for over an hour today without a single alarm.

/api/health compares our height against the heights our peers report and says
so out loud, answering 503 when a peer is demonstrably ahead and our tip has
gone cold. Two rules keep it honest: ordinary propagation delay (a peer ahead,
tip still recent) is "lagging", not an outage; and a cold tip with no peer
heights at all reports "unknown" rather than "ok", because there is nothing to
compare against and claiming health is exactly the failure being fixed. A quiet
network — every peer agreeing with us — is never a stall.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>

Author: NateIsern

server/index.ts

@@ -16,6 +16,7 @@ import { handleRouteError, parseNetwork, parseLimit, parseOffset, parseBlockOffs
 import { computeCirculatingSupply, currentBlockReward } from '../shared/supply'
 import { logger } from './lib/logger'
 import { toPublicNetworkInfo } from './lib/network-info'
+import { assessNodeHealth } from './lib/node-health'
 import { rpcWithNetwork } from '@fairco.in/rpc-client'
 import priceRouter from './routes/price'
 import statsHistoryRouter from './routes/stats-history'
@@ -476,6 +477,45 @@ app.get('/api/peers', async (req, res) => {
   }
 })
 
+/**
+ * Is the tip we are serving still the chain's tip?
+ *
+ * Every other endpoint answers 200 with whatever height the node reports, which
+ * is indistinguishable from a healthy answer when the node has silently stopped
+ * following the chain. This is the one endpoint that compares our height against
+ * what our peers claim and says so out loud — and it answers 503 when we are
+ * demonstrably behind, so an uptime check catches it without a human noticing.
+ */
+app.get('/api/health', async (req, res) => {
+  const network = parseNetwork(req.query.network)
+  try {
+    // Short TTLs: a health probe that reads a minute-old cache cannot detect a
+    // node that stopped a minute ago.
+    const [nodeHeight, tipHash, peers] = await Promise.all([
+      blockCache.get<number>('getblockcount', [], { network, ttl: 10 }),
+      blockCache.get<string>('getbestblockhash', [], { network, ttl: 10 }),
+      blockCache
+        .get<Array<{ synced_headers?: number; startingheight?: number }>>('getpeerinfo', [], { network, ttl: 20 })
+        .catch(() => []),
+    ])
+    const tip = await blockCache.getBlock(tipHash, network, true)
+    const health = assessNodeHealth({
+      nodeHeight,
+      tipTime: Number(tip?.time ?? 0),
+      peerHeights: (Array.isArray(peers) ? peers : []).flatMap((peer) => [
+        Number(peer?.synced_headers ?? -1),
+        Number(peer?.startingheight ?? -1),
+      ]),
+      now: Math.floor(Date.now() / 1000),
+    })
+    res.status(health.status === 'stalled' ? 503 : 200).json({ ...health, network })
+  } catch (error) {
+    logger.error('Health check failed:', error)
+    // An unreachable node is never "ok" — fail loud rather than answer 200.
+    res.status(503).json({ status: 'unknown', error: 'node unreachable', network })
+  }
+})
+
 app.get('/api/stats', async (req, res) => {
   try {
     const network = parseNetwork(req.query.network)

server/lib/node-health.test.ts

@@ -0,0 +1,88 @@
+import { describe, expect, it } from 'vitest'
+import { assessNodeHealth, STALE_TIP_SECONDS } from './node-health'
+
+/**
+ * The explorer used to serve its node's tip with no indication of whether that
+ * tip was still current. When the node stopped following the chain, every
+ * endpoint kept answering 200 with a frozen height and nothing looked wrong —
+ * the outage was invisible until a human happened to compare against another
+ * node. These tests lock the rule that makes that impossible: a tip is only
+ * "ok" when nobody we are connected to has more chain than we do.
+ */
+
+const NOW = 1_800_000_000
+const FRESH = NOW - 60
+
+describe('assessNodeHealth', () => {
+  it('is ok when the node matches the best peer', () => {
+    const health = assessNodeHealth({ nodeHeight: 100, tipTime: FRESH, peerHeights: [100, 99], now: NOW })
+    expect(health.status).toBe('ok')
+    expect(health.lagBlocks).toBe(0)
+    expect(health.networkHeight).toBe(100)
+  })
+
+  it('is ok when the node is ahead of every peer', () => {
+    const health = assessNodeHealth({ nodeHeight: 101, tipTime: FRESH, peerHeights: [100], now: NOW })
+    expect(health.status).toBe('ok')
+    expect(health.lagBlocks).toBe(0)
+  })
+
+  it('is lagging when a peer is ahead but the tip is still fresh', () => {
+    // A block or two behind with a recent tip is normal propagation, not an outage.
+    const health = assessNodeHealth({ nodeHeight: 98, tipTime: FRESH, peerHeights: [100], now: NOW })
+    expect(health.status).toBe('lagging')
+    expect(health.lagBlocks).toBe(2)
+  })
+
+  it('is stalled when a peer is ahead AND our tip has gone stale', () => {
+    const health = assessNodeHealth({
+      nodeHeight: 81534,
+      tipTime: NOW - STALE_TIP_SECONDS - 1,
+      peerHeights: [81603],
+      now: NOW,
+    })
+    expect(health.status).toBe('stalled')
+    expect(health.lagBlocks).toBe(69)
+    expect(health.tipAgeSeconds).toBeGreaterThan(STALE_TIP_SECONDS)
+  })
+
+  it('does not call a quiet network a stall', () => {
+    // Every peer agrees with us; the chain itself simply is not producing.
+    const health = assessNodeHealth({
+      nodeHeight: 100,
+      tipTime: NOW - STALE_TIP_SECONDS - 1,
+      peerHeights: [100, 100],
+      now: NOW,
+    })
+    expect(health.status).toBe('ok')
+    expect(health.lagBlocks).toBe(0)
+  })
+
+  it('reports unknown, never ok, when the tip is stale and we have no peer heights', () => {
+    // With nothing to compare against we cannot claim health either way, and
+    // claiming "ok" is exactly the failure this endpoint exists to prevent.
+    const health = assessNodeHealth({
+      nodeHeight: 100,
+      tipTime: NOW - STALE_TIP_SECONDS - 1,
+      peerHeights: [],
+      now: NOW,
+    })
+    expect(health.status).toBe('unknown')
+  })
+
+  it('is ok with no peers while the tip is fresh', () => {
+    const health = assessNodeHealth({ nodeHeight: 100, tipTime: FRESH, peerHeights: [], now: NOW })
+    expect(health.status).toBe('ok')
+  })
+
+  it('ignores the -1 sentinel peers report before they have synced with us', () => {
+    const health = assessNodeHealth({ nodeHeight: 100, tipTime: FRESH, peerHeights: [-1, -1], now: NOW })
+    expect(health.networkHeight).toBe(100)
+    expect(health.status).toBe('ok')
+  })
+
+  it('never reports a negative lag', () => {
+    const health = assessNodeHealth({ nodeHeight: 200, tipTime: FRESH, peerHeights: [100], now: NOW })
+    expect(health.lagBlocks).toBe(0)
+  })
+})

server/lib/node-health.ts

@@ -0,0 +1,71 @@
+/**
+ * Node health — is the tip we are serving still the chain's tip?
+ *
+ * The explorer reads everything from one faircoind over RPC. When that node
+ * stops following the chain the RPC keeps answering perfectly, so every endpoint
+ * keeps returning 200 with a frozen height and nothing looks wrong. That is how
+ * a node parked 80+ blocks behind went unnoticed for over an hour: the site was
+ * confidently serving a stale chain.
+ *
+ * The only local signal that can catch it is the peers' own heights, which the
+ * node already reports (`getpeerinfo` → `synced_headers` / `startingheight`).
+ * If somebody we are connected to has more chain than we do and our tip has gone
+ * cold, we are the problem — not the network.
+ */
+
+/** A tip older than this is cold. Target spacing is 120s; this is ~7 blocks. */
+export const STALE_TIP_SECONDS = 900
+
+export interface NodeHealthInput {
+  /** Height of the node's active chain. */
+  nodeHeight: number
+  /** Unix seconds of the tip block's timestamp. */
+  tipTime: number
+  /** Heights reported by connected peers. `-1` (not yet known) is ignored. */
+  peerHeights: number[]
+  /** Unix seconds now. */
+  now: number
+}
+
+export type NodeHealthStatus =
+  /** Serving the chain tip, as far as anyone we can see knows. */
+  | 'ok'
+  /** A peer is ahead but our tip is recent — ordinary propagation delay. */
+  | 'lagging'
+  /** A peer is ahead AND our tip is cold: we have stopped following the chain. */
+  | 'stalled'
+  /** Tip is cold and no peer height is known, so health cannot be established. */
+  | 'unknown'
+
+export interface NodeHealth {
+  status: NodeHealthStatus
+  nodeHeight: number
+  /** The best height anyone (us included) is known to have. */
+  networkHeight: number
+  /** How far behind `networkHeight` we are; never negative. */
+  lagBlocks: number
+  tipAgeSeconds: number
+}
+
+export function assessNodeHealth(input: NodeHealthInput): NodeHealth {
+  const { nodeHeight, tipTime, peerHeights, now } = input
+
+  // Peers report -1 until the handshake settles; those carry no information.
+  const knownPeerHeights = peerHeights.filter((height) => Number.isFinite(height) && height >= 0)
+  const networkHeight = Math.max(nodeHeight, ...knownPeerHeights)
+  const lagBlocks = Math.max(0, networkHeight - nodeHeight)
+  const tipAgeSeconds = now - tipTime
+  const tipIsCold = tipAgeSeconds > STALE_TIP_SECONDS
+
+  if (lagBlocks > 0) {
+    return { status: tipIsCold ? 'stalled' : 'lagging', nodeHeight, networkHeight, lagBlocks, tipAgeSeconds }
+  }
+
+  // Nobody is ahead of us. A cold tip with no peer heights at all is the one case
+  // we must not call healthy: there is simply nothing to compare against.
+  if (tipIsCold && knownPeerHeights.length === 0) {
+    return { status: 'unknown', nodeHeight, networkHeight, lagBlocks, tipAgeSeconds }
+  }
+
+  return { status: 'ok', nodeHeight, networkHeight, lagBlocks, tipAgeSeconds }
+}