Export dc-auto-top leg liveness and DCA input gauges

.changeset/wild-mangos-observe.md

@@ -0,0 +1,5 @@
+---
+"@helium/monitor-service": patch
+---
+
+Add dc-auto-top liveness and DCA-input gauges to monitor-service. `solana_auto_top_off_task_trigger{name,leg}` reports the trigger time of the tuktuk task each top-off leg points at, or 0 when none is scheduled, so `time() - value` detects a leg that has stopped rescheduling itself whatever the cause. The auto-top-off USDC balances and the USDC/USD pyth feed publish time are now exported too, covering the two inputs the HNT leg's DCA depends on.

packages/monitor-service/package.json

@@ -31,13 +31,15 @@
     "@helium/account-fetch-cache": "workspace:^",
     "@helium/circuit-breaker-sdk": "workspace:^",
     "@helium/data-credits-sdk": "workspace:^",
+    "@helium/dc-auto-top-sdk": "workspace:^",
     "@helium/helium-entity-manager-sdk": "workspace:^",
     "@helium/helium-sub-daos-sdk": "workspace:^",
     "@helium/idls": "workspace:^",
     "@helium/lazy-distributor-sdk": "workspace:^",
     "@helium/lazy-transactions-sdk": "workspace:^",
     "@helium/price-oracle-sdk": "workspace:^",
     "@helium/spl-utils": "workspace:^",
+    "@helium/tuktuk-sdk": "^0.0.9",
     "@metaplex-foundation/mpl-bubblegum": "^0.7.0",
     "@pythnetwork/pyth-solana-receiver": "^0.10.2",
     "@metaplex-foundation/mpl-token-metadata": "^2.10.0",

packages/monitor-service/src/index.ts

@@ -23,6 +23,7 @@ import { underscore } from "inflection";
 import { HNT_MINT, IOT_MINT, MOBILE_MINT } from "./env";
 import { register, totalRewardsGauge } from "./metrics";
 import { Recipient, sequelize } from "./model";
+import { monitorAutoTopOff } from "./monitors/autoTopOff";
 import {
   monitiorAssociatedTokenBalance,
   monitorSolBalance,
@@ -235,6 +236,27 @@ async function run() {
     );
   }
 
+  // The DCA leg buys its HNT with the USDC held here, transferring a whole run's worth
+  // up front. A shortfall reverts the run rather than shrinking it, which stops the leg
+  // rescheduling itself, so this balance is watched alongside the HNT it buys.
+  await monitiorAssociatedTokenBalance(
+    carrierAutoTopOff,
+    USDC_MINT,
+    "carrier_auto_topoff",
+    false,
+    "usdc"
+  );
+  await monitiorAssociatedTokenBalance(
+    mobileAutoTopOff,
+    USDC_MINT,
+    "helium_mobile_auto_topoff",
+    false,
+    "usdc"
+  );
+
+  await monitorAutoTopOff(carrierAutoTopOff, "carrier_auto_topoff");
+  await monitorAutoTopOff(mobileAutoTopOff, "helium_mobile_auto_topoff");
+
   for (const maker of makers) {
     await monitorSolBalance(
       maker.account.issuingAuthority,
@@ -295,6 +317,15 @@ async function run() {
     new PublicKey(process.env.PYTH_HNT_FEED || HNT_PYTH_PRICE_FEED),
     "hnt"
   );
+  // dc-auto-top's DCA leg prices its USDC input from this feed and refuses to start a
+  // run when it is over 5 minutes old.
+  await monitorPythFreshness(
+    new PublicKey(
+      process.env.PYTH_USDC_FEED ||
+        "6HAuqASbHEh4w4REJEUUUCginTLfj1kwCh215ZLtMkrT"
+    ),
+    "usdc"
+  );
   await monitorSolBalance(
     new PublicKey(
       process.env.MIGRATION_KEY || "mgrArTL62g582wWV6iM4fwU1LKnbUikDN6akKJ76pzK"

packages/monitor-service/src/metrics.ts

@@ -98,3 +98,14 @@ export const pythPublishTime = new client.Gauge({
   labelNames: ["name", "address"],
 });
 register.registerMetric(pythPublishTime);
+
+export const autoTopOffTaskTrigger = new client.Gauge({
+  name: "solana_auto_top_off_task_trigger",
+  help:
+    "Unix trigger time of the tuktuk task this auto top off leg currently points at, " +
+    "or 0 when no task is scheduled. A leg is stalled when time() - this exceeds one " +
+    "cron interval: a healthy leg always points at a future trigger, because each run " +
+    "reschedules itself, and 0 makes the same expression catch a swept task.",
+  labelNames: ["name", "leg", "address"],
+});
+register.registerMetric(autoTopOffTaskTrigger);

packages/monitor-service/src/monitors/autoTopOff.ts

@@ -0,0 +1,82 @@
+import { init as initDcAutoTop } from "@helium/dc-auto-top-sdk";
+import { init as initTuktuk } from "@helium/tuktuk-sdk";
+import { PublicKey } from "@solana/web3.js";
+import { autoTopOffTaskTrigger } from "../metrics";
+import { provider } from "../solana";
+import { watch } from "./watch";
+
+type Leg = "dc" | "hnt";
+
+/**
+ * Whether a task account is this leg's own.
+ *
+ * tuktuk reuses task ids, so once a leg's task is swept another program's task takes the
+ * same address and the stored pointer still resolves. The trigger it carries is then
+ * somebody else's, and a stalled leg reads as a healthy one. dc-auto-top stamps each task
+ * it queues with `topoff <leg> <truncated auto top off key>`, so the description is what
+ * distinguishes its own task from a squatter at the same id.
+ */
+function ownsTask(description: string, leg: Leg, address: string): boolean {
+  // The program truncates the key to fit tuktuk's description limit: 32 - "topoff dc ".len
+  // for the DC leg and one char less for the longer "topoff hnt " prefix.
+  const keep = leg === "dc" ? 32 - 14 : 32 - 15;
+  return description === `topoff ${leg} ${address.slice(0, keep)}`;
+}
+
+/**
+ * Tracks whether each leg of a `dc-auto-top` account is still being cranked.
+ *
+ * Both legs are self-rescheduling: a run reschedules itself as its last step, so a
+ * healthy leg always points at a task whose trigger is in the future. Every way a leg
+ * can break — a stale or wrong-owner price oracle, an empty DCA input account, a
+ * `dca` PDA left over from an undrained run, lamports below the crank reward, a
+ * reverting swap — ends the same way, with no reschedule. tuktuk then retries the
+ * task until it goes stale and is swept, after which nothing restarts the leg without
+ * `schedule_task_v0`. So the trigger time of the task each leg points at is the single
+ * signal that covers all of them.
+ */
+export async function monitorAutoTopOff(autoTopOff: PublicKey, label: string) {
+  const dcAutoTopProgram = await initDcAutoTop(provider);
+  const tuktukProgram = await initTuktuk(provider);
+  const address = autoTopOff.toBase58();
+
+  async function publish(leg: Leg, task: PublicKey) {
+    // Three ways a leg has no task of its own: the program parks its own key in the
+    // field as a "nothing scheduled" sentinel, because a zero pubkey cannot be passed as
+    // a mutable account; a task that kept failing is swept once stale; and a swept task's
+    // id gets reused, leaving the pointer resolving to somebody else's task. All three
+    // mean the leg is dead until something reschedules it, and all three report 0.
+    const acc = task.equals(autoTopOff)
+      ? null
+      : await tuktukProgram.account.taskV0.fetchNullable(task);
+    if (!acc || !ownsTask(acc.description, leg, address)) {
+      autoTopOffTaskTrigger.set({ name: label, leg, address }, 0);
+      return;
+    }
+
+    // A `now` trigger carries no time of its own, so it is overdue from when it was
+    // queued; `timestamp` holds its i64 in an unnamed field, decoded as index 0.
+    const trigger = acc.trigger as any;
+    const seconds = trigger.timestamp
+      ? trigger.timestamp[0].toNumber()
+      : acc.queuedAt.toNumber();
+    autoTopOffTaskTrigger.set({ name: label, leg, address }, seconds);
+  }
+
+  watch(autoTopOff, async (raw) => {
+    if (!raw) return;
+    try {
+      const acc = dcAutoTopProgram.coder.accounts.decode(
+        "autoTopOffV0",
+        raw.data
+      );
+      await publish("dc", acc.nextTask);
+      await publish("hnt", acc.nextHntTask);
+    } catch (e) {
+      // Leave the previous reading in place rather than zeroing it: an RPC failure is
+      // not a stalled leg, and reporting one as the other would page on every blip.
+      // `watch` re-runs this every 5 minutes, so a real stall is still picked up.
+      console.error(`autoTopOff monitor failed for ${label} (${address})`, e);
+    }
+  });
+}