diff --git a/.changeset/wild-mangos-observe.md b/.changeset/wild-mangos-observe.md new file mode 100644 index 000000000..20d498932 --- /dev/null +++ b/.changeset/wild-mangos-observe.md @@ -0,0 +1,5 @@ +--- +"@helium/monitor-service": patch +--- + +Add dc-auto-top liveness and DCA-input gauges to monitor-service. `solana_auto_top_off_task_trigger{name,leg}` reports the trigger time of the tuktuk task each top-off leg points at, or 0 when none is scheduled, so `time() - value` detects a leg that has stopped rescheduling itself whatever the cause. The auto-top-off USDC balances and the USDC/USD pyth feed publish time are now exported too, covering the two inputs the HNT leg's DCA depends on. diff --git a/packages/monitor-service/package.json b/packages/monitor-service/package.json index a0f9e5fbe..96c166aa5 100644 --- a/packages/monitor-service/package.json +++ b/packages/monitor-service/package.json @@ -31,6 +31,7 @@ "@helium/account-fetch-cache": "workspace:^", "@helium/circuit-breaker-sdk": "workspace:^", "@helium/data-credits-sdk": "workspace:^", + "@helium/dc-auto-top-sdk": "workspace:^", "@helium/helium-entity-manager-sdk": "workspace:^", "@helium/helium-sub-daos-sdk": "workspace:^", "@helium/idls": "workspace:^", @@ -38,6 +39,7 @@ "@helium/lazy-transactions-sdk": "workspace:^", "@helium/price-oracle-sdk": "workspace:^", "@helium/spl-utils": "workspace:^", + "@helium/tuktuk-sdk": "^0.0.9", "@metaplex-foundation/mpl-bubblegum": "^0.7.0", "@pythnetwork/pyth-solana-receiver": "^0.10.2", "@metaplex-foundation/mpl-token-metadata": "^2.10.0", diff --git a/packages/monitor-service/src/index.ts b/packages/monitor-service/src/index.ts index 7d45d4e04..cfb80137a 100644 --- a/packages/monitor-service/src/index.ts +++ b/packages/monitor-service/src/index.ts @@ -23,6 +23,7 @@ import { underscore } from "inflection"; import { HNT_MINT, IOT_MINT, MOBILE_MINT } from "./env"; import { register, totalRewardsGauge } from "./metrics"; import { Recipient, sequelize } from "./model"; +import { monitorAutoTopOff } from "./monitors/autoTopOff"; import { monitiorAssociatedTokenBalance, monitorSolBalance, @@ -235,6 +236,27 @@ async function run() { ); } + // The DCA leg buys its HNT with the USDC held here, transferring a whole run's worth + // up front. A shortfall reverts the run rather than shrinking it, which stops the leg + // rescheduling itself, so this balance is watched alongside the HNT it buys. + await monitiorAssociatedTokenBalance( + carrierAutoTopOff, + USDC_MINT, + "carrier_auto_topoff", + false, + "usdc" + ); + await monitiorAssociatedTokenBalance( + mobileAutoTopOff, + USDC_MINT, + "helium_mobile_auto_topoff", + false, + "usdc" + ); + + await monitorAutoTopOff(carrierAutoTopOff, "carrier_auto_topoff"); + await monitorAutoTopOff(mobileAutoTopOff, "helium_mobile_auto_topoff"); + for (const maker of makers) { await monitorSolBalance( maker.account.issuingAuthority, @@ -295,6 +317,15 @@ async function run() { new PublicKey(process.env.PYTH_HNT_FEED || HNT_PYTH_PRICE_FEED), "hnt" ); + // dc-auto-top's DCA leg prices its USDC input from this feed and refuses to start a + // run when it is over 5 minutes old. + await monitorPythFreshness( + new PublicKey( + process.env.PYTH_USDC_FEED || + "6HAuqASbHEh4w4REJEUUUCginTLfj1kwCh215ZLtMkrT" + ), + "usdc" + ); await monitorSolBalance( new PublicKey( process.env.MIGRATION_KEY || "mgrArTL62g582wWV6iM4fwU1LKnbUikDN6akKJ76pzK" diff --git a/packages/monitor-service/src/metrics.ts b/packages/monitor-service/src/metrics.ts index 6920be3cc..5313a6850 100644 --- a/packages/monitor-service/src/metrics.ts +++ b/packages/monitor-service/src/metrics.ts @@ -98,3 +98,14 @@ export const pythPublishTime = new client.Gauge({ labelNames: ["name", "address"], }); register.registerMetric(pythPublishTime); + +export const autoTopOffTaskTrigger = new client.Gauge({ + name: "solana_auto_top_off_task_trigger", + help: + "Unix trigger time of the tuktuk task this auto top off leg currently points at, " + + "or 0 when no task is scheduled. A leg is stalled when time() - this exceeds one " + + "cron interval: a healthy leg always points at a future trigger, because each run " + + "reschedules itself, and 0 makes the same expression catch a swept task.", + labelNames: ["name", "leg", "address"], +}); +register.registerMetric(autoTopOffTaskTrigger); diff --git a/packages/monitor-service/src/monitors/autoTopOff.ts b/packages/monitor-service/src/monitors/autoTopOff.ts new file mode 100644 index 000000000..77f7a99b2 --- /dev/null +++ b/packages/monitor-service/src/monitors/autoTopOff.ts @@ -0,0 +1,82 @@ +import { init as initDcAutoTop } from "@helium/dc-auto-top-sdk"; +import { init as initTuktuk } from "@helium/tuktuk-sdk"; +import { PublicKey } from "@solana/web3.js"; +import { autoTopOffTaskTrigger } from "../metrics"; +import { provider } from "../solana"; +import { watch } from "./watch"; + +type Leg = "dc" | "hnt"; + +/** + * Whether a task account is this leg's own. + * + * tuktuk reuses task ids, so once a leg's task is swept another program's task takes the + * same address and the stored pointer still resolves. The trigger it carries is then + * somebody else's, and a stalled leg reads as a healthy one. dc-auto-top stamps each task + * it queues with `topoff `, so the description is what + * distinguishes its own task from a squatter at the same id. + */ +function ownsTask(description: string, leg: Leg, address: string): boolean { + // The program truncates the key to fit tuktuk's description limit: 32 - "topoff dc ".len + // for the DC leg and one char less for the longer "topoff hnt " prefix. + const keep = leg === "dc" ? 32 - 14 : 32 - 15; + return description === `topoff ${leg} ${address.slice(0, keep)}`; +} + +/** + * Tracks whether each leg of a `dc-auto-top` account is still being cranked. + * + * Both legs are self-rescheduling: a run reschedules itself as its last step, so a + * healthy leg always points at a task whose trigger is in the future. Every way a leg + * can break — a stale or wrong-owner price oracle, an empty DCA input account, a + * `dca` PDA left over from an undrained run, lamports below the crank reward, a + * reverting swap — ends the same way, with no reschedule. tuktuk then retries the + * task until it goes stale and is swept, after which nothing restarts the leg without + * `schedule_task_v0`. So the trigger time of the task each leg points at is the single + * signal that covers all of them. + */ +export async function monitorAutoTopOff(autoTopOff: PublicKey, label: string) { + const dcAutoTopProgram = await initDcAutoTop(provider); + const tuktukProgram = await initTuktuk(provider); + const address = autoTopOff.toBase58(); + + async function publish(leg: Leg, task: PublicKey) { + // Three ways a leg has no task of its own: the program parks its own key in the + // field as a "nothing scheduled" sentinel, because a zero pubkey cannot be passed as + // a mutable account; a task that kept failing is swept once stale; and a swept task's + // id gets reused, leaving the pointer resolving to somebody else's task. All three + // mean the leg is dead until something reschedules it, and all three report 0. + const acc = task.equals(autoTopOff) + ? null + : await tuktukProgram.account.taskV0.fetchNullable(task); + if (!acc || !ownsTask(acc.description, leg, address)) { + autoTopOffTaskTrigger.set({ name: label, leg, address }, 0); + return; + } + + // A `now` trigger carries no time of its own, so it is overdue from when it was + // queued; `timestamp` holds its i64 in an unnamed field, decoded as index 0. + const trigger = acc.trigger as any; + const seconds = trigger.timestamp + ? trigger.timestamp[0].toNumber() + : acc.queuedAt.toNumber(); + autoTopOffTaskTrigger.set({ name: label, leg, address }, seconds); + } + + watch(autoTopOff, async (raw) => { + if (!raw) return; + try { + const acc = dcAutoTopProgram.coder.accounts.decode( + "autoTopOffV0", + raw.data + ); + await publish("dc", acc.nextTask); + await publish("hnt", acc.nextHntTask); + } catch (e) { + // Leave the previous reading in place rather than zeroing it: an RPC failure is + // not a stalled leg, and reporting one as the other would page on every blip. + // `watch` re-runs this every 5 minutes, so a real stall is still picked up. + console.error(`autoTopOff monitor failed for ${label} (${address})`, e); + } + }); +} diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 7f163d30b..22413197f 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -666,19 +666,19 @@ importers: version: 1.13.6(@opentelemetry/api@1.9.0) '@orpc/json-schema': specifier: 1.13.6 - version: 1.13.6(@opentelemetry/api@1.9.0)(crossws@0.3.5)(ws@8.18.0(bufferutil@4.0.7)(utf-8-validate@5.0.10)) + version: 1.13.6(@opentelemetry/api@1.9.0)(crossws@0.3.5)(ws@8.18.2(bufferutil@4.0.7)(utf-8-validate@5.0.10)) '@orpc/openapi': specifier: 1.13.6 - version: 1.13.6(@opentelemetry/api@1.9.0)(crossws@0.3.5)(ws@8.18.0(bufferutil@4.0.7)(utf-8-validate@5.0.10)) + version: 1.13.6(@opentelemetry/api@1.9.0)(crossws@0.3.5)(ws@8.18.2(bufferutil@4.0.7)(utf-8-validate@5.0.10)) '@orpc/server': specifier: 1.13.6 - version: 1.13.6(@opentelemetry/api@1.9.0)(crossws@0.3.5)(ws@8.18.0(bufferutil@4.0.7)(utf-8-validate@5.0.10)) + version: 1.13.6(@opentelemetry/api@1.9.0)(crossws@0.3.5)(ws@8.18.2(bufferutil@4.0.7)(utf-8-validate@5.0.10)) '@orpc/tanstack-query': specifier: 1.13.6 version: 1.13.6(@opentelemetry/api@1.9.0)(@orpc/client@1.13.6(@opentelemetry/api@1.9.0))(@tanstack/query-core@5.90.20) '@orpc/zod': specifier: 1.13.6 - version: 1.13.6(@opentelemetry/api@1.9.0)(@orpc/contract@1.13.6(@opentelemetry/api@1.9.0))(@orpc/server@1.13.6(@opentelemetry/api@1.9.0)(crossws@0.3.5)(ws@8.18.0(bufferutil@4.0.7)(utf-8-validate@5.0.10)))(crossws@0.3.5)(ws@8.18.0(bufferutil@4.0.7)(utf-8-validate@5.0.10))(zod@4.3.6) + version: 1.13.6(@opentelemetry/api@1.9.0)(@orpc/contract@1.13.6(@opentelemetry/api@1.9.0))(@orpc/server@1.13.6(@opentelemetry/api@1.9.0)(crossws@0.3.5)(ws@8.18.2(bufferutil@4.0.7)(utf-8-validate@5.0.10)))(crossws@0.3.5)(ws@8.18.2(bufferutil@4.0.7)(utf-8-validate@5.0.10))(zod@4.3.6) '@privy-io/react-auth': specifier: ^2.20.0 version: 2.25.0(@react-native-async-storage/async-storage@1.19.3(react-native@0.72.17(@babel/core@7.29.0)(@babel/preset-env@7.29.2(@babel/core@7.29.0))(bufferutil@4.0.7)(encoding@0.1.13)(react@19.2.6)(utf-8-validate@5.0.10)))(@solana/kit@5.5.1(bufferutil@4.0.7)(fastestsmallesttextencoderdecoder@1.0.22)(typescript@5.9.3)(utf-8-validate@5.0.10))(@solana/spl-token@0.4.14(@solana/web3.js@1.98.2(bufferutil@4.0.7)(encoding@0.1.13)(typescript@5.9.3)(utf-8-validate@5.0.10))(bufferutil@4.0.7)(encoding@0.1.13)(fastestsmallesttextencoderdecoder@1.0.22)(typescript@5.9.3)(utf-8-validate@5.0.10))(@solana/web3.js@1.98.2(bufferutil@4.0.7)(encoding@0.1.13)(typescript@5.9.3)(utf-8-validate@5.0.10))(@types/react@19.2.14)(bs58@6.0.0)(bufferutil@4.0.7)(fastestsmallesttextencoderdecoder@1.0.22)(react-dom@19.2.6(react@19.2.6))(react@19.2.6)(typescript@5.9.3)(use-sync-external-store@1.6.0(react@19.2.6))(utf-8-validate@5.0.10)(zod@4.3.6) @@ -2542,6 +2542,9 @@ importers: '@helium/data-credits-sdk': specifier: workspace:^ version: link:../data-credits-sdk + '@helium/dc-auto-top-sdk': + specifier: workspace:^ + version: link:../dc-auto-top-sdk '@helium/helium-entity-manager-sdk': specifier: workspace:^ version: link:../helium-entity-manager-sdk @@ -2563,6 +2566,9 @@ importers: '@helium/spl-utils': specifier: workspace:^ version: link:../spl-utils + '@helium/tuktuk-sdk': + specifier: ^0.0.9 + version: 0.0.9(bufferutil@4.0.7)(encoding@0.1.13)(fastestsmallesttextencoderdecoder@1.0.22)(typescript@5.9.3)(utf-8-validate@5.0.10) '@metaplex-foundation/mpl-bubblegum': specifier: ^0.7.0 version: 0.7.0(bufferutil@4.0.7)(encoding@0.1.13)(fastestsmallesttextencoderdecoder@1.0.22)(typescript@5.9.3)(utf-8-validate@5.0.10) @@ -18676,12 +18682,12 @@ snapshots: '@orpc/interop@1.13.6': {} - '@orpc/json-schema@1.13.6(@opentelemetry/api@1.9.0)(crossws@0.3.5)(ws@8.18.0(bufferutil@4.0.7)(utf-8-validate@5.0.10))': + '@orpc/json-schema@1.13.6(@opentelemetry/api@1.9.0)(crossws@0.3.5)(ws@8.18.2(bufferutil@4.0.7)(utf-8-validate@5.0.10))': dependencies: '@orpc/contract': 1.13.6(@opentelemetry/api@1.9.0) '@orpc/interop': 1.13.6 - '@orpc/openapi': 1.13.6(@opentelemetry/api@1.9.0)(crossws@0.3.5)(ws@8.18.0(bufferutil@4.0.7)(utf-8-validate@5.0.10)) - '@orpc/server': 1.13.6(@opentelemetry/api@1.9.0)(crossws@0.3.5)(ws@8.18.0(bufferutil@4.0.7)(utf-8-validate@5.0.10)) + '@orpc/openapi': 1.13.6(@opentelemetry/api@1.9.0)(crossws@0.3.5)(ws@8.18.2(bufferutil@4.0.7)(utf-8-validate@5.0.10)) + '@orpc/server': 1.13.6(@opentelemetry/api@1.9.0)(crossws@0.3.5)(ws@8.18.2(bufferutil@4.0.7)(utf-8-validate@5.0.10)) '@orpc/shared': 1.13.6(@opentelemetry/api@1.9.0) json-schema-typed: 8.0.2 transitivePeerDependencies: @@ -18699,13 +18705,13 @@ snapshots: transitivePeerDependencies: - '@opentelemetry/api' - '@orpc/openapi@1.13.6(@opentelemetry/api@1.9.0)(crossws@0.3.5)(ws@8.18.0(bufferutil@4.0.7)(utf-8-validate@5.0.10))': + '@orpc/openapi@1.13.6(@opentelemetry/api@1.9.0)(crossws@0.3.5)(ws@8.18.2(bufferutil@4.0.7)(utf-8-validate@5.0.10))': dependencies: '@orpc/client': 1.13.6(@opentelemetry/api@1.9.0) '@orpc/contract': 1.13.6(@opentelemetry/api@1.9.0) '@orpc/interop': 1.13.6 '@orpc/openapi-client': 1.13.6(@opentelemetry/api@1.9.0) - '@orpc/server': 1.13.6(@opentelemetry/api@1.9.0)(crossws@0.3.5)(ws@8.18.0(bufferutil@4.0.7)(utf-8-validate@5.0.10)) + '@orpc/server': 1.13.6(@opentelemetry/api@1.9.0)(crossws@0.3.5)(ws@8.18.2(bufferutil@4.0.7)(utf-8-validate@5.0.10)) '@orpc/shared': 1.13.6(@opentelemetry/api@1.9.0) '@orpc/standard-server': 1.13.6(@opentelemetry/api@1.9.0) json-schema-typed: 8.0.2 @@ -18716,7 +18722,7 @@ snapshots: - fastify - ws - '@orpc/server@1.13.6(@opentelemetry/api@1.9.0)(crossws@0.3.5)(ws@8.18.0(bufferutil@4.0.7)(utf-8-validate@5.0.10))': + '@orpc/server@1.13.6(@opentelemetry/api@1.9.0)(crossws@0.3.5)(ws@8.18.2(bufferutil@4.0.7)(utf-8-validate@5.0.10))': dependencies: '@orpc/client': 1.13.6(@opentelemetry/api@1.9.0) '@orpc/contract': 1.13.6(@opentelemetry/api@1.9.0) @@ -18731,7 +18737,7 @@ snapshots: cookie: 1.1.1 optionalDependencies: crossws: 0.3.5 - ws: 8.18.0(bufferutil@4.0.7)(utf-8-validate@5.0.10) + ws: 8.18.2(bufferutil@4.0.7)(utf-8-validate@5.0.10) transitivePeerDependencies: - '@opentelemetry/api' - fastify @@ -18796,12 +18802,12 @@ snapshots: transitivePeerDependencies: - '@opentelemetry/api' - '@orpc/zod@1.13.6(@opentelemetry/api@1.9.0)(@orpc/contract@1.13.6(@opentelemetry/api@1.9.0))(@orpc/server@1.13.6(@opentelemetry/api@1.9.0)(crossws@0.3.5)(ws@8.18.0(bufferutil@4.0.7)(utf-8-validate@5.0.10)))(crossws@0.3.5)(ws@8.18.0(bufferutil@4.0.7)(utf-8-validate@5.0.10))(zod@4.3.6)': + '@orpc/zod@1.13.6(@opentelemetry/api@1.9.0)(@orpc/contract@1.13.6(@opentelemetry/api@1.9.0))(@orpc/server@1.13.6(@opentelemetry/api@1.9.0)(crossws@0.3.5)(ws@8.18.2(bufferutil@4.0.7)(utf-8-validate@5.0.10)))(crossws@0.3.5)(ws@8.18.2(bufferutil@4.0.7)(utf-8-validate@5.0.10))(zod@4.3.6)': dependencies: '@orpc/contract': 1.13.6(@opentelemetry/api@1.9.0) - '@orpc/json-schema': 1.13.6(@opentelemetry/api@1.9.0)(crossws@0.3.5)(ws@8.18.0(bufferutil@4.0.7)(utf-8-validate@5.0.10)) - '@orpc/openapi': 1.13.6(@opentelemetry/api@1.9.0)(crossws@0.3.5)(ws@8.18.0(bufferutil@4.0.7)(utf-8-validate@5.0.10)) - '@orpc/server': 1.13.6(@opentelemetry/api@1.9.0)(crossws@0.3.5)(ws@8.18.0(bufferutil@4.0.7)(utf-8-validate@5.0.10)) + '@orpc/json-schema': 1.13.6(@opentelemetry/api@1.9.0)(crossws@0.3.5)(ws@8.18.2(bufferutil@4.0.7)(utf-8-validate@5.0.10)) + '@orpc/openapi': 1.13.6(@opentelemetry/api@1.9.0)(crossws@0.3.5)(ws@8.18.2(bufferutil@4.0.7)(utf-8-validate@5.0.10)) + '@orpc/server': 1.13.6(@opentelemetry/api@1.9.0)(crossws@0.3.5)(ws@8.18.2(bufferutil@4.0.7)(utf-8-validate@5.0.10)) '@orpc/shared': 1.13.6(@opentelemetry/api@1.9.0) escape-string-regexp: 5.0.0 wildcard-match: 5.1.4