fix(homelab): make memory and ZFS alerts durable

packages/docs/guides/2026-04-04_homelab-audit-runbook.md

@@ -115,6 +115,33 @@ toolkit gf query 'zfs_zpool_capacity_used_ratio'           # Pool utilization
 toolkit gf query 'node_zfs_arc_hits / (node_zfs_arc_hits + node_zfs_arc_misses)'   # ARC hit rate
 ```
 
+### ZFS maintenance workflow
+
+The weekly `zfs-maintenance-weekly` Temporal schedule discovers the managed
+`zfspv-pool-*` pools on each node's `zfs-zpool-collector` pod. Do not use
+`kubectl exec daemonset/...` for verification: the DaemonSet spans nodes with
+different pool inventories.
+
+```bash
+temporal schedule describe --schedule-id zfs-maintenance-weekly
+temporal workflow list --query "WorkflowType='runZfsMaintenanceWorkflow'"
+kubectl -n prometheus get pods -l app=zfs-zpool-collector -o wide
+toolkit gf query 'zfs_zpool_last_scrub_completion_timestamp'
+```
+
+For a failed or overdue run, select the Ready collector pod for the relevant
+node and inspect each pool it reports:
+
+```bash
+temporal workflow describe --workflow-id <WORKFLOW_ID>
+kubectl -n prometheus exec <COLLECTOR_POD> -c zfs-zpool-collector -- zpool list -H -o name
+kubectl -n prometheus exec <COLLECTOR_POD> -c zfs-zpool-collector -- zpool status <POOL>
+```
+
+An `ONLINE` pool with no known data errors is healthy but can still need a
+scrub. A zero `zfs_zpool_last_scrub_completion_timestamp` means no completed
+scrub has been recorded and is intentionally alertable.
+
 ### Velero Backups
 
 ```bash

packages/docs/plans/2026-08-08_memory-and-zfs-alert-remediation.md

@@ -0,0 +1,45 @@
+---
+id: plan-2026-08-08-memory-and-zfs-alert-remediation
+type: plan
+status: in-progress
+board: false
+---
+
+# Durable memory and ZFS alert remediation
+
+## Context
+
+The `MemoryLeakSuspected` PagerDuty incident on `liskov` was a false positive:
+the PromQL `offset 24h` modifier applied only to the historical ZFS ARC selector,
+so a large ARC drop looked like non-ARC memory growth. The corrected expression
+must offset total memory, available memory, and ARC together.
+
+The weekly `runZfsMaintenanceWorkflow` failed on 2026-08-02 after selecting the
+`liskov` `zfs-zpool-collector` pod through `kubectl exec daemonset/...`. The
+activity assumed that pod also contained `zfspv-pool-hdd`, but that pool exists
+only on `torvalds`; the failure occurred before either node reached the scrub
+loop. The maintenance activity now discovers one Ready collector pod per node
+and only operates on that node's managed `zfspv-pool-*` inventory.
+
+## Changes
+
+- Correct and regression-test the memory alert expression.
+- Make ZFS maintenance pod- and pool-aware, with fail-fast errors carrying node,
+  pod, pool, and command context.
+- Treat a zero scrub timestamp as an alert condition so never-scrubbed pools are
+  visible instead of silently excluded.
+- Document per-node verification and the Temporal workflow failure mode in the
+  homelab runbook.
+
+## Verification
+
+- Focused Temporal and CDK8s tests and lint pass; CDK8s typecheck passes.
+- The Temporal package-wide typecheck remains blocked by pre-existing missing
+  `@shepherdjerred/glitter-context/schema` and `@shepherdjerred/llm-models`
+  workspace artifacts plus dependent Glitter errors; no changed-file errors
+  were reported.
+- Rendered Prometheus rules contain the corrected memory and ZFS expressions.
+- After the GitOps rollout, run the weekly workflow once, verify each node's
+  managed pools have current scrub timestamps and no ZFS errors, then wait for
+  Prometheus/Alertmanager to clear the incidents. This operator step is not part
+  of the repository-only implementation.

packages/homelab/src/cdk8s/src/resources/monitoring/monitoring/rules/resource-monitoring.test.ts

@@ -91,3 +91,26 @@ describe("crypto-mining alerts", () => {
     expect(alert.labels?.["severity"]).toBe("critical");
   });
 });
+
+describe("memory leak alerts", () => {
+  it("applies the 24-hour offset to every historical memory and ARC selector", () => {
+    const groups = getResourceMonitoringRuleGroups();
+    const memoryGroup = groups.find(
+      (group) => group.name === "resource-memory-monitoring",
+    );
+    if (memoryGroup?.rules === undefined) {
+      throw new Error("expected resource-memory-monitoring rules");
+    }
+
+    const alert = memoryGroup.rules.find(
+      (rule) => rule.alert === "MemoryLeakSuspected",
+    );
+    if (alert === undefined) {
+      throw new Error("expected MemoryLeakSuspected alert");
+    }
+
+    expect(alert.expr.value).toBe(
+      "((node_memory_MemTotal_bytes - node_memory_MemAvailable_bytes) - on(instance) group_left node_zfs_arc_size) - ((node_memory_MemTotal_bytes offset 24h - node_memory_MemAvailable_bytes offset 24h) - on(instance) group_left node_zfs_arc_size offset 24h) > 8589934592",
+    );
+  });
+});

packages/homelab/src/cdk8s/src/resources/monitoring/monitoring/rules/resource-monitoring.ts

@@ -90,7 +90,7 @@ export function getResourceMonitoringRuleGroups(): PrometheusRuleSpecGroups[] {
             summary: "Potential memory leak detected",
           },
           expr: PrometheusRuleSpecGroupsRulesExpr.fromString(
-            "((node_memory_MemTotal_bytes - node_memory_MemAvailable_bytes) - on(instance) group_left node_zfs_arc_size) - ((node_memory_MemTotal_bytes - node_memory_MemAvailable_bytes) - on(instance) group_left node_zfs_arc_size offset 24h) > 8589934592", // 8GB increase over 24h, excluding ZFS ARC cache
+            "((node_memory_MemTotal_bytes - node_memory_MemAvailable_bytes) - on(instance) group_left node_zfs_arc_size) - ((node_memory_MemTotal_bytes offset 24h - node_memory_MemAvailable_bytes offset 24h) - on(instance) group_left node_zfs_arc_size offset 24h) > 8589934592", // 8GB increase over 24h, excluding ZFS ARC cache
           ),
           for: "4h",
           labels: { severity: "warning" },

packages/homelab/src/cdk8s/src/resources/monitoring/monitoring/rules/zfs-maintenance.test.ts

@@ -0,0 +1,22 @@
+import { describe, expect, it } from "bun:test";
+import { getZfsMaintenanceRuleGroups } from "./zfs-maintenance.ts";
+
+describe("ZfsScrubOverdue", () => {
+  it("fires for pools that have never recorded a completed scrub or are overdue", () => {
+    const group = getZfsMaintenanceRuleGroups().find(
+      (candidate) => candidate.name === "zfs-maintenance",
+    );
+    if (group?.rules === undefined) {
+      throw new Error("expected zfs-maintenance rules");
+    }
+
+    const alert = group.rules.find((rule) => rule.alert === "ZfsScrubOverdue");
+    if (alert === undefined) {
+      throw new Error("expected ZfsScrubOverdue alert");
+    }
+
+    expect(alert.expr.value).toBe(
+      "zfs_zpool_last_scrub_completion_timestamp == 0 or (time() - zfs_zpool_last_scrub_completion_timestamp) > 777600",
+    );
+  });
+});

packages/homelab/src/cdk8s/src/resources/monitoring/monitoring/rules/zfs-maintenance.ts

@@ -56,11 +56,11 @@ export function getZfsMaintenanceRuleGroups(): PrometheusRuleSpecGroups[] {
               "ZFS scrub overdue on {{ $labels.zpool_name }}",
             ),
             description: escapePrometheusTemplate(
-              "ZFS pool {{ $labels.zpool_name }} has not been scrubbed in over 9 days. The weekly Temporal maintenance workflow may have failed.",
+              "ZFS pool {{ $labels.zpool_name }} has never completed a scrub or has not been scrubbed in over 9 days. The weekly Temporal maintenance workflow may have failed.",
             ),
           },
           expr: PrometheusRuleSpecGroupsRulesExpr.fromString(
-            "zfs_zpool_last_scrub_completion_timestamp > 0 and (time() - zfs_zpool_last_scrub_completion_timestamp) > 777600",
+            "zfs_zpool_last_scrub_completion_timestamp == 0 or (time() - zfs_zpool_last_scrub_completion_timestamp) > 777600",
           ),
           for: "1h",
           labels: {

packages/homelab/src/cdk8s/src/resources/temporal/worker.ts

@@ -122,10 +122,9 @@ function createTemporalWorkerMaintenanceRbac(
   chart: Chart,
   serviceAccount: ServiceAccount,
 ) {
-  // Namespace-scoped RBAC for the ZFS maintenance workflow, which execs into
-  // the zfs-zpool-collector DaemonSet pod in the prometheus namespace.
-  // `kubectl exec daemonset/<name>` resolves the daemonset → pod via a
-  // GET on daemonsets.apps before opening the exec stream.
+  // Namespace-scoped RBAC for the ZFS maintenance workflow, which lists the
+  // zfs-zpool-collector pods and execs into one Running and Ready pod per node
+  // in the prometheus namespace.
   new KubeRole(chart, "temporal-worker-zfs-exec", {
     metadata: { name: "temporal-worker-zfs-exec", namespace: "prometheus" },
     rules: [
@@ -139,11 +138,6 @@ function createTemporalWorkerMaintenanceRbac(
         resources: ["pods"],
         verbs: ["get", "list"],
       },
-      {
-        apiGroups: ["apps"],
-        resources: ["daemonsets"],
-        verbs: ["get"],
-      },
     ],
   });
 

packages/temporal/src/activities/bugsink.ts

@@ -1,5 +1,6 @@
 import { Context } from "@temporalio/activity";
 import * as k8s from "@kubernetes/client-node";
+import { kubectlExecInPod } from "#shared/kubectl-exec.ts";
 
 const NAMESPACE = "bugsink";
 const CONTAINER = "bugsink";
@@ -35,7 +36,12 @@ export const bugsinkHousekeepingActivities = {
       // heartbeatTimeout: "90 seconds" in workflows/bugsink.ts. SDK
       // throttles transmission to 80% of timeout automatically.
       Context.current().heartbeat({ command: command.join(" ") });
-      const out = await kubectlExec(podName, command);
+      const out = await kubectlExecInPod({
+        namespace: NAMESPACE,
+        container: CONTAINER,
+        pod: podName,
+        command,
+      });
       const trimmed = out.trim();
       results.push(`${command.join(" ")}: ${trimmed === "" ? "ok" : trimmed}`);
     }
@@ -61,36 +67,3 @@ async function findRunningBugsinkPod(): Promise<string> {
 // ErrorEvent objects under Bun (the library targets Node's `ws` shim, which
 // Bun doesn't replicate exactly). Pod discovery via the HTTP API still works
 // and is kept above.
-async function kubectlExec(
-  podName: string,
-  command: string[],
-): Promise<string> {
-  const args = [
-    "kubectl",
-    "exec",
-    "--namespace",
-    NAMESPACE,
-    "--container",
-    CONTAINER,
-    podName,
-    "--",
-    ...command,
-  ];
-  const proc = Bun.spawn(args, {
-    stdin: "ignore",
-    stdout: "pipe",
-    stderr: "pipe",
-  });
-  const [stdout, stderr, exitCode] = await Promise.all([
-    new Response(proc.stdout).text(),
-    new Response(proc.stderr).text(),
-    proc.exited,
-  ]);
-  if (exitCode !== 0) {
-    const detail = stderr.trim() || stdout.trim() || "(no output)";
-    throw new Error(
-      `kubectl exec [${command.join(" ")}] in ${NAMESPACE}/${podName} exited ${String(exitCode)}: ${detail}`,
-    );
-  }
-  return stdout;
-}

packages/temporal/src/activities/velero-orphan-audit.ts

@@ -11,6 +11,12 @@ import {
   veleroOrphanLocalSnapshotsTotal,
   zfsDatasetSnapshotCount,
 } from "#observability/metrics.ts";
+import {
+  selectRunningReadyNodePods,
+  type KubernetesNodePod,
+  type KubernetesNodePodCandidate,
+} from "#shared/kubernetes-node-pods.ts";
+import { kubectlExecInPod } from "#shared/kubectl-exec.ts";
 
 // The audit detects ZFS snapshots on PVC datasets whose name does not match
 // any live `velero.io/v1/Backup` CR. Such snapshots are orphans from a prior
@@ -39,26 +45,7 @@ export type VeleroOrphanDataset = {
   liveCount: number;
 };
 
-type ZfsNodePod = {
-  node: string;
-  pod: string;
-};
-
-type ZfsNodePodCandidate = {
-  metadata?: {
-    name?: string;
-  };
-  spec?: {
-    nodeName?: string;
-  };
-  status?: {
-    phase?: string;
-    conditions?: {
-      type: string;
-      status: string;
-    }[];
-  };
-};
+type ZfsNodePod = KubernetesNodePod;
 
 export type VeleroOrphanAuditResult = {
   liveBackupCount: number;
@@ -191,39 +178,13 @@ async function findZfsNodePods(): Promise<ZfsNodePod[]> {
 }
 
 export function selectZfsNodePods(
-  pods: readonly ZfsNodePodCandidate[],
+  pods: readonly KubernetesNodePodCandidate[],
 ): ZfsNodePod[] {
-  const nodePods = new Map<string, string>();
-  for (const pod of pods) {
-    const isReady = pod.status?.conditions?.some(
-      (condition) => condition.type === "Ready" && condition.status === "True",
-    );
-    if (isReady !== true || pod.status?.phase !== "Running") {
-      continue;
-    }
-    const name = pod.metadata?.name;
-    const node = pod.spec?.nodeName;
-    if (name === undefined || node === undefined) {
-      throw new Error(
-        "Running and Ready openebs-zfs-localpv-node pod is missing metadata.name or spec.nodeName",
-      );
-    }
-    const existingPod = nodePods.get(node);
-    if (existingPod !== undefined) {
-      throw new Error(
-        `Multiple Running and Ready openebs-zfs-localpv-node pods found for node ${node}: ${existingPod}, ${name}`,
-      );
-    }
-    nodePods.set(node, name);
-  }
-  if (nodePods.size === 0) {
-    throw new Error(
-      `No Running and Ready openebs-zfs-localpv-node pods found in ${NAMESPACE_OPENEBS} (label selector: ${ZFS_NODE_LABEL})`,
-    );
-  }
-  return [...nodePods.entries()]
-    .map(([node, pod]) => ({ node, pod }))
-    .toSorted((left, right) => left.node.localeCompare(right.node));
+  return selectRunningReadyNodePods(pods, {
+    namespace: NAMESPACE_OPENEBS,
+    labelSelector: ZFS_NODE_LABEL,
+    resourceDescription: "openebs-zfs-localpv-node",
+  });
 }
 
 async function listZfsOrphanSnapshots(
@@ -372,34 +333,10 @@ export function parseZfsInventory(
 }
 
 async function execInPod(podName: string, command: string): Promise<string> {
-  const args = [
-    "kubectl",
-    "exec",
-    "-n",
-    NAMESPACE_OPENEBS,
-    "-c",
-    ZFS_NODE_CONTAINER,
-    podName,
-    "--",
-    "sh",
-    "-c",
+  return kubectlExecInPod({
+    namespace: NAMESPACE_OPENEBS,
+    container: ZFS_NODE_CONTAINER,
+    pod: podName,
     command,
-  ];
-  const proc = Bun.spawn(args, {
-    stdin: "ignore",
-    stdout: "pipe",
-    stderr: "pipe",
   });
-  const [stdout, stderr, exitCode] = await Promise.all([
-    new Response(proc.stdout).text(),
-    new Response(proc.stderr).text(),
-    proc.exited,
-  ]);
-  if (exitCode !== 0) {
-    const detail = stderr.trim() || stdout.trim() || "(no output)";
-    throw new Error(
-      `kubectl exec [${command}] in ${NAMESPACE_OPENEBS}/${podName} exited ${String(exitCode)}: ${detail}`,
-    );
-  }
-  return stdout;
 }