feat(metrics): report startup crash cause in not-running alarm

scripts/collector-daemon.js

@@ -3,7 +3,8 @@ const fs = require('fs');
 const path = require('path');
 const { pathToFileURL } = require('url');
 
-const CACHE_DIR = path.join(process.env.HOME || process.env.USERPROFILE || '', '.loongsuite-pilot');
+const HOME = process.env.HOME || process.env.USERPROFILE || '';
+const CACHE_DIR = path.join(HOME, '.loongsuite-pilot');
 const CURRENT_FILE = path.join(CACHE_DIR, 'current');
 const PREVIOUS_FILE = path.join(CACHE_DIR, 'previous');
 const VERSIONS_DIR = path.join(CACHE_DIR, 'versions');
@@ -18,12 +19,60 @@ function loadVersion(pointerFile) {
   return null;
 }
 
+// Resolve the data dir the updater will read the breadcrumb from (honors override).
+function resolveDataDir() {
+  const raw = process.env.LOONGSUITE_PILOT_DATA_DIR;
+  if (!raw) return CACHE_DIR;
+  if (raw === '~') return HOME;
+  if (raw.startsWith('~/')) return path.join(HOME, raw.slice(2));
+  return raw;
+}
+
+function resolveInstalledVersion() {
+  try {
+    const name = fs.readFileSync(CURRENT_FILE, 'utf-8').trim();
+    const content = fs.readFileSync(path.join(VERSIONS_DIR, name, 'VERSION'), 'utf-8');
+    const match = content.match(/^version=(.+)$/m);
+    return match ? match[1] : (name || 'unknown');
+  } catch {
+    return 'unknown';
+  }
+}
+
+// Fatal early-death happens during ESM module-graph resolution (e.g. a top-level
+// `import sqlite3` failing under npm12), before dist/index.js main() runs. This is
+// the only place that can capture the real cause for the updater to report.
+function writeStartupCrash(err) {
+  try {
+    const breadcrumb = {
+      schema: 1,
+      ts: Math.floor(Date.now() / 1000),
+      phase: 'module_load',
+      version: resolveInstalledVersion(),
+      pid: process.pid,
+      error_message: err && err.message ? String(err.message) : String(err),
+      error_stack_head: err && err.stack
+        ? String(err.stack).split(/\r?\n/).slice(0, 10).join('\n')
+        : '',
+    };
+    const dir = path.join(resolveDataDir(), 'logs');
+    fs.mkdirSync(dir, { recursive: true });
+    const file = path.join(dir, 'last-startup-crash.json');
+    const tmp = file + '.' + process.pid + '.' + Date.now() + '.tmp';
+    fs.writeFileSync(tmp, JSON.stringify(breadcrumb, null, 2) + '\n', 'utf8');
+    fs.renameSync(tmp, file);
+  } catch {
+    // best-effort: never mask the original error
+  }
+}
+
 const entry = loadVersion(CURRENT_FILE) || loadVersion(PREVIOUS_FILE);
 if (!entry) {
   console.error('[loongsuite-pilot] No valid collector version found');
   process.exit(1);
 }
 import(pathToFileURL(entry).href).catch(err => {
+  writeStartupCrash(err);
   console.error('[loongsuite-pilot] Failed to load collector:', err.message);
   process.exit(1);
 });

src/index.ts

@@ -3,7 +3,8 @@ import * as path from 'path';
 import { Orchestrator } from './core/orchestrator.js';
 import { loadConfig } from './core/config-loader.js';
 import { createLogger, initFileLogging } from './utils/logger.js';
-import { resolveHome } from './utils/fs-utils.js';
+import { resolveHome, readInstalledVersion } from './utils/fs-utils.js';
+import { writeStartupCrash, clearStartupCrash, resolveBreadcrumbDataDir } from './utils/crash-breadcrumb.js';
 import { handleWorkerCli } from './local-workers/worker-cli.js';
 
 const logger = createLogger('Main');
@@ -23,10 +24,14 @@ async function main(): Promise<void> {
 
   const config = await loadConfig();
 
-  const logDir = path.join(resolveHome(config.dataDir), 'logs');
+  const dataDir = resolveHome(config.dataDir);
+  const logDir = path.join(dataDir, 'logs');
   await initFileLogging(path.join(logDir, 'loongsuite-pilot-service.log'));
 
   if (!config.enabled) {
+    // A deliberate, non-crash exit: drop any stale breadcrumb so it is not later
+    // misread as this run's failure cause.
+    clearStartupCrash(resolveBreadcrumbDataDir());
     logger.info('analytics disabled via config or LOONGSUITE_PILOT_ENABLED=false');
     return;
   }
@@ -43,6 +48,11 @@ async function main(): Promise<void> {
 
   await orchestrator.start();
 
+  // Reached a healthy running state: clear any stale crash breadcrumb so a lingering
+  // one always reflects the most recent *failed* startup attempt. The breadcrumb dir
+  // must match the daemon writer and the updater reader (env-or-default), not config.dataDir.
+  clearStartupCrash(resolveBreadcrumbDataDir());
+
   logger.info('AI Agent Input is running', {
     dataDir: config.dataDir,
     flushers: Object.entries(config.flushers)
@@ -53,6 +63,13 @@ async function main(): Promise<void> {
 
 main().catch((err) => {
   logger.error('fatal startup error', { error: String(err) });
+  const breadcrumbDir = resolveBreadcrumbDataDir();
+  writeStartupCrash({
+    dataDir: breadcrumbDir,
+    phase: 'startup',
+    version: readInstalledVersion(breadcrumbDir),
+    error: err,
+  });
   process.exit(1);
 });
 

src/updater/startup-crash-classifier.ts

@@ -0,0 +1,77 @@
+import type { StartupCrashBreadcrumb } from '../utils/crash-breadcrumb.js';
+
+export type StartupCrashReason =
+  | 'native_module_missing'
+  | 'module_not_found'
+  | 'config_error'
+  | 'permission_or_disk'
+  | 'unknown';
+
+export interface StartupCrashClassification {
+  reason: StartupCrashReason;
+  detailHead: string;
+}
+
+const DETAIL_MAX_CHARS = 300;
+
+/**
+ * Maps a raw crash breadcrumb to a stable `reason` label (for aggregation/alarming)
+ * plus a human-readable detail head. Rules are evaluated in order; the first match
+ * wins, and `unknown` always carries the raw message so nothing is lost.
+ */
+export function classifyStartupCrash(breadcrumb: StartupCrashBreadcrumb): StartupCrashClassification {
+  const message = (breadcrumb.error_message || '').toLowerCase();
+  const full = `${breadcrumb.error_message}\n${breadcrumb.error_stack_head}`.toLowerCase();
+  return {
+    reason: detectReason(message, full, breadcrumb.phase),
+    detailHead: sanitizeDetail(firstLine(breadcrumb.error_message)),
+  };
+}
+
+function detectReason(message: string, full: string, phase: string): StartupCrashReason {
+  if (
+    full.includes('sqlite3')
+    || full.includes('err_dlopen_failed')
+    || full.includes('did not self-register')
+    || /cannot find module\s+['"][^'"]*\.node['"]/.test(full)
+    || full.includes('install scripts')
+    || full.includes('node_module_version')
+    || full.includes('compiled against a different node')
+  ) {
+    return 'native_module_missing';
+  }
+  if (full.includes('cannot find module')) {
+    return 'module_not_found';
+  }
+  // Check permission/disk before config so a startup-phase EACCES whose message merely
+  // mentions "config" is not mislabeled as a configuration error.
+  if (full.includes('eacces') || full.includes('erofs') || full.includes('enospc')) {
+    return 'permission_or_disk';
+  }
+  // config_error is intentionally narrow: only a JSON-parse signature in the error
+  // *message* (not the stack, which routinely contains config-loader.ts paths) during
+  // the startup phase. Bare "config"/"json" substrings are too broad.
+  if (
+    phase === 'startup'
+    && (
+      message.includes('unexpected token')
+      || message.includes('unexpected end of json')
+      || message.includes(' in json')
+      || message.includes('not valid json')
+      || message.includes('json.parse')
+    )
+  ) {
+    return 'config_error';
+  }
+  return 'unknown';
+}
+
+function firstLine(text: string): string {
+  return (text || '').split(/\r?\n/)[0] ?? '';
+}
+
+// Keep the detail safe to embed in `detail="..."` inside the alarm message: no quotes
+// or control chars that could break downstream parsing/readability.
+function sanitizeDetail(text: string): string {
+  return text.replace(/["\r\n\t]+/g, ' ').replace(/\s+/g, ' ').trim().slice(0, DETAIL_MAX_CHARS);
+}

src/updater/updater-metrics.ts

@@ -6,6 +6,8 @@ import { resolveLocalIp } from '../utils/network-utils.js';
 import { flattenToStrings } from '../utils/record-utils.js';
 import { checkProcessLiveness, COLLECTOR_PROCESS_PATTERNS } from '../utils/pid-utils.js';
 import type { ProcessLiveness } from '../utils/pid-utils.js';
+import { readStartupCrash } from '../utils/crash-breadcrumb.js';
+import { classifyStartupCrash } from './startup-crash-classifier.js';
 import { sendAlarm, sendStatus } from '../internal/sender.js';
 import type { AlarmLevel, AlarmType, AlarmEntry } from '../metrics/alarm-manager.js';
 
@@ -50,6 +52,7 @@ export interface UpdaterMetricsOptions {
 
 export class UpdaterMetrics {
   private readonly logsDir: string;
+  private readonly dataDir: string;
   private readonly version: string;
   private readonly ip: string;
   private readonly userId: string;
@@ -66,6 +69,7 @@ export class UpdaterMetrics {
 
   constructor(opts: UpdaterMetricsOptions) {
     this.logsDir = path.join(opts.dataDir, 'logs', 'metric_alarm');
+    this.dataDir = opts.dataDir;
     this.version = opts.version;
     this.collectorPidFile = opts.collectorPidFile;
     this.collectorLiveness = opts.collectorLiveness
@@ -203,8 +207,18 @@ export class UpdaterMetrics {
     this.lastCollectorAlarmAt = now;
     this.writeAlarm(
       'SERVICE_NOT_RUNNING_ALARM', '3',
-      `loongsuite-pilot collector process is not running after ${this.collectorConsecutiveFailures} checks: ${liveness.reason}`,
+      this.buildNotRunningMessage(liveness.reason),
     );
   }
+
+  // Enrich the not-running alarm (after #133 debounce has confirmed absence) with the
+  // real startup-failure cause the dying collector recorded — message only, no schema change.
+  private buildNotRunningMessage(livenessReason: string): string {
+    const base = `loongsuite-pilot collector process is not running after ${this.collectorConsecutiveFailures} checks: ${livenessReason}`;
+    const breadcrumb = readStartupCrash(this.dataDir);
+    if (!breadcrumb) return base;
+    const { reason, detailHead } = classifyStartupCrash(breadcrumb);
+    return `${base} | cause=${reason} detail="${detailHead}" phase=${breadcrumb.phase} version=${breadcrumb.version}`;
+  }
 }
 

src/utils/crash-breadcrumb.ts

@@ -0,0 +1,97 @@
+import * as fs from 'node:fs';
+import * as path from 'node:path';
+import { resolveHome } from './fs-utils.js';
+
+export type StartupCrashPhase = 'module_load' | 'startup' | 'runtime';
+
+export interface StartupCrashBreadcrumb {
+  schema: 1;
+  ts: number;
+  phase: StartupCrashPhase;
+  version: string;
+  pid: number;
+  error_message: string;
+  error_stack_head: string;
+}
+
+const FILE_NAME = 'last-startup-crash.json';
+const STACK_HEAD_MAX_LINES = 10;
+const STACK_HEAD_MAX_CHARS = 4000;
+
+export function startupCrashPath(dataDir: string): string {
+  return path.join(dataDir, 'logs', FILE_NAME);
+}
+
+/**
+ * The single data dir the breadcrumb lives in. It MUST match where the updater (the
+ * reader, src/updater/index.ts) and the bootstrap (scripts/collector-daemon.js, the
+ * earliest writer) look — both use env-or-default, NOT config.dataDir. Aligning the
+ * writer, clearer and reader on this one directory is what makes the "lingering
+ * breadcrumb = most recent failed startup" invariant hold even when config.json
+ * overrides dataDir.
+ */
+export function resolveBreadcrumbDataDir(): string {
+  return resolveHome(process.env.LOONGSUITE_PILOT_DATA_DIR ?? '~/.loongsuite-pilot');
+}
+
+function truncateStackHead(stack: string | undefined): string {
+  if (!stack) return '';
+  const head = stack.split(/\r?\n/).slice(0, STACK_HEAD_MAX_LINES).join('\n');
+  return head.length > STACK_HEAD_MAX_CHARS ? head.slice(0, STACK_HEAD_MAX_CHARS) : head;
+}
+
+/**
+ * Persists the cause of an abnormal collector exit as a breadcrumb the updater can
+ * read later. Synchronous and best-effort: a write failure must never mask the
+ * original error or alter the exit code.
+ */
+export function writeStartupCrash(opts: {
+  dataDir: string;
+  phase: StartupCrashPhase;
+  version: string;
+  error: unknown;
+}): void {
+  try {
+    const { error } = opts;
+    const breadcrumb: StartupCrashBreadcrumb = {
+      schema: 1,
+      ts: Math.floor(Date.now() / 1000),
+      phase: opts.phase,
+      version: opts.version || 'unknown',
+      pid: process.pid,
+      error_message: error instanceof Error ? error.message : String(error),
+      error_stack_head: truncateStackHead(error instanceof Error ? error.stack : undefined),
+    };
+    const file = startupCrashPath(opts.dataDir);
+    fs.mkdirSync(path.dirname(file), { recursive: true });
+    const tmp = `${file}.${process.pid}.${Date.now()}.tmp`;
+    fs.writeFileSync(tmp, `${JSON.stringify(breadcrumb, null, 2)}\n`, 'utf8');
+    fs.renameSync(tmp, file);
+  } catch {
+    // best-effort
+  }
+}
+
+/**
+ * Removes the breadcrumb after a healthy startup so a lingering file always means
+ * the most recent startup attempt failed (PID/version-independent correlation).
+ */
+export function clearStartupCrash(dataDir: string): void {
+  try {
+    fs.rmSync(startupCrashPath(dataDir), { force: true });
+  } catch {
+    // ignore
+  }
+}
+
+/**
+ * Reads the last-startup-crash breadcrumb, or null when absent/unreadable/unknown schema.
+ */
+export function readStartupCrash(dataDir: string): StartupCrashBreadcrumb | null {
+  try {
+    const parsed = JSON.parse(fs.readFileSync(startupCrashPath(dataDir), 'utf8')) as StartupCrashBreadcrumb;
+    return parsed && parsed.schema === 1 ? parsed : null;
+  } catch {
+    return null;
+  }
+}