diff --git a/scripts/collector-daemon.js b/scripts/collector-daemon.js index c0a4aed5..5abf87c9 100644 --- a/scripts/collector-daemon.js +++ b/scripts/collector-daemon.js @@ -3,7 +3,8 @@ const fs = require('fs'); const path = require('path'); const { pathToFileURL } = require('url'); -const CACHE_DIR = path.join(process.env.HOME || process.env.USERPROFILE || '', '.loongsuite-pilot'); +const HOME = process.env.HOME || process.env.USERPROFILE || ''; +const CACHE_DIR = path.join(HOME, '.loongsuite-pilot'); const CURRENT_FILE = path.join(CACHE_DIR, 'current'); const PREVIOUS_FILE = path.join(CACHE_DIR, 'previous'); const VERSIONS_DIR = path.join(CACHE_DIR, 'versions'); @@ -18,12 +19,60 @@ function loadVersion(pointerFile) { return null; } +// Resolve the data dir the updater will read the breadcrumb from (honors override). +function resolveDataDir() { + const raw = process.env.LOONGSUITE_PILOT_DATA_DIR; + if (!raw) return CACHE_DIR; + if (raw === '~') return HOME; + if (raw.startsWith('~/')) return path.join(HOME, raw.slice(2)); + return raw; +} + +function resolveInstalledVersion() { + try { + const name = fs.readFileSync(CURRENT_FILE, 'utf-8').trim(); + const content = fs.readFileSync(path.join(VERSIONS_DIR, name, 'VERSION'), 'utf-8'); + const match = content.match(/^version=(.+)$/m); + return match ? match[1] : (name || 'unknown'); + } catch { + return 'unknown'; + } +} + +// Fatal early-death happens during ESM module-graph resolution (e.g. a top-level +// `import sqlite3` failing under npm12), before dist/index.js main() runs. This is +// the only place that can capture the real cause for the updater to report. +function writeStartupCrash(err) { + try { + const breadcrumb = { + schema: 1, + ts: Math.floor(Date.now() / 1000), + phase: 'module_load', + version: resolveInstalledVersion(), + pid: process.pid, + error_message: err && err.message ? String(err.message) : String(err), + error_stack_head: err && err.stack + ? String(err.stack).split(/\r?\n/).slice(0, 10).join('\n') + : '', + }; + const dir = path.join(resolveDataDir(), 'logs'); + fs.mkdirSync(dir, { recursive: true }); + const file = path.join(dir, 'last-startup-crash.json'); + const tmp = file + '.' + process.pid + '.' + Date.now() + '.tmp'; + fs.writeFileSync(tmp, JSON.stringify(breadcrumb, null, 2) + '\n', 'utf8'); + fs.renameSync(tmp, file); + } catch { + // best-effort: never mask the original error + } +} + const entry = loadVersion(CURRENT_FILE) || loadVersion(PREVIOUS_FILE); if (!entry) { console.error('[loongsuite-pilot] No valid collector version found'); process.exit(1); } import(pathToFileURL(entry).href).catch(err => { + writeStartupCrash(err); console.error('[loongsuite-pilot] Failed to load collector:', err.message); process.exit(1); }); diff --git a/src/index.ts b/src/index.ts index 8009b6be..188eaf77 100644 --- a/src/index.ts +++ b/src/index.ts @@ -3,7 +3,8 @@ import * as path from 'path'; import { Orchestrator } from './core/orchestrator.js'; import { loadConfig } from './core/config-loader.js'; import { createLogger, initFileLogging } from './utils/logger.js'; -import { resolveHome } from './utils/fs-utils.js'; +import { resolveHome, readInstalledVersion } from './utils/fs-utils.js'; +import { writeStartupCrash, clearStartupCrash, resolveBreadcrumbDataDir } from './utils/crash-breadcrumb.js'; import { handleWorkerCli } from './local-workers/worker-cli.js'; const logger = createLogger('Main'); @@ -23,10 +24,14 @@ async function main(): Promise { const config = await loadConfig(); - const logDir = path.join(resolveHome(config.dataDir), 'logs'); + const dataDir = resolveHome(config.dataDir); + const logDir = path.join(dataDir, 'logs'); await initFileLogging(path.join(logDir, 'loongsuite-pilot-service.log')); if (!config.enabled) { + // A deliberate, non-crash exit: drop any stale breadcrumb so it is not later + // misread as this run's failure cause. + clearStartupCrash(resolveBreadcrumbDataDir()); logger.info('analytics disabled via config or LOONGSUITE_PILOT_ENABLED=false'); return; } @@ -43,6 +48,11 @@ async function main(): Promise { await orchestrator.start(); + // Reached a healthy running state: clear any stale crash breadcrumb so a lingering + // one always reflects the most recent *failed* startup attempt. The breadcrumb dir + // must match the daemon writer and the updater reader (env-or-default), not config.dataDir. + clearStartupCrash(resolveBreadcrumbDataDir()); + logger.info('AI Agent Input is running', { dataDir: config.dataDir, flushers: Object.entries(config.flushers) @@ -53,6 +63,13 @@ async function main(): Promise { main().catch((err) => { logger.error('fatal startup error', { error: String(err) }); + const breadcrumbDir = resolveBreadcrumbDataDir(); + writeStartupCrash({ + dataDir: breadcrumbDir, + phase: 'startup', + version: readInstalledVersion(breadcrumbDir), + error: err, + }); process.exit(1); }); diff --git a/src/updater/startup-crash-classifier.ts b/src/updater/startup-crash-classifier.ts new file mode 100644 index 00000000..d9fff240 --- /dev/null +++ b/src/updater/startup-crash-classifier.ts @@ -0,0 +1,77 @@ +import type { StartupCrashBreadcrumb } from '../utils/crash-breadcrumb.js'; + +export type StartupCrashReason = + | 'native_module_missing' + | 'module_not_found' + | 'config_error' + | 'permission_or_disk' + | 'unknown'; + +export interface StartupCrashClassification { + reason: StartupCrashReason; + detailHead: string; +} + +const DETAIL_MAX_CHARS = 300; + +/** + * Maps a raw crash breadcrumb to a stable `reason` label (for aggregation/alarming) + * plus a human-readable detail head. Rules are evaluated in order; the first match + * wins, and `unknown` always carries the raw message so nothing is lost. + */ +export function classifyStartupCrash(breadcrumb: StartupCrashBreadcrumb): StartupCrashClassification { + const message = (breadcrumb.error_message || '').toLowerCase(); + const full = `${breadcrumb.error_message}\n${breadcrumb.error_stack_head}`.toLowerCase(); + return { + reason: detectReason(message, full, breadcrumb.phase), + detailHead: sanitizeDetail(firstLine(breadcrumb.error_message)), + }; +} + +function detectReason(message: string, full: string, phase: string): StartupCrashReason { + if ( + full.includes('sqlite3') + || full.includes('err_dlopen_failed') + || full.includes('did not self-register') + || /cannot find module\s+['"][^'"]*\.node['"]/.test(full) + || full.includes('install scripts') + || full.includes('node_module_version') + || full.includes('compiled against a different node') + ) { + return 'native_module_missing'; + } + if (full.includes('cannot find module')) { + return 'module_not_found'; + } + // Check permission/disk before config so a startup-phase EACCES whose message merely + // mentions "config" is not mislabeled as a configuration error. + if (full.includes('eacces') || full.includes('erofs') || full.includes('enospc')) { + return 'permission_or_disk'; + } + // config_error is intentionally narrow: only a JSON-parse signature in the error + // *message* (not the stack, which routinely contains config-loader.ts paths) during + // the startup phase. Bare "config"/"json" substrings are too broad. + if ( + phase === 'startup' + && ( + message.includes('unexpected token') + || message.includes('unexpected end of json') + || message.includes(' in json') + || message.includes('not valid json') + || message.includes('json.parse') + ) + ) { + return 'config_error'; + } + return 'unknown'; +} + +function firstLine(text: string): string { + return (text || '').split(/\r?\n/)[0] ?? ''; +} + +// Keep the detail safe to embed in `detail="..."` inside the alarm message: no quotes +// or control chars that could break downstream parsing/readability. +function sanitizeDetail(text: string): string { + return text.replace(/["\r\n\t]+/g, ' ').replace(/\s+/g, ' ').trim().slice(0, DETAIL_MAX_CHARS); +} diff --git a/src/updater/updater-metrics.ts b/src/updater/updater-metrics.ts index b3bf54d3..d27a1365 100644 --- a/src/updater/updater-metrics.ts +++ b/src/updater/updater-metrics.ts @@ -6,6 +6,8 @@ import { resolveLocalIp } from '../utils/network-utils.js'; import { flattenToStrings } from '../utils/record-utils.js'; import { checkProcessLiveness, COLLECTOR_PROCESS_PATTERNS } from '../utils/pid-utils.js'; import type { ProcessLiveness } from '../utils/pid-utils.js'; +import { readStartupCrash } from '../utils/crash-breadcrumb.js'; +import { classifyStartupCrash } from './startup-crash-classifier.js'; import { sendAlarm, sendStatus } from '../internal/sender.js'; import type { AlarmLevel, AlarmType, AlarmEntry } from '../metrics/alarm-manager.js'; @@ -50,6 +52,7 @@ export interface UpdaterMetricsOptions { export class UpdaterMetrics { private readonly logsDir: string; + private readonly dataDir: string; private readonly version: string; private readonly ip: string; private readonly userId: string; @@ -66,6 +69,7 @@ export class UpdaterMetrics { constructor(opts: UpdaterMetricsOptions) { this.logsDir = path.join(opts.dataDir, 'logs', 'metric_alarm'); + this.dataDir = opts.dataDir; this.version = opts.version; this.collectorPidFile = opts.collectorPidFile; this.collectorLiveness = opts.collectorLiveness @@ -203,8 +207,18 @@ export class UpdaterMetrics { this.lastCollectorAlarmAt = now; this.writeAlarm( 'SERVICE_NOT_RUNNING_ALARM', '3', - `loongsuite-pilot collector process is not running after ${this.collectorConsecutiveFailures} checks: ${liveness.reason}`, + this.buildNotRunningMessage(liveness.reason), ); } + + // Enrich the not-running alarm (after #133 debounce has confirmed absence) with the + // real startup-failure cause the dying collector recorded — message only, no schema change. + private buildNotRunningMessage(livenessReason: string): string { + const base = `loongsuite-pilot collector process is not running after ${this.collectorConsecutiveFailures} checks: ${livenessReason}`; + const breadcrumb = readStartupCrash(this.dataDir); + if (!breadcrumb) return base; + const { reason, detailHead } = classifyStartupCrash(breadcrumb); + return `${base} | cause=${reason} detail="${detailHead}" phase=${breadcrumb.phase} version=${breadcrumb.version}`; + } } diff --git a/src/utils/crash-breadcrumb.ts b/src/utils/crash-breadcrumb.ts new file mode 100644 index 00000000..c8645ffd --- /dev/null +++ b/src/utils/crash-breadcrumb.ts @@ -0,0 +1,97 @@ +import * as fs from 'node:fs'; +import * as path from 'node:path'; +import { resolveHome } from './fs-utils.js'; + +export type StartupCrashPhase = 'module_load' | 'startup' | 'runtime'; + +export interface StartupCrashBreadcrumb { + schema: 1; + ts: number; + phase: StartupCrashPhase; + version: string; + pid: number; + error_message: string; + error_stack_head: string; +} + +const FILE_NAME = 'last-startup-crash.json'; +const STACK_HEAD_MAX_LINES = 10; +const STACK_HEAD_MAX_CHARS = 4000; + +export function startupCrashPath(dataDir: string): string { + return path.join(dataDir, 'logs', FILE_NAME); +} + +/** + * The single data dir the breadcrumb lives in. It MUST match where the updater (the + * reader, src/updater/index.ts) and the bootstrap (scripts/collector-daemon.js, the + * earliest writer) look — both use env-or-default, NOT config.dataDir. Aligning the + * writer, clearer and reader on this one directory is what makes the "lingering + * breadcrumb = most recent failed startup" invariant hold even when config.json + * overrides dataDir. + */ +export function resolveBreadcrumbDataDir(): string { + return resolveHome(process.env.LOONGSUITE_PILOT_DATA_DIR ?? '~/.loongsuite-pilot'); +} + +function truncateStackHead(stack: string | undefined): string { + if (!stack) return ''; + const head = stack.split(/\r?\n/).slice(0, STACK_HEAD_MAX_LINES).join('\n'); + return head.length > STACK_HEAD_MAX_CHARS ? head.slice(0, STACK_HEAD_MAX_CHARS) : head; +} + +/** + * Persists the cause of an abnormal collector exit as a breadcrumb the updater can + * read later. Synchronous and best-effort: a write failure must never mask the + * original error or alter the exit code. + */ +export function writeStartupCrash(opts: { + dataDir: string; + phase: StartupCrashPhase; + version: string; + error: unknown; +}): void { + try { + const { error } = opts; + const breadcrumb: StartupCrashBreadcrumb = { + schema: 1, + ts: Math.floor(Date.now() / 1000), + phase: opts.phase, + version: opts.version || 'unknown', + pid: process.pid, + error_message: error instanceof Error ? error.message : String(error), + error_stack_head: truncateStackHead(error instanceof Error ? error.stack : undefined), + }; + const file = startupCrashPath(opts.dataDir); + fs.mkdirSync(path.dirname(file), { recursive: true }); + const tmp = `${file}.${process.pid}.${Date.now()}.tmp`; + fs.writeFileSync(tmp, `${JSON.stringify(breadcrumb, null, 2)}\n`, 'utf8'); + fs.renameSync(tmp, file); + } catch { + // best-effort + } +} + +/** + * Removes the breadcrumb after a healthy startup so a lingering file always means + * the most recent startup attempt failed (PID/version-independent correlation). + */ +export function clearStartupCrash(dataDir: string): void { + try { + fs.rmSync(startupCrashPath(dataDir), { force: true }); + } catch { + // ignore + } +} + +/** + * Reads the last-startup-crash breadcrumb, or null when absent/unreadable/unknown schema. + */ +export function readStartupCrash(dataDir: string): StartupCrashBreadcrumb | null { + try { + const parsed = JSON.parse(fs.readFileSync(startupCrashPath(dataDir), 'utf8')) as StartupCrashBreadcrumb; + return parsed && parsed.schema === 1 ? parsed : null; + } catch { + return null; + } +} diff --git a/tests/unit/updater/startup-crash-classifier.test.ts b/tests/unit/updater/startup-crash-classifier.test.ts new file mode 100644 index 00000000..01df2282 --- /dev/null +++ b/tests/unit/updater/startup-crash-classifier.test.ts @@ -0,0 +1,77 @@ +import { describe, it, expect } from 'vitest'; +import { classifyStartupCrash } from '../../../src/updater/startup-crash-classifier.js'; +import type { StartupCrashBreadcrumb, StartupCrashPhase } from '../../../src/utils/crash-breadcrumb.js'; + +function bc( + error_message: string, + opts: { phase?: StartupCrashPhase; stack?: string } = {}, +): StartupCrashBreadcrumb { + return { + schema: 1, + ts: 0, + phase: opts.phase ?? 'module_load', + version: '1.0.0', + pid: 1, + error_message, + error_stack_head: opts.stack ?? '', + }; +} + +describe('classifyStartupCrash', () => { + it('maps sqlite3 / native addon failures to native_module_missing', () => { + expect(classifyStartupCrash(bc('Error: Cannot open sqlite3 binding')).reason).toBe('native_module_missing'); + expect(classifyStartupCrash(bc('ERR_DLOPEN_FAILED: dlopen failed')).reason).toBe('native_module_missing'); + expect(classifyStartupCrash(bc("Module did not self-register")).reason).toBe('native_module_missing'); + expect(classifyStartupCrash(bc("Cannot find module '/x/build/Release/node_sqlite3.node'")).reason).toBe('native_module_missing'); + expect(classifyStartupCrash(bc('The module was compiled against a different Node.js version using NODE_MODULE_VERSION 108')).reason).toBe('native_module_missing'); + expect(classifyStartupCrash(bc('npm error install scripts were not run')).reason).toBe('native_module_missing'); + }); + + it('maps a plain missing module to module_not_found', () => { + expect(classifyStartupCrash(bc("Cannot find module 'lodash'")).reason).toBe('module_not_found'); + }); + + it('maps startup-phase JSON-parse errors to config_error', () => { + expect(classifyStartupCrash(bc('Unexpected token } in JSON at position 5', { phase: 'startup' })).reason).toBe('config_error'); + expect(classifyStartupCrash(bc('Unexpected end of JSON input', { phase: 'startup' })).reason).toBe('config_error'); + }); + + it('does not treat JSON errors as config_error outside startup phase', () => { + expect(classifyStartupCrash(bc('Unexpected token in JSON', { phase: 'module_load' })).reason).toBe('unknown'); + }); + + it('does not label a bare "config" mention (no JSON signature) as config_error', () => { + // narrowed: bare "config" is no longer a trigger, avoids false positives + expect(classifyStartupCrash(bc('failed to load config file', { phase: 'startup' })).reason).toBe('unknown'); + }); + + it('does not mislabel config_error from a stack path containing config-loader.ts', () => { + const res = classifyStartupCrash(bc('some novel failure', { + phase: 'startup', + stack: 'Error: some novel failure\n at loadConfig (/app/src/core/config-loader.ts:208:11)', + })); + expect(res.reason).toBe('unknown'); + }); + + it('maps permission / disk errors', () => { + expect(classifyStartupCrash(bc('EACCES: permission denied')).reason).toBe('permission_or_disk'); + expect(classifyStartupCrash(bc('ENOSPC: no space left on device')).reason).toBe('permission_or_disk'); + }); + + it('prefers permission_or_disk over config_error when both could match', () => { + // startup-phase EACCES whose message also mentions JSON/config must stay permission_or_disk + expect(classifyStartupCrash(bc('EACCES: permission denied, open config.json', { phase: 'startup' })).reason).toBe('permission_or_disk'); + }); + + it('falls back to unknown and carries a sanitized raw message head', () => { + const res = classifyStartupCrash(bc('some totally novel failure\nsecond line')); + expect(res.reason).toBe('unknown'); + expect(res.detailHead).toBe('some totally novel failure'); + }); + + it('sanitizes quotes/whitespace in the detail head', () => { + const res = classifyStartupCrash(bc('boom "quoted" spaced')); + expect(res.detailHead).not.toContain('"'); + expect(res.detailHead).toBe('boom quoted spaced'); + }); +}); diff --git a/tests/unit/updater/updater-metrics.test.ts b/tests/unit/updater/updater-metrics.test.ts index a5e35162..ee75e98c 100644 --- a/tests/unit/updater/updater-metrics.test.ts +++ b/tests/unit/updater/updater-metrics.test.ts @@ -274,6 +274,33 @@ describe('UpdaterMetrics', () => { await m.stop(); }); + it('enriches SERVICE_NOT_RUNNING_ALARM with the startup-crash cause when a breadcrumb exists', async () => { + mockReadFileSync.mockImplementation((p: string) => { + if (p.includes('last-startup-crash.json')) { + return JSON.stringify({ + schema: 1, ts: 1, phase: 'module_load', version: '9.9.9', pid: 1, + error_message: 'Cannot open sqlite3 binding', + error_stack_head: 'Error: Cannot open sqlite3 binding', + }); + } + return '12345\n'; + }); + const m = createMetrics('test-user', () => down('no matching collector command found')); + + await m.start(); + await vi.advanceTimersByTimeAsync(3 * 60_000 + 60_000); + await vi.advanceTimersByTimeAsync(60_000); + + const alarmLines = appendedLines.filter(l => l.path.includes('pilot-alarms.jsonl')); + expect(alarmLines).toHaveLength(1); + const parsed = JSON.parse(alarmLines[0].line); + expect(parsed.alarm_type).toBe('SERVICE_NOT_RUNNING_ALARM'); + expect(parsed.alarm_message).toContain('cause=native_module_missing'); + expect(parsed.alarm_message).toContain('version=9.9.9'); + expect(parsed.alarm_message).toContain('no matching collector command found'); + await m.stop(); + }); + it('does not alarm when collector PID changes but command identity is found', async () => { const m = createMetrics('test-user', () => ({ running: true, diff --git a/tests/unit/utils/crash-breadcrumb.test.ts b/tests/unit/utils/crash-breadcrumb.test.ts new file mode 100644 index 00000000..e45ea67c --- /dev/null +++ b/tests/unit/utils/crash-breadcrumb.test.ts @@ -0,0 +1,71 @@ +import { describe, it, expect, beforeEach, afterEach } from 'vitest'; +import * as fs from 'node:fs'; +import * as os from 'node:os'; +import * as path from 'node:path'; +import { + writeStartupCrash, + clearStartupCrash, + readStartupCrash, + startupCrashPath, +} from '../../../src/utils/crash-breadcrumb.js'; + +let dataDir: string; + +beforeEach(() => { + dataDir = fs.mkdtempSync(path.join(os.tmpdir(), 'lsp-crash-')); +}); + +afterEach(() => { + fs.rmSync(dataDir, { recursive: true, force: true }); +}); + +describe('crash-breadcrumb', () => { + it('writes a breadcrumb from an Error and reads it back', () => { + writeStartupCrash({ + dataDir, + phase: 'module_load', + version: '1.2.3', + error: new Error('Cannot open sqlite3 .node'), + }); + + expect(fs.existsSync(startupCrashPath(dataDir))).toBe(true); + const bc = readStartupCrash(dataDir); + expect(bc).not.toBeNull(); + expect(bc!.schema).toBe(1); + expect(bc!.phase).toBe('module_load'); + expect(bc!.version).toBe('1.2.3'); + expect(bc!.error_message).toBe('Cannot open sqlite3 .node'); + expect(bc!.error_stack_head).toContain('Error: Cannot open sqlite3 .node'); + expect(typeof bc!.ts).toBe('number'); + expect(bc!.pid).toBe(process.pid); + }); + + it('accepts a non-Error value and empties the stack head', () => { + writeStartupCrash({ dataDir, phase: 'startup', version: '', error: 'boom' }); + const bc = readStartupCrash(dataDir); + expect(bc!.error_message).toBe('boom'); + expect(bc!.error_stack_head).toBe(''); + expect(bc!.version).toBe('unknown'); + }); + + it('clears the breadcrumb and tolerates a missing file', () => { + writeStartupCrash({ dataDir, phase: 'startup', version: '1', error: new Error('x') }); + clearStartupCrash(dataDir); + expect(readStartupCrash(dataDir)).toBeNull(); + // second clear on an absent file must not throw + expect(() => clearStartupCrash(dataDir)).not.toThrow(); + }); + + it('returns null when no breadcrumb exists', () => { + expect(readStartupCrash(dataDir)).toBeNull(); + }); + + it('is best-effort and never throws on an unwritable data dir', () => { + // point at a path whose parent is a file, so mkdir/write fail internally + const fileAsDir = path.join(dataDir, 'not-a-dir'); + fs.writeFileSync(fileAsDir, 'x'); + expect(() => + writeStartupCrash({ dataDir: fileAsDir, phase: 'runtime', version: '1', error: new Error('y') }), + ).not.toThrow(); + }); +});