From 99fb89f794d602c8afeb4bd36d046acca9647dc3 Mon Sep 17 00:00:00 2001 From: Snssn <1502062504@qq.com> Date: Tue, 21 Jul 2026 09:52:14 +0800 Subject: [PATCH 1/2] feat(metrics): report startup crash cause in not-running alarm MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When the collector dies during startup (e.g. sqlite3 native module failing to load under npm12), the updater only reported a generic "collector process is not running", losing the real cause. Record the cause as a breadcrumb the updater can read and surface it in the alarm message: - collector-daemon.js: write last-startup-crash.json (phase=module_load) in the bootstrap import().catch — the only place that observes a fatal module-graph load failure before main() runs. - index.ts: write the breadcrumb (phase=startup) in main().catch and clear it after a healthy start, so a lingering breadcrumb always reflects the most recent failed startup (PID/version-independent, survives auto-update). - updater-metrics: after the existing #133 debounce confirms absence, classify the breadcrumb (native_module_missing / module_not_found / config_error / permission_or_disk / unknown) and append cause/detail/phase/version to the SERVICE_NOT_RUNNING_ALARM message. Message-only enrichment: no new alarm type and no AlarmEntry schema change. Complements #164 (validate sqlite3 before activation) by making the failure observable when it still happens. Adds unit tests for the breadcrumb writer, classifier, and alarm enrichment. --- scripts/collector-daemon.js | 51 ++++++++++- src/index.ts | 17 +++- src/updater/startup-crash-classifier.ts | 59 +++++++++++++ src/updater/updater-metrics.ts | 16 +++- src/utils/crash-breadcrumb.ts | 84 +++++++++++++++++++ .../updater/startup-crash-classifier.test.ts | 53 ++++++++++++ tests/unit/updater/updater-metrics.test.ts | 27 ++++++ tests/unit/utils/crash-breadcrumb.test.ts | 71 ++++++++++++++++ 8 files changed, 374 insertions(+), 4 deletions(-) create mode 100644 src/updater/startup-crash-classifier.ts create mode 100644 src/utils/crash-breadcrumb.ts create mode 100644 tests/unit/updater/startup-crash-classifier.test.ts create mode 100644 tests/unit/utils/crash-breadcrumb.test.ts diff --git a/scripts/collector-daemon.js b/scripts/collector-daemon.js index c0a4aed5..5abf87c9 100644 --- a/scripts/collector-daemon.js +++ b/scripts/collector-daemon.js @@ -3,7 +3,8 @@ const fs = require('fs'); const path = require('path'); const { pathToFileURL } = require('url'); -const CACHE_DIR = path.join(process.env.HOME || process.env.USERPROFILE || '', '.loongsuite-pilot'); +const HOME = process.env.HOME || process.env.USERPROFILE || ''; +const CACHE_DIR = path.join(HOME, '.loongsuite-pilot'); const CURRENT_FILE = path.join(CACHE_DIR, 'current'); const PREVIOUS_FILE = path.join(CACHE_DIR, 'previous'); const VERSIONS_DIR = path.join(CACHE_DIR, 'versions'); @@ -18,12 +19,60 @@ function loadVersion(pointerFile) { return null; } +// Resolve the data dir the updater will read the breadcrumb from (honors override). +function resolveDataDir() { + const raw = process.env.LOONGSUITE_PILOT_DATA_DIR; + if (!raw) return CACHE_DIR; + if (raw === '~') return HOME; + if (raw.startsWith('~/')) return path.join(HOME, raw.slice(2)); + return raw; +} + +function resolveInstalledVersion() { + try { + const name = fs.readFileSync(CURRENT_FILE, 'utf-8').trim(); + const content = fs.readFileSync(path.join(VERSIONS_DIR, name, 'VERSION'), 'utf-8'); + const match = content.match(/^version=(.+)$/m); + return match ? match[1] : (name || 'unknown'); + } catch { + return 'unknown'; + } +} + +// Fatal early-death happens during ESM module-graph resolution (e.g. a top-level +// `import sqlite3` failing under npm12), before dist/index.js main() runs. This is +// the only place that can capture the real cause for the updater to report. +function writeStartupCrash(err) { + try { + const breadcrumb = { + schema: 1, + ts: Math.floor(Date.now() / 1000), + phase: 'module_load', + version: resolveInstalledVersion(), + pid: process.pid, + error_message: err && err.message ? String(err.message) : String(err), + error_stack_head: err && err.stack + ? String(err.stack).split(/\r?\n/).slice(0, 10).join('\n') + : '', + }; + const dir = path.join(resolveDataDir(), 'logs'); + fs.mkdirSync(dir, { recursive: true }); + const file = path.join(dir, 'last-startup-crash.json'); + const tmp = file + '.' + process.pid + '.' + Date.now() + '.tmp'; + fs.writeFileSync(tmp, JSON.stringify(breadcrumb, null, 2) + '\n', 'utf8'); + fs.renameSync(tmp, file); + } catch { + // best-effort: never mask the original error + } +} + const entry = loadVersion(CURRENT_FILE) || loadVersion(PREVIOUS_FILE); if (!entry) { console.error('[loongsuite-pilot] No valid collector version found'); process.exit(1); } import(pathToFileURL(entry).href).catch(err => { + writeStartupCrash(err); console.error('[loongsuite-pilot] Failed to load collector:', err.message); process.exit(1); }); diff --git a/src/index.ts b/src/index.ts index 8009b6be..c49f1234 100644 --- a/src/index.ts +++ b/src/index.ts @@ -3,11 +3,15 @@ import * as path from 'path'; import { Orchestrator } from './core/orchestrator.js'; import { loadConfig } from './core/config-loader.js'; import { createLogger, initFileLogging } from './utils/logger.js'; -import { resolveHome } from './utils/fs-utils.js'; +import { resolveHome, readInstalledVersion } from './utils/fs-utils.js'; +import { writeStartupCrash, clearStartupCrash } from './utils/crash-breadcrumb.js'; import { handleWorkerCli } from './local-workers/worker-cli.js'; const logger = createLogger('Main'); +// Captured once config is loaded so the module-level catch can locate the data dir. +let resolvedDataDir: string | null = null; + async function main(): Promise { const argv = process.argv.slice(2); if (await handleWorkerCli(argv)) { @@ -23,7 +27,9 @@ async function main(): Promise { const config = await loadConfig(); - const logDir = path.join(resolveHome(config.dataDir), 'logs'); + const dataDir = resolveHome(config.dataDir); + resolvedDataDir = dataDir; + const logDir = path.join(dataDir, 'logs'); await initFileLogging(path.join(logDir, 'loongsuite-pilot-service.log')); if (!config.enabled) { @@ -43,6 +49,10 @@ async function main(): Promise { await orchestrator.start(); + // Reached a healthy running state: clear any stale crash breadcrumb so a lingering + // one always reflects the most recent *failed* startup attempt. + clearStartupCrash(dataDir); + logger.info('AI Agent Input is running', { dataDir: config.dataDir, flushers: Object.entries(config.flushers) @@ -53,6 +63,9 @@ async function main(): Promise { main().catch((err) => { logger.error('fatal startup error', { error: String(err) }); + const dataDir = resolvedDataDir + ?? resolveHome(process.env.LOONGSUITE_PILOT_DATA_DIR ?? '~/.loongsuite-pilot'); + writeStartupCrash({ dataDir, phase: 'startup', version: readInstalledVersion(dataDir), error: err }); process.exit(1); }); diff --git a/src/updater/startup-crash-classifier.ts b/src/updater/startup-crash-classifier.ts new file mode 100644 index 00000000..c0f917e1 --- /dev/null +++ b/src/updater/startup-crash-classifier.ts @@ -0,0 +1,59 @@ +import type { StartupCrashBreadcrumb } from '../utils/crash-breadcrumb.js'; + +export type StartupCrashReason = + | 'native_module_missing' + | 'module_not_found' + | 'config_error' + | 'permission_or_disk' + | 'unknown'; + +export interface StartupCrashClassification { + reason: StartupCrashReason; + detailHead: string; +} + +const DETAIL_MAX_CHARS = 300; + +/** + * Maps a raw crash breadcrumb to a stable `reason` label (for aggregation/alarming) + * plus a human-readable detail head. Rules are evaluated in order; the first match + * wins, and `unknown` always carries the raw message so nothing is lost. + */ +export function classifyStartupCrash(breadcrumb: StartupCrashBreadcrumb): StartupCrashClassification { + const haystack = `${breadcrumb.error_message}\n${breadcrumb.error_stack_head}`.toLowerCase(); + return { + reason: detectReason(haystack, breadcrumb.phase), + detailHead: firstLine(breadcrumb.error_message).slice(0, DETAIL_MAX_CHARS), + }; +} + +function detectReason(text: string, phase: string): StartupCrashReason { + if ( + text.includes('sqlite3') + || text.includes('err_dlopen_failed') + || text.includes('did not self-register') + || /cannot find module\s+['"][^'"]*\.node['"]/.test(text) + || text.includes('install scripts') + || text.includes('node_module_version') + || text.includes('compiled against a different node') + ) { + return 'native_module_missing'; + } + if (text.includes('cannot find module')) { + return 'module_not_found'; + } + if ( + phase === 'startup' + && (text.includes('json') || text.includes('unexpected token') || text.includes('config')) + ) { + return 'config_error'; + } + if (text.includes('eacces') || text.includes('erofs') || text.includes('enospc')) { + return 'permission_or_disk'; + } + return 'unknown'; +} + +function firstLine(text: string): string { + return (text || '').split(/\r?\n/)[0] ?? ''; +} diff --git a/src/updater/updater-metrics.ts b/src/updater/updater-metrics.ts index b3bf54d3..d27a1365 100644 --- a/src/updater/updater-metrics.ts +++ b/src/updater/updater-metrics.ts @@ -6,6 +6,8 @@ import { resolveLocalIp } from '../utils/network-utils.js'; import { flattenToStrings } from '../utils/record-utils.js'; import { checkProcessLiveness, COLLECTOR_PROCESS_PATTERNS } from '../utils/pid-utils.js'; import type { ProcessLiveness } from '../utils/pid-utils.js'; +import { readStartupCrash } from '../utils/crash-breadcrumb.js'; +import { classifyStartupCrash } from './startup-crash-classifier.js'; import { sendAlarm, sendStatus } from '../internal/sender.js'; import type { AlarmLevel, AlarmType, AlarmEntry } from '../metrics/alarm-manager.js'; @@ -50,6 +52,7 @@ export interface UpdaterMetricsOptions { export class UpdaterMetrics { private readonly logsDir: string; + private readonly dataDir: string; private readonly version: string; private readonly ip: string; private readonly userId: string; @@ -66,6 +69,7 @@ export class UpdaterMetrics { constructor(opts: UpdaterMetricsOptions) { this.logsDir = path.join(opts.dataDir, 'logs', 'metric_alarm'); + this.dataDir = opts.dataDir; this.version = opts.version; this.collectorPidFile = opts.collectorPidFile; this.collectorLiveness = opts.collectorLiveness @@ -203,8 +207,18 @@ export class UpdaterMetrics { this.lastCollectorAlarmAt = now; this.writeAlarm( 'SERVICE_NOT_RUNNING_ALARM', '3', - `loongsuite-pilot collector process is not running after ${this.collectorConsecutiveFailures} checks: ${liveness.reason}`, + this.buildNotRunningMessage(liveness.reason), ); } + + // Enrich the not-running alarm (after #133 debounce has confirmed absence) with the + // real startup-failure cause the dying collector recorded — message only, no schema change. + private buildNotRunningMessage(livenessReason: string): string { + const base = `loongsuite-pilot collector process is not running after ${this.collectorConsecutiveFailures} checks: ${livenessReason}`; + const breadcrumb = readStartupCrash(this.dataDir); + if (!breadcrumb) return base; + const { reason, detailHead } = classifyStartupCrash(breadcrumb); + return `${base} | cause=${reason} detail="${detailHead}" phase=${breadcrumb.phase} version=${breadcrumb.version}`; + } } diff --git a/src/utils/crash-breadcrumb.ts b/src/utils/crash-breadcrumb.ts new file mode 100644 index 00000000..e2d3e26b --- /dev/null +++ b/src/utils/crash-breadcrumb.ts @@ -0,0 +1,84 @@ +import * as fs from 'node:fs'; +import * as path from 'node:path'; + +export type StartupCrashPhase = 'module_load' | 'startup' | 'runtime'; + +export interface StartupCrashBreadcrumb { + schema: 1; + ts: number; + phase: StartupCrashPhase; + version: string; + pid: number; + error_message: string; + error_stack_head: string; +} + +const FILE_NAME = 'last-startup-crash.json'; +const STACK_HEAD_MAX_LINES = 10; +const STACK_HEAD_MAX_CHARS = 4000; + +export function startupCrashPath(dataDir: string): string { + return path.join(dataDir, 'logs', FILE_NAME); +} + +function truncateStackHead(stack: string | undefined): string { + if (!stack) return ''; + const head = stack.split(/\r?\n/).slice(0, STACK_HEAD_MAX_LINES).join('\n'); + return head.length > STACK_HEAD_MAX_CHARS ? head.slice(0, STACK_HEAD_MAX_CHARS) : head; +} + +/** + * Persists the cause of an abnormal collector exit as a breadcrumb the updater can + * read later. Synchronous and best-effort: a write failure must never mask the + * original error or alter the exit code. + */ +export function writeStartupCrash(opts: { + dataDir: string; + phase: StartupCrashPhase; + version: string; + error: unknown; +}): void { + try { + const { error } = opts; + const breadcrumb: StartupCrashBreadcrumb = { + schema: 1, + ts: Math.floor(Date.now() / 1000), + phase: opts.phase, + version: opts.version || 'unknown', + pid: process.pid, + error_message: error instanceof Error ? error.message : String(error), + error_stack_head: truncateStackHead(error instanceof Error ? error.stack : undefined), + }; + const file = startupCrashPath(opts.dataDir); + fs.mkdirSync(path.dirname(file), { recursive: true }); + const tmp = `${file}.${process.pid}.${Date.now()}.tmp`; + fs.writeFileSync(tmp, `${JSON.stringify(breadcrumb, null, 2)}\n`, 'utf8'); + fs.renameSync(tmp, file); + } catch { + // best-effort + } +} + +/** + * Removes the breadcrumb after a healthy startup so a lingering file always means + * the most recent startup attempt failed (PID/version-independent correlation). + */ +export function clearStartupCrash(dataDir: string): void { + try { + fs.rmSync(startupCrashPath(dataDir), { force: true }); + } catch { + // ignore + } +} + +/** + * Reads the last-startup-crash breadcrumb, or null when absent/unreadable/unknown schema. + */ +export function readStartupCrash(dataDir: string): StartupCrashBreadcrumb | null { + try { + const parsed = JSON.parse(fs.readFileSync(startupCrashPath(dataDir), 'utf8')) as StartupCrashBreadcrumb; + return parsed && parsed.schema === 1 ? parsed : null; + } catch { + return null; + } +} diff --git a/tests/unit/updater/startup-crash-classifier.test.ts b/tests/unit/updater/startup-crash-classifier.test.ts new file mode 100644 index 00000000..4dd99531 --- /dev/null +++ b/tests/unit/updater/startup-crash-classifier.test.ts @@ -0,0 +1,53 @@ +import { describe, it, expect } from 'vitest'; +import { classifyStartupCrash } from '../../../src/updater/startup-crash-classifier.js'; +import type { StartupCrashBreadcrumb, StartupCrashPhase } from '../../../src/utils/crash-breadcrumb.js'; + +function bc( + error_message: string, + opts: { phase?: StartupCrashPhase; stack?: string } = {}, +): StartupCrashBreadcrumb { + return { + schema: 1, + ts: 0, + phase: opts.phase ?? 'module_load', + version: '1.0.0', + pid: 1, + error_message, + error_stack_head: opts.stack ?? '', + }; +} + +describe('classifyStartupCrash', () => { + it('maps sqlite3 / native addon failures to native_module_missing', () => { + expect(classifyStartupCrash(bc('Error: Cannot open sqlite3 binding')).reason).toBe('native_module_missing'); + expect(classifyStartupCrash(bc('ERR_DLOPEN_FAILED: dlopen failed')).reason).toBe('native_module_missing'); + expect(classifyStartupCrash(bc("Module did not self-register")).reason).toBe('native_module_missing'); + expect(classifyStartupCrash(bc("Cannot find module '/x/build/Release/node_sqlite3.node'")).reason).toBe('native_module_missing'); + expect(classifyStartupCrash(bc('The module was compiled against a different Node.js version using NODE_MODULE_VERSION 108')).reason).toBe('native_module_missing'); + expect(classifyStartupCrash(bc('npm error install scripts were not run')).reason).toBe('native_module_missing'); + }); + + it('maps a plain missing module to module_not_found', () => { + expect(classifyStartupCrash(bc("Cannot find module 'lodash'")).reason).toBe('module_not_found'); + }); + + it('maps startup-phase config/JSON errors to config_error', () => { + expect(classifyStartupCrash(bc('Unexpected token } in JSON at position 5', { phase: 'startup' })).reason).toBe('config_error'); + expect(classifyStartupCrash(bc('failed to parse config file', { phase: 'startup' })).reason).toBe('config_error'); + }); + + it('does not treat JSON errors as config_error outside startup phase', () => { + expect(classifyStartupCrash(bc('Unexpected token in JSON', { phase: 'module_load' })).reason).toBe('unknown'); + }); + + it('maps permission / disk errors', () => { + expect(classifyStartupCrash(bc('EACCES: permission denied')).reason).toBe('permission_or_disk'); + expect(classifyStartupCrash(bc('ENOSPC: no space left on device')).reason).toBe('permission_or_disk'); + }); + + it('falls back to unknown and carries the raw message head', () => { + const res = classifyStartupCrash(bc('some totally novel failure\nsecond line')); + expect(res.reason).toBe('unknown'); + expect(res.detailHead).toBe('some totally novel failure'); + }); +}); diff --git a/tests/unit/updater/updater-metrics.test.ts b/tests/unit/updater/updater-metrics.test.ts index a5e35162..ee75e98c 100644 --- a/tests/unit/updater/updater-metrics.test.ts +++ b/tests/unit/updater/updater-metrics.test.ts @@ -274,6 +274,33 @@ describe('UpdaterMetrics', () => { await m.stop(); }); + it('enriches SERVICE_NOT_RUNNING_ALARM with the startup-crash cause when a breadcrumb exists', async () => { + mockReadFileSync.mockImplementation((p: string) => { + if (p.includes('last-startup-crash.json')) { + return JSON.stringify({ + schema: 1, ts: 1, phase: 'module_load', version: '9.9.9', pid: 1, + error_message: 'Cannot open sqlite3 binding', + error_stack_head: 'Error: Cannot open sqlite3 binding', + }); + } + return '12345\n'; + }); + const m = createMetrics('test-user', () => down('no matching collector command found')); + + await m.start(); + await vi.advanceTimersByTimeAsync(3 * 60_000 + 60_000); + await vi.advanceTimersByTimeAsync(60_000); + + const alarmLines = appendedLines.filter(l => l.path.includes('pilot-alarms.jsonl')); + expect(alarmLines).toHaveLength(1); + const parsed = JSON.parse(alarmLines[0].line); + expect(parsed.alarm_type).toBe('SERVICE_NOT_RUNNING_ALARM'); + expect(parsed.alarm_message).toContain('cause=native_module_missing'); + expect(parsed.alarm_message).toContain('version=9.9.9'); + expect(parsed.alarm_message).toContain('no matching collector command found'); + await m.stop(); + }); + it('does not alarm when collector PID changes but command identity is found', async () => { const m = createMetrics('test-user', () => ({ running: true, diff --git a/tests/unit/utils/crash-breadcrumb.test.ts b/tests/unit/utils/crash-breadcrumb.test.ts new file mode 100644 index 00000000..e45ea67c --- /dev/null +++ b/tests/unit/utils/crash-breadcrumb.test.ts @@ -0,0 +1,71 @@ +import { describe, it, expect, beforeEach, afterEach } from 'vitest'; +import * as fs from 'node:fs'; +import * as os from 'node:os'; +import * as path from 'node:path'; +import { + writeStartupCrash, + clearStartupCrash, + readStartupCrash, + startupCrashPath, +} from '../../../src/utils/crash-breadcrumb.js'; + +let dataDir: string; + +beforeEach(() => { + dataDir = fs.mkdtempSync(path.join(os.tmpdir(), 'lsp-crash-')); +}); + +afterEach(() => { + fs.rmSync(dataDir, { recursive: true, force: true }); +}); + +describe('crash-breadcrumb', () => { + it('writes a breadcrumb from an Error and reads it back', () => { + writeStartupCrash({ + dataDir, + phase: 'module_load', + version: '1.2.3', + error: new Error('Cannot open sqlite3 .node'), + }); + + expect(fs.existsSync(startupCrashPath(dataDir))).toBe(true); + const bc = readStartupCrash(dataDir); + expect(bc).not.toBeNull(); + expect(bc!.schema).toBe(1); + expect(bc!.phase).toBe('module_load'); + expect(bc!.version).toBe('1.2.3'); + expect(bc!.error_message).toBe('Cannot open sqlite3 .node'); + expect(bc!.error_stack_head).toContain('Error: Cannot open sqlite3 .node'); + expect(typeof bc!.ts).toBe('number'); + expect(bc!.pid).toBe(process.pid); + }); + + it('accepts a non-Error value and empties the stack head', () => { + writeStartupCrash({ dataDir, phase: 'startup', version: '', error: 'boom' }); + const bc = readStartupCrash(dataDir); + expect(bc!.error_message).toBe('boom'); + expect(bc!.error_stack_head).toBe(''); + expect(bc!.version).toBe('unknown'); + }); + + it('clears the breadcrumb and tolerates a missing file', () => { + writeStartupCrash({ dataDir, phase: 'startup', version: '1', error: new Error('x') }); + clearStartupCrash(dataDir); + expect(readStartupCrash(dataDir)).toBeNull(); + // second clear on an absent file must not throw + expect(() => clearStartupCrash(dataDir)).not.toThrow(); + }); + + it('returns null when no breadcrumb exists', () => { + expect(readStartupCrash(dataDir)).toBeNull(); + }); + + it('is best-effort and never throws on an unwritable data dir', () => { + // point at a path whose parent is a file, so mkdir/write fail internally + const fileAsDir = path.join(dataDir, 'not-a-dir'); + fs.writeFileSync(fileAsDir, 'x'); + expect(() => + writeStartupCrash({ dataDir: fileAsDir, phase: 'runtime', version: '1', error: new Error('y') }), + ).not.toThrow(); + }); +}); From f1e0b0df19857258cfe39d82af59d406edc5df87 Mon Sep 17 00:00:00 2001 From: Snssn <1502062504@qq.com> Date: Tue, 21 Jul 2026 16:28:33 +0800 Subject: [PATCH 2/2] =?UTF-8?q?fix(metrics):=20address=20review=20?= =?UTF-8?q?=E2=80=94=20consistent=20breadcrumb=20dir=20&=20narrower=20caus?= =?UTF-8?q?e=20classification?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Medium #1 (dataDir divergence): the breadcrumb reader (updater) and the daemon writer both use env-or-default, but index.ts used config.dataDir (which includes file.dataDir). When dataDir is set only in config.json, clear-on-success and the startup-phase write missed the reader's directory, breaking the "lingering breadcrumb = latest failed startup" invariant. Introduce resolveBreadcrumbDataDir() (env ?? ~/.loongsuite-pilot) and use it for all write/clear in index.ts. Medium #2 (config_error too broad): reorder permission_or_disk before config_error, and match config_error only on a JSON-parse signature in the error message (not the full stack, which routinely contains config-loader.ts paths). Drop bare "config"/"json" substrings. Low #1: clear the breadcrumb on the `!config.enabled` deliberate-exit path. Low #3: sanitize detailHead (strip quotes/control chars, collapse whitespace) so it is safe to embed in the alarm `detail="..."`. Update classifier tests for the narrowed rules, ordering, stack-path safety and detail sanitization. --- src/index.ts | 24 +++++---- src/updater/startup-crash-classifier.ts | 50 +++++++++++++------ src/utils/crash-breadcrumb.ts | 13 +++++ .../updater/startup-crash-classifier.test.ts | 30 +++++++++-- 4 files changed, 88 insertions(+), 29 deletions(-) diff --git a/src/index.ts b/src/index.ts index c49f1234..188eaf77 100644 --- a/src/index.ts +++ b/src/index.ts @@ -4,14 +4,11 @@ import { Orchestrator } from './core/orchestrator.js'; import { loadConfig } from './core/config-loader.js'; import { createLogger, initFileLogging } from './utils/logger.js'; import { resolveHome, readInstalledVersion } from './utils/fs-utils.js'; -import { writeStartupCrash, clearStartupCrash } from './utils/crash-breadcrumb.js'; +import { writeStartupCrash, clearStartupCrash, resolveBreadcrumbDataDir } from './utils/crash-breadcrumb.js'; import { handleWorkerCli } from './local-workers/worker-cli.js'; const logger = createLogger('Main'); -// Captured once config is loaded so the module-level catch can locate the data dir. -let resolvedDataDir: string | null = null; - async function main(): Promise { const argv = process.argv.slice(2); if (await handleWorkerCli(argv)) { @@ -28,11 +25,13 @@ async function main(): Promise { const config = await loadConfig(); const dataDir = resolveHome(config.dataDir); - resolvedDataDir = dataDir; const logDir = path.join(dataDir, 'logs'); await initFileLogging(path.join(logDir, 'loongsuite-pilot-service.log')); if (!config.enabled) { + // A deliberate, non-crash exit: drop any stale breadcrumb so it is not later + // misread as this run's failure cause. + clearStartupCrash(resolveBreadcrumbDataDir()); logger.info('analytics disabled via config or LOONGSUITE_PILOT_ENABLED=false'); return; } @@ -50,8 +49,9 @@ async function main(): Promise { await orchestrator.start(); // Reached a healthy running state: clear any stale crash breadcrumb so a lingering - // one always reflects the most recent *failed* startup attempt. - clearStartupCrash(dataDir); + // one always reflects the most recent *failed* startup attempt. The breadcrumb dir + // must match the daemon writer and the updater reader (env-or-default), not config.dataDir. + clearStartupCrash(resolveBreadcrumbDataDir()); logger.info('AI Agent Input is running', { dataDir: config.dataDir, @@ -63,9 +63,13 @@ async function main(): Promise { main().catch((err) => { logger.error('fatal startup error', { error: String(err) }); - const dataDir = resolvedDataDir - ?? resolveHome(process.env.LOONGSUITE_PILOT_DATA_DIR ?? '~/.loongsuite-pilot'); - writeStartupCrash({ dataDir, phase: 'startup', version: readInstalledVersion(dataDir), error: err }); + const breadcrumbDir = resolveBreadcrumbDataDir(); + writeStartupCrash({ + dataDir: breadcrumbDir, + phase: 'startup', + version: readInstalledVersion(breadcrumbDir), + error: err, + }); process.exit(1); }); diff --git a/src/updater/startup-crash-classifier.ts b/src/updater/startup-crash-classifier.ts index c0f917e1..d9fff240 100644 --- a/src/updater/startup-crash-classifier.ts +++ b/src/updater/startup-crash-classifier.ts @@ -20,40 +20,58 @@ const DETAIL_MAX_CHARS = 300; * wins, and `unknown` always carries the raw message so nothing is lost. */ export function classifyStartupCrash(breadcrumb: StartupCrashBreadcrumb): StartupCrashClassification { - const haystack = `${breadcrumb.error_message}\n${breadcrumb.error_stack_head}`.toLowerCase(); + const message = (breadcrumb.error_message || '').toLowerCase(); + const full = `${breadcrumb.error_message}\n${breadcrumb.error_stack_head}`.toLowerCase(); return { - reason: detectReason(haystack, breadcrumb.phase), - detailHead: firstLine(breadcrumb.error_message).slice(0, DETAIL_MAX_CHARS), + reason: detectReason(message, full, breadcrumb.phase), + detailHead: sanitizeDetail(firstLine(breadcrumb.error_message)), }; } -function detectReason(text: string, phase: string): StartupCrashReason { +function detectReason(message: string, full: string, phase: string): StartupCrashReason { if ( - text.includes('sqlite3') - || text.includes('err_dlopen_failed') - || text.includes('did not self-register') - || /cannot find module\s+['"][^'"]*\.node['"]/.test(text) - || text.includes('install scripts') - || text.includes('node_module_version') - || text.includes('compiled against a different node') + full.includes('sqlite3') + || full.includes('err_dlopen_failed') + || full.includes('did not self-register') + || /cannot find module\s+['"][^'"]*\.node['"]/.test(full) + || full.includes('install scripts') + || full.includes('node_module_version') + || full.includes('compiled against a different node') ) { return 'native_module_missing'; } - if (text.includes('cannot find module')) { + if (full.includes('cannot find module')) { return 'module_not_found'; } + // Check permission/disk before config so a startup-phase EACCES whose message merely + // mentions "config" is not mislabeled as a configuration error. + if (full.includes('eacces') || full.includes('erofs') || full.includes('enospc')) { + return 'permission_or_disk'; + } + // config_error is intentionally narrow: only a JSON-parse signature in the error + // *message* (not the stack, which routinely contains config-loader.ts paths) during + // the startup phase. Bare "config"/"json" substrings are too broad. if ( phase === 'startup' - && (text.includes('json') || text.includes('unexpected token') || text.includes('config')) + && ( + message.includes('unexpected token') + || message.includes('unexpected end of json') + || message.includes(' in json') + || message.includes('not valid json') + || message.includes('json.parse') + ) ) { return 'config_error'; } - if (text.includes('eacces') || text.includes('erofs') || text.includes('enospc')) { - return 'permission_or_disk'; - } return 'unknown'; } function firstLine(text: string): string { return (text || '').split(/\r?\n/)[0] ?? ''; } + +// Keep the detail safe to embed in `detail="..."` inside the alarm message: no quotes +// or control chars that could break downstream parsing/readability. +function sanitizeDetail(text: string): string { + return text.replace(/["\r\n\t]+/g, ' ').replace(/\s+/g, ' ').trim().slice(0, DETAIL_MAX_CHARS); +} diff --git a/src/utils/crash-breadcrumb.ts b/src/utils/crash-breadcrumb.ts index e2d3e26b..c8645ffd 100644 --- a/src/utils/crash-breadcrumb.ts +++ b/src/utils/crash-breadcrumb.ts @@ -1,5 +1,6 @@ import * as fs from 'node:fs'; import * as path from 'node:path'; +import { resolveHome } from './fs-utils.js'; export type StartupCrashPhase = 'module_load' | 'startup' | 'runtime'; @@ -21,6 +22,18 @@ export function startupCrashPath(dataDir: string): string { return path.join(dataDir, 'logs', FILE_NAME); } +/** + * The single data dir the breadcrumb lives in. It MUST match where the updater (the + * reader, src/updater/index.ts) and the bootstrap (scripts/collector-daemon.js, the + * earliest writer) look — both use env-or-default, NOT config.dataDir. Aligning the + * writer, clearer and reader on this one directory is what makes the "lingering + * breadcrumb = most recent failed startup" invariant hold even when config.json + * overrides dataDir. + */ +export function resolveBreadcrumbDataDir(): string { + return resolveHome(process.env.LOONGSUITE_PILOT_DATA_DIR ?? '~/.loongsuite-pilot'); +} + function truncateStackHead(stack: string | undefined): string { if (!stack) return ''; const head = stack.split(/\r?\n/).slice(0, STACK_HEAD_MAX_LINES).join('\n'); diff --git a/tests/unit/updater/startup-crash-classifier.test.ts b/tests/unit/updater/startup-crash-classifier.test.ts index 4dd99531..01df2282 100644 --- a/tests/unit/updater/startup-crash-classifier.test.ts +++ b/tests/unit/updater/startup-crash-classifier.test.ts @@ -31,23 +31,47 @@ describe('classifyStartupCrash', () => { expect(classifyStartupCrash(bc("Cannot find module 'lodash'")).reason).toBe('module_not_found'); }); - it('maps startup-phase config/JSON errors to config_error', () => { + it('maps startup-phase JSON-parse errors to config_error', () => { expect(classifyStartupCrash(bc('Unexpected token } in JSON at position 5', { phase: 'startup' })).reason).toBe('config_error'); - expect(classifyStartupCrash(bc('failed to parse config file', { phase: 'startup' })).reason).toBe('config_error'); + expect(classifyStartupCrash(bc('Unexpected end of JSON input', { phase: 'startup' })).reason).toBe('config_error'); }); it('does not treat JSON errors as config_error outside startup phase', () => { expect(classifyStartupCrash(bc('Unexpected token in JSON', { phase: 'module_load' })).reason).toBe('unknown'); }); + it('does not label a bare "config" mention (no JSON signature) as config_error', () => { + // narrowed: bare "config" is no longer a trigger, avoids false positives + expect(classifyStartupCrash(bc('failed to load config file', { phase: 'startup' })).reason).toBe('unknown'); + }); + + it('does not mislabel config_error from a stack path containing config-loader.ts', () => { + const res = classifyStartupCrash(bc('some novel failure', { + phase: 'startup', + stack: 'Error: some novel failure\n at loadConfig (/app/src/core/config-loader.ts:208:11)', + })); + expect(res.reason).toBe('unknown'); + }); + it('maps permission / disk errors', () => { expect(classifyStartupCrash(bc('EACCES: permission denied')).reason).toBe('permission_or_disk'); expect(classifyStartupCrash(bc('ENOSPC: no space left on device')).reason).toBe('permission_or_disk'); }); - it('falls back to unknown and carries the raw message head', () => { + it('prefers permission_or_disk over config_error when both could match', () => { + // startup-phase EACCES whose message also mentions JSON/config must stay permission_or_disk + expect(classifyStartupCrash(bc('EACCES: permission denied, open config.json', { phase: 'startup' })).reason).toBe('permission_or_disk'); + }); + + it('falls back to unknown and carries a sanitized raw message head', () => { const res = classifyStartupCrash(bc('some totally novel failure\nsecond line')); expect(res.reason).toBe('unknown'); expect(res.detailHead).toBe('some totally novel failure'); }); + + it('sanitizes quotes/whitespace in the detail head', () => { + const res = classifyStartupCrash(bc('boom "quoted" spaced')); + expect(res.detailHead).not.toContain('"'); + expect(res.detailHead).toBe('boom quoted spaced'); + }); });