fix: 增强数据库启动容错，解决重启后 ECONNREFUSED 和 57P01 crash

docker-compose.portainer.yml

@@ -14,7 +14,8 @@ services:
       test: ["CMD-SHELL", "pg_isready -U ${POSTGRES_USER:-lore} -d ${POSTGRES_DB:-lore}"]
       interval: 10s
       timeout: 5s
-      retries: 10
+      retries: 30
+      start_period: 30s
 
   redis:
     image: ${LORE_REDIS_IMAGE:-redis:7-alpine}

docker-compose.yml

@@ -16,7 +16,8 @@ services:
       test: ["CMD-SHELL", "pg_isready -U ${POSTGRES_USER:-lore} -d ${POSTGRES_DB:-lore}"]
       interval: 10s
       timeout: 5s
-      retries: 10
+      retries: 30
+      start_period: 30s
 
   redis:
     image: ${LORE_REDIS_IMAGE:-redis:7-alpine}
@@ -43,7 +44,7 @@ services:
         condition: service_healthy
     environment:
       TZ: ${TZ:-Asia/Shanghai}
-      DATABASE_URL: ${DATABASE_URL:-postgresql://lore:change-me@postgres:5432/lore}
+      DATABASE_URL: ${DATABASE_URL:-postgresql://${POSTGRES_USER:-lore}:${POSTGRES_PASSWORD:-change-me}@postgres:5432/${POSTGRES_DB:-lore}}
       API_TOKEN: ${API_TOKEN:-}
       REDIS_URL: ${REDIS_URL:-redis://redis:6379/0}
       CACHE_KEY_PREFIX: ${CACHE_KEY_PREFIX:-lore}

web/server/db.ts

@@ -52,6 +52,10 @@ function getPool(): Pool {
     connectionTimeoutMillis: 10_000,
   });
 
+  pool.on('error', (error) => {
+    console.error('[db] idle client error', (error as Error)?.message || String(error));
+  });
+
   globalThis.__lorePgPool = pool;
   return pool;
 }

web/server/lore/__tests__/db.test.ts

@@ -1,12 +1,25 @@
 import { describe, it, expect, vi, beforeEach } from 'vitest';
 
+const pgMocks = vi.hoisted(() => ({
+  on: vi.fn(),
+  query: vi.fn().mockResolvedValue({ rows: [], rowCount: 0 }),
+}));
+
 vi.mock('pg', () => {
-  const mockQuery = vi.fn().mockResolvedValue({ rows: [], rowCount: 0 });
-  const MockPool = vi.fn().mockImplementation(() => ({ query: mockQuery }));
+  const MockPool = vi.fn().mockImplementation(() => ({ query: pgMocks.query, on: pgMocks.on }));
   return { Pool: MockPool };
 });
 
-import { _normalizeDatabaseUrl as normalizeDatabaseUrl, _buildSslConfig as buildSslConfig } from '../../db';
+import { Pool } from 'pg';
+import { _normalizeDatabaseUrl as normalizeDatabaseUrl, _buildSslConfig as buildSslConfig, getPool } from '../../db';
+
+beforeEach(() => {
+  globalThis.__lorePgPool = undefined;
+  delete process.env.DATABASE_URL;
+  pgMocks.on.mockClear();
+  pgMocks.query.mockClear();
+  vi.mocked(Pool).mockClear();
+});
 
 describe('normalizeDatabaseUrl', () => {
   it('returns empty for empty input', () => {
@@ -27,6 +40,24 @@ describe('normalizeDatabaseUrl', () => {
   });
 });
 
+describe('getPool', () => {
+  it('logs idle client errors without throwing', () => {
+    process.env.DATABASE_URL = 'postgresql://user:pass@postgres/db';
+
+    getPool();
+
+    expect(Pool).toHaveBeenCalledTimes(1);
+    expect(pgMocks.on).toHaveBeenCalledWith('error', expect.any(Function));
+    const errorHandler = pgMocks.on.mock.calls.find(([event]) => event === 'error')?.[1] as (error: Error) => void;
+    const consoleError = vi.spyOn(console, 'error').mockImplementation(() => undefined);
+
+    expect(() => errorHandler(new Error('terminating connection due to administrator command'))).not.toThrow();
+    expect(consoleError).toHaveBeenCalledWith('[db] idle client error', 'terminating connection due to administrator command');
+
+    consoleError.mockRestore();
+  });
+});
+
 describe('buildSslConfig', () => {
   it('returns false for localhost', () => {
     expect(buildSslConfig('postgresql://user:pass@localhost/db')).toBe(false);

web/server/lore/ops/__tests__/migrations-startup.test.ts

@@ -0,0 +1,49 @@
+import { describe, expect, it, vi, beforeEach } from 'vitest';
+
+const dbMocks = vi.hoisted(() => ({
+  query: vi.fn(),
+}));
+
+vi.mock('../../../db', () => ({
+  getPool: () => dbMocks,
+}));
+
+vi.mock('node:fs', () => ({
+  existsSync: vi.fn(() => false),
+  readdirSync: vi.fn(),
+  readFileSync: vi.fn(),
+}));
+
+import { runMigrations } from '../migrations';
+
+beforeEach(() => {
+  dbMocks.query.mockReset();
+  process.env.LORE_DB_STARTUP_WAIT_MS = '50';
+  process.env.LORE_DB_STARTUP_RETRY_MS = '1';
+});
+
+describe('runMigrations startup database readiness', () => {
+  it('retries transient database connection failures before running migrations', async () => {
+    const connectionError = Object.assign(new Error('connect ECONNREFUSED 192.168.107.3:5432'), {
+      code: 'ECONNREFUSED',
+    });
+    dbMocks.query
+      .mockRejectedValueOnce(connectionError)
+      .mockResolvedValueOnce({ rows: [], rowCount: 0 })
+      .mockResolvedValueOnce({ rows: [], rowCount: 0 })
+      .mockResolvedValueOnce({ rows: [], rowCount: 0 });
+    const consoleWarn = vi.spyOn(console, 'warn').mockImplementation(() => undefined);
+    const consoleLog = vi.spyOn(console, 'log').mockImplementation(() => undefined);
+
+    await runMigrations();
+
+    expect(dbMocks.query).toHaveBeenCalledTimes(4);
+    expect(dbMocks.query).toHaveBeenNthCalledWith(1, 'SELECT 1');
+    expect(dbMocks.query).toHaveBeenNthCalledWith(2, 'SELECT 1');
+    expect(dbMocks.query).toHaveBeenNthCalledWith(4, 'SELECT version FROM schema_migrations');
+    expect(consoleWarn).toHaveBeenCalledWith(expect.stringContaining('[migrations] waiting for database (ECONNREFUSED)'));
+
+    consoleWarn.mockRestore();
+    consoleLog.mockRestore();
+  });
+});

web/server/lore/ops/migrations.ts

@@ -19,6 +19,72 @@ import * as path from 'node:path';
 import { getPool } from '../../db';
 
 const MIGRATIONS_DIR = path.resolve(process.cwd(), 'migrations');
+const DEFAULT_DB_STARTUP_WAIT_MS = 120_000;
+const DEFAULT_DB_STARTUP_RETRY_MS = 2_000;
+const TRANSIENT_CONNECTION_CODES = new Set([
+  'ECONNREFUSED',
+  'ECONNRESET',
+  'ETIMEDOUT',
+  'ENOTFOUND',
+  'EAI_AGAIN',
+  '57P01',
+  '08000',
+  '08001',
+  '08003',
+  '08006',
+]);
+
+function getPositiveIntegerEnv(name: string, fallback: number): number {
+  const value = Number(process.env[name]);
+  return Number.isFinite(value) && value > 0 ? value : fallback;
+}
+
+function getDatabaseErrorCode(error: unknown): string | undefined {
+  if (!error || typeof error !== 'object') return undefined;
+  const code = (error as { code?: unknown }).code;
+  return typeof code === 'string' ? code : undefined;
+}
+
+function isTransientDatabaseStartupError(error: unknown): boolean {
+  const code = getDatabaseErrorCode(error);
+  if (code && TRANSIENT_CONNECTION_CODES.has(code)) return true;
+
+  const message = error instanceof Error ? error.message : String(error || '');
+  return /terminating connection|connection terminated|database system is starting up|the database system is in recovery mode/i.test(message);
+}
+
+function sleep(ms: number): Promise<void> {
+  return new Promise((resolve) => setTimeout(resolve, ms));
+}
+
+async function waitForDatabaseReady(): Promise<void> {
+  const waitMs = getPositiveIntegerEnv('LORE_DB_STARTUP_WAIT_MS', DEFAULT_DB_STARTUP_WAIT_MS);
+  const retryMs = getPositiveIntegerEnv('LORE_DB_STARTUP_RETRY_MS', DEFAULT_DB_STARTUP_RETRY_MS);
+  const startedAt = Date.now();
+  let attempt = 0;
+
+  while (true) {
+    attempt += 1;
+    try {
+      await getPool().query('SELECT 1');
+      if (attempt > 1) console.log('[migrations] database is ready');
+      return;
+    } catch (error) {
+      const elapsedMs = Date.now() - startedAt;
+      const remainingMs = waitMs - elapsedMs;
+      if (!isTransientDatabaseStartupError(error) || remainingMs <= 0) {
+        throw error;
+      }
+
+      const code = getDatabaseErrorCode(error);
+      const message = error instanceof Error ? error.message : String(error);
+      console.warn(
+        `[migrations] waiting for database (${code || 'unknown'}): ${message}; retrying in ${Math.min(retryMs, remainingMs)}ms`,
+      );
+      await sleep(Math.min(retryMs, remainingMs));
+    }
+  }
+}
 
 async function ensureTrackingTable(): Promise<void> {
   await getPool().query(`
@@ -45,6 +111,7 @@ function loadMigrationFiles(): { version: number; name: string; sql: string }[]
 }
 
 export async function runMigrations(): Promise<void> {
+  await waitForDatabaseReady();
   await ensureTrackingTable();
 
   const applied = await getPool().query('SELECT version FROM schema_migrations');