From 798871c27b777817f7cf46df3ac2b4a3284fbbdb Mon Sep 17 00:00:00 2001 From: wct <2207561089@qq.com> Date: Tue, 9 Jun 2026 19:19:51 +0800 Subject: [PATCH] =?UTF-8?q?fix:=20=E5=A2=9E=E5=BC=BA=E6=95=B0=E6=8D=AE?= =?UTF-8?q?=E5=BA=93=E5=90=AF=E5=8A=A8=E5=AE=B9=E9=94=99=EF=BC=8C=E8=A7=A3?= =?UTF-8?q?=E5=86=B3=E9=87=8D=E5=90=AF=E5=90=8E=20ECONNREFUSED=20=E5=92=8C?= =?UTF-8?q?=2057P01=20crash?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 三层防御： - db.ts: pg.Pool 增加 idle-error 监听器，避免数据库重启导致 uncaughtException - migrations.ts: runMigrations 前等待数据库就绪，自动重试临时性连接错误 - docker-compose: Postgres healthcheck 增加 start_period 和更多 retries，修复 DATABASE_URL 默认值硬编码问题 --- docker-compose.portainer.yml | 3 +- docker-compose.yml | 5 +- web/server/db.ts | 4 ++ web/server/lore/__tests__/db.test.ts | 37 +++++++++- .../ops/__tests__/migrations-startup.test.ts | 49 ++++++++++++++ web/server/lore/ops/migrations.ts | 67 +++++++++++++++++++ 6 files changed, 159 insertions(+), 6 deletions(-) create mode 100644 web/server/lore/ops/__tests__/migrations-startup.test.ts diff --git a/docker-compose.portainer.yml b/docker-compose.portainer.yml index 3053862..beae350 100644 --- a/docker-compose.portainer.yml +++ b/docker-compose.portainer.yml @@ -14,7 +14,8 @@ services: test: ["CMD-SHELL", "pg_isready -U ${POSTGRES_USER:-lore} -d ${POSTGRES_DB:-lore}"] interval: 10s timeout: 5s - retries: 10 + retries: 30 + start_period: 30s redis: image: ${LORE_REDIS_IMAGE:-redis:7-alpine} diff --git a/docker-compose.yml b/docker-compose.yml index d622b4b..dc49de0 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -16,7 +16,8 @@ services: test: ["CMD-SHELL", "pg_isready -U ${POSTGRES_USER:-lore} -d ${POSTGRES_DB:-lore}"] interval: 10s timeout: 5s - retries: 10 + retries: 30 + start_period: 30s redis: image: ${LORE_REDIS_IMAGE:-redis:7-alpine} @@ -43,7 +44,7 @@ services: condition: service_healthy environment: TZ: ${TZ:-Asia/Shanghai} - DATABASE_URL: ${DATABASE_URL:-postgresql://lore:change-me@postgres:5432/lore} + DATABASE_URL: ${DATABASE_URL:-postgresql://${POSTGRES_USER:-lore}:${POSTGRES_PASSWORD:-change-me}@postgres:5432/${POSTGRES_DB:-lore}} API_TOKEN: ${API_TOKEN:-} REDIS_URL: ${REDIS_URL:-redis://redis:6379/0} CACHE_KEY_PREFIX: ${CACHE_KEY_PREFIX:-lore} diff --git a/web/server/db.ts b/web/server/db.ts index 4f4761f..7c044b0 100644 --- a/web/server/db.ts +++ b/web/server/db.ts @@ -52,6 +52,10 @@ function getPool(): Pool { connectionTimeoutMillis: 10_000, }); + pool.on('error', (error) => { + console.error('[db] idle client error', (error as Error)?.message || String(error)); + }); + globalThis.__lorePgPool = pool; return pool; } diff --git a/web/server/lore/__tests__/db.test.ts b/web/server/lore/__tests__/db.test.ts index 4ec2e95..f34ed5a 100644 --- a/web/server/lore/__tests__/db.test.ts +++ b/web/server/lore/__tests__/db.test.ts @@ -1,12 +1,25 @@ import { describe, it, expect, vi, beforeEach } from 'vitest'; +const pgMocks = vi.hoisted(() => ({ + on: vi.fn(), + query: vi.fn().mockResolvedValue({ rows: [], rowCount: 0 }), +})); + vi.mock('pg', () => { - const mockQuery = vi.fn().mockResolvedValue({ rows: [], rowCount: 0 }); - const MockPool = vi.fn().mockImplementation(() => ({ query: mockQuery })); + const MockPool = vi.fn().mockImplementation(() => ({ query: pgMocks.query, on: pgMocks.on })); return { Pool: MockPool }; }); -import { _normalizeDatabaseUrl as normalizeDatabaseUrl, _buildSslConfig as buildSslConfig } from '../../db'; +import { Pool } from 'pg'; +import { _normalizeDatabaseUrl as normalizeDatabaseUrl, _buildSslConfig as buildSslConfig, getPool } from '../../db'; + +beforeEach(() => { + globalThis.__lorePgPool = undefined; + delete process.env.DATABASE_URL; + pgMocks.on.mockClear(); + pgMocks.query.mockClear(); + vi.mocked(Pool).mockClear(); +}); describe('normalizeDatabaseUrl', () => { it('returns empty for empty input', () => { @@ -27,6 +40,24 @@ describe('normalizeDatabaseUrl', () => { }); }); +describe('getPool', () => { + it('logs idle client errors without throwing', () => { + process.env.DATABASE_URL = 'postgresql://user:pass@postgres/db'; + + getPool(); + + expect(Pool).toHaveBeenCalledTimes(1); + expect(pgMocks.on).toHaveBeenCalledWith('error', expect.any(Function)); + const errorHandler = pgMocks.on.mock.calls.find(([event]) => event === 'error')?.[1] as (error: Error) => void; + const consoleError = vi.spyOn(console, 'error').mockImplementation(() => undefined); + + expect(() => errorHandler(new Error('terminating connection due to administrator command'))).not.toThrow(); + expect(consoleError).toHaveBeenCalledWith('[db] idle client error', 'terminating connection due to administrator command'); + + consoleError.mockRestore(); + }); +}); + describe('buildSslConfig', () => { it('returns false for localhost', () => { expect(buildSslConfig('postgresql://user:pass@localhost/db')).toBe(false); diff --git a/web/server/lore/ops/__tests__/migrations-startup.test.ts b/web/server/lore/ops/__tests__/migrations-startup.test.ts new file mode 100644 index 0000000..c524fb1 --- /dev/null +++ b/web/server/lore/ops/__tests__/migrations-startup.test.ts @@ -0,0 +1,49 @@ +import { describe, expect, it, vi, beforeEach } from 'vitest'; + +const dbMocks = vi.hoisted(() => ({ + query: vi.fn(), +})); + +vi.mock('../../../db', () => ({ + getPool: () => dbMocks, +})); + +vi.mock('node:fs', () => ({ + existsSync: vi.fn(() => false), + readdirSync: vi.fn(), + readFileSync: vi.fn(), +})); + +import { runMigrations } from '../migrations'; + +beforeEach(() => { + dbMocks.query.mockReset(); + process.env.LORE_DB_STARTUP_WAIT_MS = '50'; + process.env.LORE_DB_STARTUP_RETRY_MS = '1'; +}); + +describe('runMigrations startup database readiness', () => { + it('retries transient database connection failures before running migrations', async () => { + const connectionError = Object.assign(new Error('connect ECONNREFUSED 192.168.107.3:5432'), { + code: 'ECONNREFUSED', + }); + dbMocks.query + .mockRejectedValueOnce(connectionError) + .mockResolvedValueOnce({ rows: [], rowCount: 0 }) + .mockResolvedValueOnce({ rows: [], rowCount: 0 }) + .mockResolvedValueOnce({ rows: [], rowCount: 0 }); + const consoleWarn = vi.spyOn(console, 'warn').mockImplementation(() => undefined); + const consoleLog = vi.spyOn(console, 'log').mockImplementation(() => undefined); + + await runMigrations(); + + expect(dbMocks.query).toHaveBeenCalledTimes(4); + expect(dbMocks.query).toHaveBeenNthCalledWith(1, 'SELECT 1'); + expect(dbMocks.query).toHaveBeenNthCalledWith(2, 'SELECT 1'); + expect(dbMocks.query).toHaveBeenNthCalledWith(4, 'SELECT version FROM schema_migrations'); + expect(consoleWarn).toHaveBeenCalledWith(expect.stringContaining('[migrations] waiting for database (ECONNREFUSED)')); + + consoleWarn.mockRestore(); + consoleLog.mockRestore(); + }); +}); diff --git a/web/server/lore/ops/migrations.ts b/web/server/lore/ops/migrations.ts index a7065a1..410fa90 100644 --- a/web/server/lore/ops/migrations.ts +++ b/web/server/lore/ops/migrations.ts @@ -19,6 +19,72 @@ import * as path from 'node:path'; import { getPool } from '../../db'; const MIGRATIONS_DIR = path.resolve(process.cwd(), 'migrations'); +const DEFAULT_DB_STARTUP_WAIT_MS = 120_000; +const DEFAULT_DB_STARTUP_RETRY_MS = 2_000; +const TRANSIENT_CONNECTION_CODES = new Set([ + 'ECONNREFUSED', + 'ECONNRESET', + 'ETIMEDOUT', + 'ENOTFOUND', + 'EAI_AGAIN', + '57P01', + '08000', + '08001', + '08003', + '08006', +]); + +function getPositiveIntegerEnv(name: string, fallback: number): number { + const value = Number(process.env[name]); + return Number.isFinite(value) && value > 0 ? value : fallback; +} + +function getDatabaseErrorCode(error: unknown): string | undefined { + if (!error || typeof error !== 'object') return undefined; + const code = (error as { code?: unknown }).code; + return typeof code === 'string' ? code : undefined; +} + +function isTransientDatabaseStartupError(error: unknown): boolean { + const code = getDatabaseErrorCode(error); + if (code && TRANSIENT_CONNECTION_CODES.has(code)) return true; + + const message = error instanceof Error ? error.message : String(error || ''); + return /terminating connection|connection terminated|database system is starting up|the database system is in recovery mode/i.test(message); +} + +function sleep(ms: number): Promise { + return new Promise((resolve) => setTimeout(resolve, ms)); +} + +async function waitForDatabaseReady(): Promise { + const waitMs = getPositiveIntegerEnv('LORE_DB_STARTUP_WAIT_MS', DEFAULT_DB_STARTUP_WAIT_MS); + const retryMs = getPositiveIntegerEnv('LORE_DB_STARTUP_RETRY_MS', DEFAULT_DB_STARTUP_RETRY_MS); + const startedAt = Date.now(); + let attempt = 0; + + while (true) { + attempt += 1; + try { + await getPool().query('SELECT 1'); + if (attempt > 1) console.log('[migrations] database is ready'); + return; + } catch (error) { + const elapsedMs = Date.now() - startedAt; + const remainingMs = waitMs - elapsedMs; + if (!isTransientDatabaseStartupError(error) || remainingMs <= 0) { + throw error; + } + + const code = getDatabaseErrorCode(error); + const message = error instanceof Error ? error.message : String(error); + console.warn( + `[migrations] waiting for database (${code || 'unknown'}): ${message}; retrying in ${Math.min(retryMs, remainingMs)}ms`, + ); + await sleep(Math.min(retryMs, remainingMs)); + } + } +} async function ensureTrackingTable(): Promise { await getPool().query(` @@ -45,6 +111,7 @@ function loadMigrationFiles(): { version: number; name: string; sql: string }[] } export async function runMigrations(): Promise { + await waitForDatabaseReady(); await ensureTrackingTable(); const applied = await getPool().query('SELECT version FROM schema_migrations');