v1.5.15 — TCP keepalive on remote Postgres connections (#77)

Author: epheterson

CHANGELOG.md

@@ -1,5 +1,10 @@
 # Changelog
 
+## 1.5.15 — 2026-06-23
+
+### Fixes
+- **Keep remote Postgres connections alive in split deployments (#74).** When the worker and the database sit on different network segments, a stateful firewall or NAT can silently reap an idle connection; the worker then hangs on the next read until `read ETIMEDOUT` and doesn't recover (intermittent, since only idle connections get reaped). Immich doesn't expose a keepalive setting, so the worker now preloads a small shim that enables TCP keepAlive on every `pg` connection, keeping the socket warm so it isn't dropped. No-op for same-host setups; Immich's source is untouched. Tunable/disable via `IMMICH_ACCEL_PG_KEEPALIVE` / `IMMICH_ACCEL_PG_KEEPALIVE_MS`. Reported by [@shtefko](https://github.com/shtefko).
+
 ## 1.5.14 — 2026-06-23
 
 ### Fixes

VERSION

@@ -1 +1 @@
-1.5.14
+1.5.15

immich_accelerator/__main__.py

@@ -3473,6 +3473,20 @@ def cmd_start(args):
             f"{existing} {require_arg}".strip() if existing else require_arg
         )
 
+    # pg keepalive shim (issue #74): in a split deployment a stateful firewall
+    # between the worker and a remote Postgres can silently reap an idle
+    # connection, after which the next read hangs to ETIMEDOUT and the worker
+    # never recovers. Immich doesn't expose a keepalive env var, so we wrap the
+    # `pg` module to set keepAlive on every connection. Same --require
+    # interposition; Immich's source is untouched. No-op for same-host setups.
+    pg_keepalive_shim = Path(__file__).parent / "hooks" / "pg_keepalive_shim.js"
+    if pg_keepalive_shim.exists():
+        existing = worker_env.get("NODE_OPTIONS", "").strip()
+        require_arg = f'--require "{pg_keepalive_shim}"'
+        worker_env["NODE_OPTIONS"] = (
+            f"{existing} {require_arg}".strip() if existing else require_arg
+        )
+
     # /build link points to our build-data dir (set up during setup).
     # Required for Immich 2.7+ plugin WASM paths stored in the shared DB.
     build_data = DATA_DIR / "build-data"

immich_accelerator/hooks/pg_keepalive_shim.js

@@ -0,0 +1,91 @@
+// pg_keepalive_shim.js
+//
+// Runtime interposition to enable TCP keepalive on Immich's Postgres
+// connections (issue #74).
+//
+// In a split deployment the worker holds long-lived connections across the
+// network to a remote Postgres. A stateful firewall or NAT between them (e.g.
+// the worker on one VLAN, the database on another) silently reaps a connection
+// once it has been idle past the firewall's timeout. node-postgres never learns
+// the socket is gone, so the next query's read just hangs until it fails with
+// `read ETIMEDOUT`, and the worker does not recover. This is intermittent by
+// nature — only idle connections get reaped, active ones survive — which is
+// what makes it confusing to diagnose.
+//
+// node-postgres supports `keepAlive` on the socket, but Immich doesn't expose
+// it as an env var (it's a Pool/Client option, not part of DB_URL). Rather than
+// patch Immich's source (which would break the "unmodified" invariant), we
+// preload this module via `NODE_OPTIONS=--require ...` and wrap the `pg`
+// module's Pool and Client constructors so every connection sets keepAlive,
+// keeping the socket warm so the firewall never considers it idle.
+//
+// Same interposition pattern as the pg_dump and HEIC shims. Off the hot path
+// (only runs at connection construction) and a no-op for same-host setups where
+// nothing reaps the connection. Opt-out / tune via env (see below).
+
+'use strict';
+
+// keepAlive is benign on a healthy LAN, so default it on. The initial delay is
+// how long a connection sits idle before the first keepalive probe; 10s is well
+// under typical firewall idle timeouts (often 60–300s) while staying cheap.
+const ENABLED = process.env.IMMICH_ACCEL_PG_KEEPALIVE !== '0';
+const DELAY_MS = parseInt(
+    process.env.IMMICH_ACCEL_PG_KEEPALIVE_MS || '10000', 10
+);
+
+function withKeepAlive(config) {
+    // pg accepts the config as the first constructor arg (object or undefined;
+    // a connection-string shorthand is also allowed). Only augment the object
+    // form, and never override an explicit choice the caller already made.
+    if (config == null || typeof config !== 'object') {
+        config = { connectionString: config == null ? undefined : config };
+    }
+    if (config.keepAlive === undefined) {
+        config.keepAlive = true;
+    }
+    if (config.keepAliveInitialDelayMillis === undefined) {
+        config.keepAliveInitialDelayMillis = DELAY_MS;
+    }
+    return config;
+}
+
+// A construct-trap Proxy preserves instanceof, the prototype chain, and static
+// properties (kysely and pg's own internals reference these), unlike wrapping
+// the class in a plain function.
+function wrapCtor(Original) {
+    return new Proxy(Original, {
+        construct(target, args, newTarget) {
+            const next = args.slice();
+            next[0] = withKeepAlive(next[0]);
+            return Reflect.construct(target, next, newTarget);
+        },
+    });
+}
+
+function patchPg(pg) {
+    if (pg.Pool) pg.Pool = wrapCtor(pg.Pool);
+    if (pg.Client) pg.Client = wrapCtor(pg.Client);
+    process.stderr.write(
+        `[immich-accelerator] pg keepalive enabled (initial delay ${DELAY_MS}ms)\n`
+    );
+}
+
+if (ENABLED) {
+    const Module = require('module');
+    const origLoad = Module._load;
+    Module._load = function (request, parent, isMain) {
+        const mod = origLoad.apply(this, arguments);
+        if (request === 'pg' && mod && !mod.__keepAlivePatched) {
+            try {
+                patchPg(mod);
+                mod.__keepAlivePatched = true;
+            } catch (e) {
+                process.stderr.write(
+                    '[immich-accelerator] pg keepalive shim failed: ' +
+                    String((e && e.message) || e) + '\n'
+                );
+            }
+        }
+        return mod;
+    };
+}

tests/test_fresh_install.py

@@ -1484,3 +1484,93 @@ def test_intercepts_sharp_routes_heic_passes_through_others(self, tmp_path):
             "INPUT_TYPE:buffer",
             "INPUT_TYPE:string",
         ], f"expected heic→buffer, other→string; got {types}\n{result.stdout}"
+
+
+class TestPgKeepaliveShim:
+    """The pg keepalive shim sets keepAlive on Immich's Postgres connections so
+    a stateful firewall between worker and a remote DB can't reap idle
+    connections (issue #74). Immich's source is never touched — the shim wraps
+    the `pg` module via NODE_OPTIONS=--require."""
+
+    SHIM_PATH = REPO_ROOT / "immich_accelerator" / "hooks" / "pg_keepalive_shim.js"
+
+    def test_shim_file_exists(self):
+        assert self.SHIM_PATH.exists(), f"hook shim missing: {self.SHIM_PATH}"
+
+    def test_shim_is_referenced_by_cmd_start(self):
+        src = (REPO_ROOT / "immich_accelerator" / "__main__.py").read_text()
+        assert "pg_keepalive_shim.js" in src
+        assert "NODE_OPTIONS" in src
+
+    @pytest.mark.slow
+    def test_shim_injects_keepalive_into_pg(self, tmp_path):
+        """Run node with the shim preloaded against a fake `pg` module (no real
+        DB), construct a Pool, and confirm keepAlive is injected while
+        instanceof and an explicit caller value are both preserved."""
+        import shutil
+
+        node = shutil.which("node")
+        if not node:
+            pytest.skip("node not installed")
+
+        # Minimal stand-in for node-postgres: Pool/Client that stash their config.
+        pgdir = tmp_path / "node_modules" / "pg"
+        pgdir.mkdir(parents=True)
+        (pgdir / "index.js").write_text(
+            "class Pool { constructor(c){ this.options = c || {}; } }\n"
+            "class Client { constructor(c){ this.options = c || {}; } }\n"
+            "module.exports = { Pool, Client };\n"
+        )
+
+        caller = tmp_path / "caller.js"
+        caller.write_text(
+            "const { Pool } = require('pg');\n"
+            "const p = new Pool({ host: 'db' });\n"
+            "console.log('KEEPALIVE:' + p.options.keepAlive);\n"
+            "console.log('DELAY:' + p.options.keepAliveInitialDelayMillis);\n"
+            "console.log('INSTANCE:' + (p instanceof Pool));\n"
+            "const q = new Pool({ keepAlive: false });\n"
+            "console.log('EXPLICIT:' + q.options.keepAlive);\n"
+        )
+        result = subprocess.run(
+            [node, "--require", str(self.SHIM_PATH), str(caller)],
+            cwd=str(tmp_path),
+            capture_output=True,
+            text=True,
+            timeout=15,
+        )
+        assert result.returncode == 0, f"stdout={result.stdout}\nstderr={result.stderr}"
+        out = result.stdout
+        assert "KEEPALIVE:true" in out, out
+        assert "DELAY:10000" in out, out
+        assert "INSTANCE:true" in out, out  # Proxy preserves instanceof
+        assert "EXPLICIT:false" in out, out  # never override an explicit choice
+
+    @pytest.mark.slow
+    def test_shim_disabled_via_env(self, tmp_path):
+        import shutil
+
+        node = shutil.which("node")
+        if not node:
+            pytest.skip("node not installed")
+        pgdir = tmp_path / "node_modules" / "pg"
+        pgdir.mkdir(parents=True)
+        (pgdir / "index.js").write_text(
+            "class Pool { constructor(c){ this.options = c || {}; } }\n"
+            "module.exports = { Pool };\n"
+        )
+        caller = tmp_path / "caller.js"
+        caller.write_text(
+            "const { Pool } = require('pg');\n"
+            "console.log('KEEPALIVE:' + new Pool({}).options.keepAlive);\n"
+        )
+        result = subprocess.run(
+            [node, "--require", str(self.SHIM_PATH), str(caller)],
+            cwd=str(tmp_path),
+            env={"IMMICH_ACCEL_PG_KEEPALIVE": "0", "PATH": "/usr/bin:/bin"},
+            capture_output=True,
+            text=True,
+            timeout=15,
+        )
+        assert result.returncode == 0, result.stderr
+        assert "KEEPALIVE:undefined" in result.stdout, result.stdout