db/import: Consolidate PostgreSQL memory config with dynamic scaling

- Add session-level import optimization function to migration
  (admin.set_optimal_import_session_settings) that boosts work_mem
  and maintenance_work_mem during batch imports

- Consolidate all PostgreSQL memory settings into single source of truth:
  - DB_MEM_LIMIT in .env.config drives all derived values
  - CLI calculates: shared_buffers (25%), effective_cache_size (75%),
    maintenance_work_mem (25%), work_mem (1%), temp_buffers (12.5%),
    wal_buffers (1.5%)
  - start-postgres.sh applies settings via command-line flags

- Fix temp_buffers runtime error: move from session-level SET to
  server startup (cannot be changed after temp tables are accessed)

- Add wal_buffers to dynamic scaling (was hardcoded at 64MB)

- Remove obsolete hotpatch files (now integrated into migrations)

- Delete redundant migrations that were created with future timestamps

Author: jhf

cli/src/manage.cr

@@ -192,14 +192,18 @@ module Statbus
       db_mem_limit : String
 
     # Type-safe derived memory configuration structure
+    # All memory settings are derived from DB_MEM_LIMIT for consistent scaling.
+    # See tmp/db-memory-todo.md for tuning rationale.
     record DbMemoryEnv,
       db_shm_size : String,
       db_mem_limit : String,
       db_mem_reservation : String,
       db_shared_buffers : String,
       db_maintenance_work_mem : String,
       db_effective_cache_size : String,
-      db_work_mem : String
+      db_work_mem : String,
+      db_temp_buffers : String,
+      db_wal_buffers : String
 
     # Configuration values that are derived from other settings
     record DerivedEnv,
@@ -396,12 +400,28 @@ module Statbus
         end
 
         # Calculate derived memory values based on DB_MEM_LIMIT
-        # See tmp/db-memory-todo.md for rationale.
+        # All PostgreSQL memory settings scale from this single source of truth.
+        # See tmp/db-memory-todo.md for detailed rationale.
+        #
+        # Memory allocation strategy (for 4GB total):
+        #   shared_buffers:       25% = 1GB    (main buffer cache)
+        #   effective_cache_size: 75% = 3GB    (planner hint for OS cache)
+        #   maintenance_work_mem: 25% = 1GB    (VACUUM, CREATE INDEX, etc.)
+        #   work_mem:             1%  = 40MB   (per-operation sorts/hashes, multiplied by queries)
+        #   temp_buffers:        12.5% = 512MB (temporary tables, min 256MB for imports)
+        #   wal_buffers:         ~1.5% = 64MB  (WAL write buffering, min 16MB, max 256MB)
+        #
         mem_limit_mb = parse_mem_size_to_mb(config.db_mem_limit)
         shared_buffers_mb = (mem_limit_mb * 0.25).to_i64
         maintenance_work_mem_mb = (mem_limit_mb * 0.25).to_i64
         effective_cache_size_mb = (mem_limit_mb * 0.75).to_i64
         work_mem_mb = Math.max(4_i64, mem_limit_mb // 100) # Min 4MB for safety
+        # temp_buffers: ~12.5% of memory, min 256MB for import temp tables
+        # Must be set at server startup (can't be changed after temp tables are accessed)
+        temp_buffers_mb = Math.max(256_i64, mem_limit_mb // 8)
+        # wal_buffers: ~1.5% of memory, clamped between 16MB and 256MB
+        # Larger values reduce disk I/O by buffering more WAL before write
+        wal_buffers_mb = Math.min(256_i64, Math.max(16_i64, (mem_limit_mb * 0.015).to_i64))
         reservation_mb = (mem_limit_mb // 2).to_i64
 
         db_mem = DbMemoryEnv.new(
@@ -411,7 +431,9 @@ module Statbus
           db_shared_buffers: format_mb_for_pg(shared_buffers_mb),
           db_maintenance_work_mem: format_mb_for_pg(maintenance_work_mem_mb),
           db_effective_cache_size: format_mb_for_pg(effective_cache_size_mb),
-          db_work_mem: format_mb_for_pg(work_mem_mb)
+          db_work_mem: format_mb_for_pg(work_mem_mb),
+          db_temp_buffers: format_mb_for_pg(temp_buffers_mb),
+          db_wal_buffers: format_mb_for_pg(wal_buffers_mb)
         )
 
         # Calculate derived values
@@ -606,14 +628,16 @@ module Statbus
 
         # PostgreSQL memory configuration
         # These control the docker container resource limits and postgresql.conf settings.
-        # They are derived from DB_MEM_LIMIT in .env.config.
+        # All values are derived from DB_MEM_LIMIT in .env.config (single source of truth).
         env.set("DB_MEM_LIMIT", db_mem.db_mem_limit)
         env.set("DB_SHM_SIZE", db_mem.db_shm_size)
         env.set("DB_MEM_RESERVATION", db_mem.db_mem_reservation)
         env.set("DB_SHARED_BUFFERS", db_mem.db_shared_buffers)
         env.set("DB_MAINTENANCE_WORK_MEM", db_mem.db_maintenance_work_mem)
         env.set("DB_EFFECTIVE_CACHE_SIZE", db_mem.db_effective_cache_size)
         env.set("DB_WORK_MEM", db_mem.db_work_mem)
+        env.set("DB_TEMP_BUFFERS", db_mem.db_temp_buffers)
+        env.set("DB_WAL_BUFFERS", db_mem.db_wal_buffers)
 
         env.set("ACCESS_JWT_EXPIRY", config.access_jwt_expiry)
         env.set("REFRESH_JWT_EXPIRY", config.refresh_jwt_expiry)

doc/performance/hotpatches-applied.md

@@ -108,6 +108,7 @@ DROP FUNCTION IF EXISTS admin.import_job_notify();
 DROP FUNCTION IF EXISTS admin.import_job_derive();
 DROP PROCEDURE IF EXISTS worker.notify_is_importing_start();
 DROP PROCEDURE IF EXISTS worker.notify_is_importing_stop();
+DROP FUNCTION IF EXISTS admin.set_optimal_import_session_settings();
 DROP FUNCTION IF EXISTS admin.trigger_validate_import_definition();
 DROP FUNCTION IF EXISTS admin.validate_import_definition(INT);
 DROP FUNCTION IF EXISTS admin.import_job_next_state(public.import_job);

migrations/20250423000000_add_import_jobs.down.sql

@@ -175,10 +175,73 @@ COMMENT ON TABLE public.import_definition_step IS 'Connects an import definition
 
 -- Removed trigger prevent_non_draft_definition_step_changes
 
+-- Session-Level Import Optimization Function
+-- Called by worker when processing import tasks to temporarily boost PostgreSQL settings
+-- for heavy import operations. These are session-level overrides that revert after the
+-- transaction completes, allowing aggressive memory usage during imports without
+-- affecting other concurrent operations.
+--
+-- Server-level memory settings (configured via DB_MEM_LIMIT in .env.config):
+--   work_mem, maintenance_work_mem, temp_buffers, wal_buffers, etc.
+-- are set conservatively to allow multiple concurrent operations.
+--
+-- This function boosts memory settings specifically for import batch processing.
+CREATE OR REPLACE FUNCTION admin.set_optimal_import_session_settings()
+RETURNS void 
+LANGUAGE plpgsql 
+SECURITY DEFINER  -- Required: regular users can't change PostgreSQL settings
+SET search_path = public, admin, pg_temp
+AS $set_optimal_import_session_settings$
+BEGIN
+    -- Memory boosts for import operations (session-level, reverts after transaction)
+    -- These override the conservative server defaults during batch imports.
+    -- Note: temp_buffers and wal_buffers cannot be changed at runtime.
+    SET LOCAL work_mem = '1GB';                    -- Boost for large hash joins and sorts
+    SET LOCAL maintenance_work_mem = '2GB';        -- Boost for index operations during temporal_merge
+    
+    -- Join strategy optimization (session-level, reverts after transaction)
+    SET LOCAL enable_hashjoin = on;                -- Prefer hash joins for large lookups
+    SET LOCAL enable_nestloop = off;               -- Avoid nested loops for large datasets  
+    SET LOCAL enable_mergejoin = off;              -- Avoid expensive sort-based merge joins
+    
+    -- Query optimizer hints for import workloads (session-level)
+    SET LOCAL random_page_cost = 1.1;              -- Optimize for modern storage (SSD)
+    SET LOCAL cpu_tuple_cost = 0.01;               -- Slight preference for CPU over I/O
+    SET LOCAL hash_mem_multiplier = 8.0;           -- Allow very large hash tables
+    
+    -- Enable more aggressive query optimization for complex import operations
+    SET LOCAL from_collapse_limit = 20;            -- Allow more complex query flattening
+    SET LOCAL join_collapse_limit = 20;            -- Allow more join reordering for optimization
+    
+    -- Log the optimization application for debugging
+    RAISE DEBUG 'Import session optimization applied: work_mem=1GB, maintenance_work_mem=2GB, hash_mem_multiplier=8x';
+END;
+$set_optimal_import_session_settings$;
+
+-- Grant execution to application user role (worker processes run as statbus_dev)
+GRANT EXECUTE ON FUNCTION admin.set_optimal_import_session_settings() TO statbus_dev;
+
+-- Comment explaining the function's purpose and usage
+COMMENT ON FUNCTION admin.set_optimal_import_session_settings() IS 
+'Applies session-level PostgreSQL optimizations for import operations. 
+Called by worker when processing import queue tasks. Boosts work_mem and 
+maintenance_work_mem beyond server defaults for batch processing. Uses 
+SECURITY DEFINER since regular users cannot change PostgreSQL settings. 
+All settings use SET LOCAL and automatically revert after the transaction completes.';
+
 -- Procedure to notify about import_job_process start
 CREATE PROCEDURE worker.notify_is_importing_start()
 LANGUAGE plpgsql AS $procedure$
 BEGIN
+    -- Apply session-level PostgreSQL optimizations for import operations
+    -- This ensures all subsequent queries in this transaction benefit from:
+    -- - Increased work_mem (1GB) for large hash tables and sorts
+    -- - Optimized join strategies (hash joins preferred over merge/nested loops)
+    -- - Large hash_mem_multiplier (8x) for complex operations
+    -- Settings automatically revert when transaction completes
+    PERFORM admin.set_optimal_import_session_settings();
+    
+    -- Notify that importing has started
     PERFORM pg_notify('worker_status', json_build_object('type', 'is_importing', 'status', true)::text);
 END;
 $procedure$;

migrations/20250423000000_add_import_jobs.up.sql

@@ -25,7 +25,7 @@ client_connection_check_interval = 5s
 # These are now set dynamically in start-postgres.sh based on environment variables.
 
 # Write-ahead log settings
-wal_buffers = 64MB                # Set WAL buffers to 64MB, allowing more WAL data to be stored in memory before being written to disk, reducing write frequency.
+# wal_buffers is now set dynamically in start-postgres.sh based on DB_WAL_BUFFERS env var.
 synchronous_commit = off          # Disable synchronous_commit to avoid waiting for WAL flush to disk, increasing performance at the cost of durability.
 wal_writer_delay = 500ms          # Delay WAL writes to disk by 500ms to batch more transactions and reduce disk I/O pressure.
 
@@ -49,7 +49,8 @@ log_min_messages = fatal
 log_min_duration_statement = 1000 # Log queries slower than 1000ms by default.
 
 # Other settings
-temp_buffers = 256MB              # Allocate memory for temporary tables operations.
+# temp_buffers is now set dynamically in start-postgres.sh based on DB_TEMP_BUFFERS env var.
+# Note: temp_buffers cannot be changed after any temporary tables have been accessed in a session.
 max_stack_depth = 7MB             # Increase stack depth for complex recursive queries (7MB is below the 7680kB limit).
 max_locks_per_transaction = 2048 # Increased to handle batch processing
 

postgres/postgresql.conf

@@ -13,11 +13,17 @@ PG_PARAMS="$PG_PARAMS -c logging_collector=off"
 PG_PARAMS="$PG_PARAMS -c log_destination=stderr"
 
 # Dynamic memory configuration (overrides postgresql.conf)
-# These can be set in the .env file. See tmp/db-memory-todo.md for tuning guidance.
+# All memory settings are derived from DB_MEM_LIMIT in .env.config (single source of truth).
+# These are calculated by the CLI and passed via environment variables.
+# See tmp/db-memory-todo.md for tuning rationale.
 PG_PARAMS="$PG_PARAMS -c shared_buffers=${DB_SHARED_BUFFERS:-1GB}"
 PG_PARAMS="$PG_PARAMS -c maintenance_work_mem=${DB_MAINTENANCE_WORK_MEM:-1GB}"
 PG_PARAMS="$PG_PARAMS -c effective_cache_size=${DB_EFFECTIVE_CACHE_SIZE:-3GB}"
-PG_PARAMS="$PG_PARAMS -c work_mem=${DB_WORK_MEM:-100MB}"
+PG_PARAMS="$PG_PARAMS -c work_mem=${DB_WORK_MEM:-40MB}"
+# temp_buffers must be set at server startup; cannot be changed after temp tables are accessed
+PG_PARAMS="$PG_PARAMS -c temp_buffers=${DB_TEMP_BUFFERS:-512MB}"
+# wal_buffers controls WAL write buffering; larger values reduce disk I/O
+PG_PARAMS="$PG_PARAMS -c wal_buffers=${DB_WAL_BUFFERS:-64MB}"
 
 # Default logging levels (these match postgresql.conf but will be overridden if DEBUG=true)
 # These values are set here to be passed as command-line arguments,