diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c3cedb7c..9fc0b2ef 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -97,12 +97,9 @@ jobs: # TODO: Re-enable once pg_regress tests are stabilized # - name: Run pg_regress tests # id: pg_regress - # env: - # PGDATABASE: contrib_regression # run: | - # ./scripts/pg-start.sh - # cd test/regress - # make installcheck + # ./scripts/pg-start.sh contrib_regression + # PG_CONFIG=$(cargo pgrx info pg-config pg17) make installcheck # - name: Upload pg_regress results on failure # if: steps.pg_regress.outcome == 'failure' diff --git a/Makefile b/Makefile index e765a839..435fde0d 100644 --- a/Makefile +++ b/Makefile @@ -22,9 +22,11 @@ test-e2e: ./scripts/test.sh --e2e # Run pg_regress tests (requires PostgreSQL running) +# Respects PGHOST, PGPORT, PGUSER, PGDATABASE environment variables installcheck: - @echo "Running pg_regress tests (PostgreSQL must be running with PGDATABASE=contrib_regression)..." - @cd test/regress && make installcheck + @echo "Running pg_regress tests (using PGHOST=$${PGHOST:-localhost} PGPORT=$${PGPORT:-5432})..." + @echo "Is pg_durable.database_name configured to match the pg_regress database?" + @cd test/regress && $(MAKE) installcheck # Alias for installcheck check: installcheck diff --git a/docs/dbname_and_host_gucs.md b/docs/dbname_and_host_gucs.md new file mode 100644 index 00000000..9960182c --- /dev/null +++ b/docs/dbname_and_host_gucs.md @@ -0,0 +1,398 @@ +# Design: Database and Host Configuration for pg_durable + +**Status**: Implemented +**Date**: 2026-01-20 + +## Overview + +This document describes the GUC configuration and security model for pg_durable, focusing on: +1. Database connection configuration (host/socket directory and target database) +2. OS-level peer authentication for all connections +3. Extension-based schema creation with worker-based initialization + +## Goals + +1. **Unix Domain Sockets**: Use Unix domain sockets exclusively for all connections to avoid network overhead and improve security +2. **Configurable Database**: Allow administrators to specify which database the background worker connects to +3. **Peer Authentication**: Use OS-level peer authentication (no custom roles needed) +4. **Clean Extension Model**: Extension creates schema structure, worker populates it +5. **Simple Security**: Rely on PostgreSQL's built-in authentication mechanisms + +## Design + +### 1. Configuration Parameters (GUCs) + +#### 1.1. `pg_durable.host` + +**Purpose**: Specify the Unix domain socket directory for PostgreSQL connections. + +**Type**: String +**Context**: `postmaster` (requires server restart) +**Default**: Empty string (uses PostgreSQL's default Unix socket directory) + +**Behavior**: +- If empty/not set: Use PostgreSQL's default Unix socket directory (typically `/tmp` or `/var/run/postgresql`) +- If set: Use the specified directory path + +**Example**: +```sql +-- postgresql.conf +pg_durable.host = '/var/run/postgresql' +``` + +**Note**: Only Unix socket directories are supported - network connections are not allowed. + +#### 1.2. `pg_durable.database_name` + +**Purpose**: Specify the database to which the background worker connects. + +**Type**: String +**Context**: `postmaster` (requires server restart) +**Default**: `"postgres"` + +**Behavior**: +- Background worker connects to this database on startup +- All duroxide internal tables (`duroxide` schema) live in this database +- All `df.*` user-facing tables live in this database + +**Example**: +```sql +-- postgresql.conf +pg_durable.database_name = 'myapp_db' +``` + +### 2. Security Model + +#### 2.1. Peer Authentication + +**Purpose**: Use OS-level authentication for all database connections. + +**Mechanism**: Connection strings use an empty user field, triggering PostgreSQL's peer authentication which authenticates based on the OS user running the process (typically the `postgres` user). + +**Privileges**: +- Worker runs as the OS user that started PostgreSQL +- No custom database roles created +- Relies on PostgreSQL's built-in security model + +**Rationale**: +- Simpler security model +- No role management overhead +- Consistent with how PostgreSQL background workers typically operate +- Reduced attack surface + +#### 2.2. Schema Ownership + +**The `duroxide` Schema**: +- Created by `CREATE EXTENSION pg_durable` (via SQL script) +- Owner: The role that created the extension +- Contains duroxide internal tables (workflow state, history, etc.) +- Populated by background worker on first startup + +**The `df` Schema**: +- Created by pgrx-generated SQL during `CREATE EXTENSION` +- Contains user-facing tables and functions +- Owner: The role that created the extension + +#### 2.3. Connection Context + +**Single Connection Context**: +- User: OS user (via peer authentication) +- Database: Target database (from `pg_durable.database_name` GUC) +- Used for: All operations (duroxide state management, workflow orchestration, user function execution) +- Connection string: `postgresql:///?host={host}&port={port}&dbname={database_name}` + +**Note**: Empty user field in connection string triggers peer authentication. + +### 3. Connection String Construction + +**Single Helper Function**: + +```rust +pub fn postgres_connection_string() -> String { + let host = get_host(); + let port = get_port(); + let database = get_database_name(); + + format!("postgresql:///?host={}&port={}&dbname={}", host, port, database) +} +``` + +**Format**: +``` +postgresql:///?host={host}&port={port}&dbname={database_name} +``` + +**Parameter Sources**: +- `host`: `pg_durable.host` GUC (empty = PostgreSQL's default Unix socket dir) +- `port`: Read from `PostPortNumber` system GUC +- `dbname`: `pg_durable.database_name` GUC (default: "postgres") + +**Examples**: +```rust +// With default GUCs (empty host, "postgres" database) +postgres_connection_string() +// -> "postgresql:///?host=/tmp&port=5432&dbname=postgres" + +// With custom host and database +postgres_connection_string() +// -> "postgresql:///?host=/var/run/postgresql&port=5432&dbname=myapp_db" +``` + +**Note**: The `host` parameter with a directory path makes PostgreSQL/sqlx use Unix domain sockets. The `port` is required to identify the correct socket file (e.g., `.s.PGSQL.5432`). + +### 4. Background Worker Initialization Flow + +**Startup Sequence** (`duroxide_worker_main`): + +``` +1. Attach signal handlers +2. Initialize tracing +3. Read GUCs (pg_durable.database_name, pg_durable.host, PostPortNumber) +4. Initialize tokio runtime +5. [async] Enter wait loop: + a. Connect to target database (peer auth) + b. Check if pg_durable extension is created (query pg_extension) + c. Check if duroxide schema exists + d. If not ready: sleep 1 second, retry + e. Max 60 retries (1 minute timeout) + f. Once both conditions met: break wait loop +6. [async] Create sqlx connection pool (peer auth) +7. [async] Initialize duroxide runtime with PostgresProvider + - PostgresProvider creates duroxide tables if they don't exist + - Worker populates the duroxide schema created by extension +8. [async] Enter main loop (check for shutdown signals, execute workflows) +9. Shutdown tokio runtime +``` + +**Error Handling**: +- Extension not created within timeout: Log error and exit +- Schema doesn't exist: Log error and exit +- Duroxide initialization failure: Log error and exit +- PostgreSQL will automatically restart the worker + +**Key Design Decision**: Extension creates empty schema, worker populates it with duroxide tables. This avoids mixing synchronous SQL with async sqlx during extension creation. + +### 5. Extension Creation (`CREATE EXTENSION pg_durable`) + +**Extension Creation Flow**: + +```sql +-- In sql/pg_durable--0.1.0.sql (or pgrx-generated SQL) + +-- 1. Verify current database matches pg_durable.database_name GUC +DO $$ +DECLARE + current_db TEXT := current_database(); + target_db TEXT; +BEGIN + SELECT setting INTO target_db FROM pg_settings WHERE name = 'pg_durable.database_name'; + IF target_db IS DISTINCT FROM current_db THEN + RAISE EXCEPTION 'Cannot create pg_durable extension in database %. Expected database: %. Set pg_durable.database_name or create extension in the correct database.', + current_db, target_db; + END IF; +END $$; + +-- 2. Verify pg_durable is in shared_preload_libraries +DO $$ +BEGIN + IF NOT EXISTS( + SELECT 1 FROM pg_settings + WHERE name = 'shared_preload_libraries' + AND setting LIKE '%pg_durable%' + ) THEN + RAISE EXCEPTION 'pg_durable must be loaded via shared_preload_libraries. Add to postgresql.conf and restart PostgreSQL.'; + END IF; +END $$; + +-- 3. Create duroxide schema (empty - worker will populate) +CREATE SCHEMA IF NOT EXISTS duroxide; + +-- 4. Create df schema and tables (pgrx-generated SQL) +-- ... rest of extension SQL ... +``` + +**Responsibilities**: +- Validate GUC configuration +- Create empty `duroxide` schema +- Create `df` schema with user-facing tables and functions +- Validation ensures worker will be able to connect successfully + +**Worker's Responsibility**: +- Wait for extension creation +- Connect to database with duroxide schema +- Initialize duroxide-pg-opt runtime (creates duroxide tables) +- Execute durable workflows + +**Rationale**: +- Extension creation is fast and synchronous +- Worker handles async initialization +- Clean separation of concerns +- DROP EXTENSION CASCADE properly removes everything + +### 6. Error Cases and Diagnostics + +**Common Error Scenarios**: + +1. **Extension created in wrong database**: + - Error during `CREATE EXTENSION` (validation catches this) + - Message: "Cannot create pg_durable extension in database X. Expected database: Y" + +2. **Extension created before shared_preload_libraries configured**: + - Error during `CREATE EXTENSION` (validation catches this) + - Message: "pg_durable must be loaded via shared_preload_libraries" + +3. **Worker times out waiting for extension**: + - Error in worker logs after 60 seconds + - Worker exits and PostgreSQL restarts it + - Check: Was extension created? Is database name correct? + +4. **Connection failures**: + - Check: Is socket directory accessible? + - Check: Does OS user have permission? + - Check: Is PostgreSQL running? + +### 7. Testing Strategy + +#### Unit Tests +- Test `postgres_connection_string()` with various GUC combinations +- Test GUC default values +- Test PostPortNumber reading + +#### E2E Tests + +**Test 1: Fresh Installation** +- Start PostgreSQL with pg_durable in shared_preload_libraries +- Verify worker starts and waits +- Create extension in correct database +- Verify worker detects extension and initializes +- Run durable function + +**Test 2: Wrong Database** +- Try creating extension in database different from `pg_durable.database_name` +- Verify error message during `CREATE EXTENSION` + +**Test 3: Missing shared_preload_libraries** +- Try creating extension without shared_preload_libraries +- Verify error message during `CREATE EXTENSION` + +**Test 4: Custom Socket Directory** +- Set `pg_durable.host = '/custom/socket/dir'` +- Verify connections work + +**Test 5: Extension Ownership** +- Create extension +- Verify duroxide schema exists +- Verify `DROP EXTENSION pg_durable CASCADE` removes duroxide schema + +#### Manual Testing +- Test on fresh PostgreSQL instance +- Test with multiple databases +- Verify with `lsof` that Unix sockets are used +- Check process owner matches PostgreSQL server user + +## Migration Path + +### For New Installations + +1. **Configure postgresql.conf**: + ``` + shared_preload_libraries = 'pg_durable' + pg_durable.database_name = 'postgres' # or your database + pg_durable.host = '' # or custom socket directory + ``` + +2. **Restart PostgreSQL** + +3. **Create extension** (in target database): + ```sql + CREATE EXTENSION pg_durable; + ``` + +4. **Verify** worker is running: + - Check PostgreSQL logs for "pg_durable background worker initialized" + - Run a test durable function + +### For Existing Installations (from older versions) + +Users upgrading from versions with `durable_worker` role will need to: + +1. **Update postgresql.conf**: + ``` + # Old GUCs (remove): + # pg_durable.socket_dir = '...' + # pg_durable.database = '...' + + # New GUCs: + shared_preload_libraries = 'pg_durable' + pg_durable.host = '' # or custom path + pg_durable.database_name = 'postgres' # or your database + ``` + +2. **Restart PostgreSQL** + +3. **Drop and recreate extension** (in target database): + ```sql + -- Warning: This will cancel running orchestrations + DROP EXTENSION pg_durable CASCADE; + CREATE EXTENSION pg_durable; + ``` + +4. **Optional: Remove old role** (if it exists): + ```sql + DROP ROLE IF EXISTS durable_worker; + ``` + +## Security Considerations + +1. **Principle of Least Privilege**: Worker runs as PostgreSQL's OS user with standard permissions +2. **Unix Sockets Only**: Network connections are not supported, reducing attack surface +3. **Peer Authentication**: Leverages PostgreSQL's built-in OS-level authentication +4. **Schema Isolation**: Duroxide internal tables are isolated in a dedicated schema +5. **No Custom Roles**: Simpler security model, easier to audit and maintain + +## Implementation Details + +### Key Files Modified + +- `src/lib.rs`: GUC definitions (`pg_durable.host`, `pg_durable.database_name`) +- `src/types.rs`: Connection string helper (`postgres_connection_string()`) +- `src/worker.rs`: Worker initialization and wait loop +- `sql/pg_durable--0.1.0.sql`: Extension creation with validation (if using manual SQL) + +### Constants + +```rust +// src/types.rs +pub const DUROXIDE_SCHEMA: &str = "duroxide"; +``` + +### Helper Functions + +```rust +// src/types.rs +pub fn get_host() -> String; +pub fn get_port() -> u16; +pub fn get_database_name() -> String; +pub fn postgres_connection_string() -> String; +``` + +## Known Limitations + +1. **Single Database**: Only one target database per PostgreSQL instance +2. **Unix Sockets Only**: Windows support would require additional work +3. **Extension Drop**: Running orchestrations may error during `DROP EXTENSION CASCADE` +4. **No Live Migration**: Upgrades require extension recreation (loses running workflows) + +## Future Enhancements + +1. **Multi-Database Support**: Allow pg_durable in multiple databases +2. **Graceful Shutdown**: Handle `DROP EXTENSION` more gracefully +3. **User Context Preservation**: Track and use the role that called `df.start()` for function execution +4. **Network Socket Support**: Optional TCP connections for remote management + +## References + +- PostgreSQL GUC documentation: https://www.postgresql.org/docs/current/runtime-config-custom.html +- PostgreSQL background workers: https://www.postgresql.org/docs/current/bgworker.html +- PostgreSQL peer authentication: https://www.postgresql.org/docs/current/auth-peer.html +- sqlx connection strings: https://docs.rs/sqlx/latest/sqlx/postgres/struct.PgConnectOptions.html diff --git a/scripts/pg-start.sh b/scripts/pg-start.sh index 3006f25f..d4bf56e3 100755 --- a/scripts/pg-start.sh +++ b/scripts/pg-start.sh @@ -1,10 +1,17 @@ #!/bin/bash # pg-start.sh - Start local PostgreSQL with pg_durable extension # -# Usage: ./scripts/pg-start.sh +# Usage: ./scripts/pg-start.sh [database_name] +# database_name: Optional value for pg_durable.database_name GUC set -e +# Parse optional database_name parameter +DATABASE_GUC="" +if [ -n "$1" ]; then + DATABASE_GUC="$1" +fi + SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" PROJECT_DIR="$(cd "$SCRIPT_DIR/.." && pwd)" DATA_DIR="$HOME/.pgrx/data-17" @@ -27,6 +34,14 @@ if [ -f "$PG_CONF" ]; then echo -e "\033[0;33mConfiguring shared_preload_libraries...\033[0m" echo "shared_preload_libraries = 'pg_durable'" >> "$PG_CONF" fi + + # Configure pg_durable.database_name GUC if provided + if [ -n "$DATABASE_GUC" ]; then + # Remove any existing pg_durable.database_name setting (portable sed -i usage) + sed -i.bak '/^pg_durable\.database_name/d' "$PG_CONF" && rm -f "$PG_CONF.bak" + echo -e "\033[0;33mSetting pg_durable.database_name = '$DATABASE_GUC'...\033[0m" + echo "pg_durable.database_name = '$DATABASE_GUC'" >> "$PG_CONF" + fi fi echo -e "\033[0;33mStarting PostgreSQL...\033[0m" @@ -47,6 +62,11 @@ done VERSION=$(~/.pgrx/17.7/pgrx-install/bin/psql -h localhost -p 28817 -d postgres -t -c "SELECT df.version();" 2>/dev/null | tr -d ' \n') echo -e "\033[0;32mPostgreSQL started with pg_durable $VERSION\033[0m" +# Show configured GUC if set +if [ -n "$DATABASE_GUC" ]; then + echo -e "\033[0;32mConfigured: pg_durable.database_name = '$DATABASE_GUC'\033[0m" +fi + echo "" echo -e "\033[0;36mConnect:\033[0m" echo " ~/.pgrx/17.7/pgrx-install/bin/psql -h localhost -p 28817 -d postgres" diff --git a/scripts/test-e2e-local.sh b/scripts/test-e2e-local.sh index 9c40c69f..0aa538c7 100755 --- a/scripts/test-e2e-local.sh +++ b/scripts/test-e2e-local.sh @@ -126,6 +126,8 @@ ensure_config() { sed -i.bak '/^#*port = /d' "$DATA_DIR/postgresql.conf" echo "port = $PG_PORT" >> "$DATA_DIR/postgresql.conf" fi + # Remove pg_durable.database_name (use default behavior: current database) + sed -i.bak '/^#*pg_durable\.database_name/d' "$DATA_DIR/postgresql.conf" fi } diff --git a/src/lib.rs b/src/lib.rs index 06e27a23..93885547 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -4,6 +4,7 @@ //! using the Duroxide runtime for persistence. use pgrx::prelude::*; +use std::ffi::CString; // Module declarations pub mod activities; @@ -21,15 +22,84 @@ pub use types::Durofut; ::pgrx::pg_module_magic!(name, version); +// ============================================================================ +// Configuration Parameters (GUCs) +// ============================================================================ + +/// Unix domain socket directory (host) for PostgreSQL connections +/// None means use PostgreSQL's default Unix socket directory +pub static HOST: pgrx::GucSetting> = >>::new(None); + +/// Database name to which the background worker connects +pub static DATABASE_NAME: pgrx::GucSetting> = + >>::new(Some(c"postgres")); + // ============================================================================ // Background Worker Registration // ============================================================================ #[pg_guard] pub extern "C-unwind" fn _PG_init() { + // Register configuration parameters + pgrx::GucRegistry::define_string_guc( + c"pg_durable.host", + c"Unix domain socket directory (host) for PostgreSQL connections", + c"Specify the directory containing PostgreSQL's Unix domain socket. Empty string uses PostgreSQL's default.", + &HOST, + pgrx::GucContext::Postmaster, + pgrx::GucFlags::default(), + ); + + pgrx::GucRegistry::define_string_guc( + c"pg_durable.database_name", + c"Database name to which the background worker connects", + c"The database where pg_durable metadata and duroxide runtime state are stored. Defaults to 'postgres'.", + &DATABASE_NAME, + pgrx::GucContext::Postmaster, + pgrx::GucFlags::default(), + ); + + // Don't start the background worker during pgrx tests to avoid database locking issues + #[cfg(not(feature = "pg_test"))] worker::register_background_worker(); } +// ============================================================================ +// Extension Initialization Functions +// ============================================================================ + +/// Validate that extension is created in the correct database +#[pg_extern(sql = r#" +DO $$ +DECLARE + current_db TEXT := current_database(); + target_db TEXT; +BEGIN + SELECT setting INTO target_db + FROM pg_settings + WHERE name = 'pg_durable.database_name'; + + IF current_db != COALESCE(target_db, 'postgres') THEN + RAISE EXCEPTION 'pg_durable extension must be created in database "%" (currently in "%"). Set pg_durable.database_name in postgresql.conf or create the extension in the correct database.', + COALESCE(target_db, 'postgres'), current_db; + END IF; +END $$; + +DO $$ +BEGIN + IF NOT EXISTS( + SELECT 1 FROM pg_settings + WHERE name = 'shared_preload_libraries' + AND setting LIKE '%pg_durable%' + ) THEN + RAISE EXCEPTION 'pg_durable must be in shared_preload_libraries. Add "shared_preload_libraries = ''pg_durable''" to postgresql.conf and restart PostgreSQL.'; + END IF; +END $$; + +CREATE SCHEMA duroxide; +"#)] +fn __validate_extension_requirements() {} + // ============================================================================ // Schema Declaration // ============================================================================ @@ -1461,6 +1531,9 @@ pub mod pg_test { #[must_use] pub fn postgresql_conf_options() -> Vec<&'static str> { - vec!["shared_preload_libraries = 'pg_durable'"] + vec![ + "shared_preload_libraries = 'pg_durable'", + "pg_durable.database_name = 'pgrx_tests'", + ] } } diff --git a/src/types.rs b/src/types.rs index 948a9b9f..c8d243f4 100644 --- a/src/types.rs +++ b/src/types.rs @@ -25,28 +25,77 @@ pub fn short_id() -> String { .collect() } -/// PostgreSQL connection string for the background worker and Duroxide runtime -pub fn postgres_connection_string() -> String { - let host = std::env::var("PGHOST").unwrap_or_else(|_| "127.0.0.1".to_string()); - let port = std::env::var("PGPORT").unwrap_or_else(|_| { - if let Ok(pgdata) = std::env::var("PGDATA") { - if pgdata.contains(".pgrx") { - "28817".to_string() - } else { - "5432".to_string() +/// Get the configured socket directory (host) from GUC +/// Returns empty string if not configured, allowing sqlx to use defaults +pub fn get_host() -> String { + if let Some(host) = crate::HOST.get() { + if let Ok(dir_str) = host.to_str() { + if !dir_str.is_empty() { + pgrx::log!("pg_durable: host from GUC: {}", dir_str); + return dir_str.to_string(); } - } else { - "28817".to_string() } - }); - let user = std::env::var("PGUSER") - .or_else(|_| std::env::var("USER")) - .unwrap_or_else(|_| "postgres".to_string()); - let database = std::env::var("POSTGRES_DB") - .or_else(|_| std::env::var("PGDATABASE")) - .unwrap_or_else(|_| "postgres".to_string()); - - format!("postgres://{user}@{host}:{port}/{database}") + } + + pgrx::log!("pg_durable: host not configured, using empty string (sqlx defaults)"); + String::new() +} + +/// Get the configured database name from GUC +pub fn get_database_name() -> String { + if let Some(db_name) = crate::DATABASE_NAME.get() { + if let Ok(name_str) = db_name.to_str() { + if !name_str.is_empty() { + pgrx::log!("pg_durable: database_name from GUC: {}", name_str); + return name_str.to_string(); + } + } + } + + pgrx::log!("pg_durable: database_name not configured, using default: postgres"); + "postgres".to_string() +} + +/// Get the PostgreSQL port from the PostPortNumber system GUC +pub fn get_port() -> String { + let port = unsafe { pgrx::pg_sys::PostPortNumber }; + pgrx::log!("pg_durable: port from PostPortNumber: {}", port); + port.to_string() +} + +/// Build a PostgreSQL connection string using Unix domain sockets with peer authentication +/// +/// Uses OS-level peer authentication (empty user field) and connects to the configured database. +/// +/// # Returns +/// Connection string in format: `postgresql:///?host={socket_dir}&port={port}&dbname={database}` +/// +/// # Examples +/// ```text +/// postgresql:///?host=/home/vscode/.pgrx&port=28817&dbname=postgres +/// postgresql:///?port=5432&dbname=mydb (when host is empty, uses default socket dir) +/// ``` +pub fn postgres_connection_string() -> String { + let host = get_host(); + let port = get_port(); + let database = get_database_name(); + + let mut conn_str = "postgresql:///?".to_string(); + + // Add socket directory if configured (otherwise sqlx will use defaults) + if !host.is_empty() { + conn_str.push_str(&format!("host={}&", host)); + } + + // Always add port (required for Unix socket file identification) + // Port identifies the specific socket file: .s.PGSQL.{port} + conn_str.push_str(&format!("port={}&", port)); + + conn_str.push_str(&format!("dbname={}", database)); + + pgrx::log!("pg_durable: connection string (peer auth): {}", conn_str); + + conn_str } /// Schema name for Duroxide internal tables diff --git a/src/worker.rs b/src/worker.rs index 95dc1b31..99367ab8 100644 --- a/src/worker.rs +++ b/src/worker.rs @@ -11,6 +11,7 @@ use std::time::Duration; use duroxide::runtime; use duroxide_pg_opt::PostgresProvider; use sqlx::postgres::PgPoolOptions; +use sqlx::PgPool; use tracing_subscriber::EnvFilter; use crate::registry::{create_activity_registry, create_orchestration_registry}; @@ -65,62 +66,82 @@ pub extern "C-unwind" fn duroxide_worker_main(_arg: pg_sys::Datum) { { Ok(rt) => rt, Err(e) => { - log!("pg_durable: failed to create tokio runtime: {}", e); + warning!("pg_durable: failed to create tokio runtime: {}", e); return; } }; rt.block_on(async { - run_duroxide_runtime().await; - }); - - rt.shutdown_timeout(Duration::from_secs(5)); - log!("pg_durable: duroxide background worker terminated cleanly"); -} + // Create shared connection pool early (used for health checks and activities) + let pg_conn_str = postgres_connection_string(); + let pg_pool = match create_shared_pool(&pg_conn_str).await { + Some(pool) => Arc::new(pool), + None => { + warning!("pg_durable: failed to create connection pool, worker exiting"); + return; + } + }; -/// Run the duroxide runtime with proper shutdown handling -async fn run_duroxide_runtime() { - log!("pg_durable: initializing duroxide runtime with PostgreSQL store..."); + log!("pg_durable: shared connection pool created"); - let pg_conn_str = postgres_connection_string(); - log!( - "pg_durable: connecting to PostgreSQL at {} (schema: {})", - pg_conn_str, - DUROXIDE_SCHEMA - ); + let mut duroxide_runtime: Option> = None; - // Retry connection with exponential backoff (useful for pg_regress tests where database may not exist yet) - let store = loop { - match PostgresProvider::new_with_schema(&pg_conn_str, Some(DUROXIDE_SCHEMA)).await { - Ok(s) => break Arc::new(s), - Err(e) => { - log!( - "pg_durable: failed to create PostgreSQL store (will retry): {}", - e - ); + loop { + // 1. Check for shutdown signal + let should_shutdown = tokio::task::spawn_blocking(is_shutdown_requested) + .await + .unwrap_or(false); - // Check for shutdown before retrying - if tokio::task::spawn_blocking(is_shutdown_requested) - .await - .unwrap_or(false) - { - log!("pg_durable: shutdown requested during connection retry, exiting"); - return; + if should_shutdown { + log!("pg_durable: shutdown signal received"); + if let Some(runtime) = duroxide_runtime.take() { + log!("pg_durable: shutting down duroxide runtime..."); + runtime.shutdown(Some(10_000)).await; + log!("pg_durable: duroxide runtime shutdown complete"); } + break; + } + + // 2. If runtime is initialized, check if we should stop it + if duroxide_runtime.is_some() && check_schema_or_tables_missing(&pg_pool).await { + log!("pg_durable: duroxide schema or tables dropped, stopping runtime..."); + duroxide_runtime + .take() + .unwrap() + .shutdown(Some(10_000)) + .await; + log!("pg_durable: duroxide runtime stopped"); + } - // Retry after 5 seconds - tokio::time::sleep(Duration::from_secs(5)).await; + // 3. If runtime is not initialized, check if we should start it + if duroxide_runtime.is_none() && check_schema_exists(&pg_pool).await { + log!("pg_durable: duroxide schema detected, initializing runtime..."); + duroxide_runtime = initialize_duroxide_runtime(pg_pool.clone()).await; + if duroxide_runtime.is_some() { + log!("pg_durable: duroxide runtime started, processing durable functions..."); + } } + + // 4. Sleep before next iteration + tokio::time::sleep(Duration::from_millis(1000)).await; } - }; - log!( - "pg_durable: PostgreSQL store created in schema '{}'", - DUROXIDE_SCHEMA - ); + pg_pool.close().await; + }); + + rt.shutdown_timeout(Duration::from_secs(5)); + log!("pg_durable: duroxide background worker terminated cleanly"); +} + +/// Create shared connection pool with special handling for test databases. +/// Returns None if database doesn't exist (except for regression databases which retry). +async fn create_shared_pool(pg_conn_str: &str) -> Option { + use crate::types::get_database_name; + + let database_name = get_database_name(); + let is_regression_db = database_name == "regression" || database_name == "contrib_regression"; - // Create connection pool with session variable marking workflow context - let pg_pool = loop { + loop { match PgPoolOptions::new() .max_connections(5) .after_connect(|conn, _meta| { @@ -132,58 +153,111 @@ async fn run_duroxide_runtime() { Ok(()) }) }) - .connect(&pg_conn_str) + .connect(pg_conn_str) .await { - Ok(pool) => { - log!("pg_durable: PostgreSQL connection pool created"); - break Arc::new(pool); - } + Ok(pool) => return Some(pool), Err(e) => { - log!( - "pg_durable: failed to create PostgreSQL pool (will retry): {}", - e - ); + // Check if this is a "database does not exist" error (SQLSTATE 3D000) + let is_db_not_exists = e + .as_database_error() + .and_then(|db_err| db_err.code()) + .map(|code| code == "3D000") + .unwrap_or(false); - // Check for shutdown before retrying - if tokio::task::spawn_blocking(is_shutdown_requested) - .await - .unwrap_or(false) - { - log!("pg_durable: shutdown requested during pool creation, exiting"); - return; + // Exit immediately only for non-regression databases that don't exist + if !is_regression_db && is_db_not_exists { + warning!("pg_durable: database '{}' does not exist", database_name); + return None; } - // Retry after 5 seconds - tokio::time::sleep(Duration::from_secs(5)).await; + // For all other errors (regression DB doesn't exist, auth failures, etc.), retry + warning!( + "pg_durable: failed to connect to database '{}': {}, retrying in 1s...", + database_name, + e + ); + tokio::time::sleep(Duration::from_secs(1)).await; } } - }; + } +} - // Create registries - let activities = create_activity_registry(pg_pool); - let orchestrations = create_orchestration_registry(); +/// Check if duroxide schema exists (used when waiting to initialize duroxide-pg) +/// Returns true if schema exists, false otherwise. +/// Only checks schema existence - tables will be created by PostgresProvider::new_with_schema(). +async fn check_schema_exists(pool: &PgPool) -> bool { + let result: Result = + sqlx::query_scalar("SELECT EXISTS(SELECT 1 FROM pg_namespace WHERE nspname = 'duroxide')") + .fetch_one(pool) + .await; - let duroxide_runtime = - runtime::Runtime::start_with_store(store.clone(), activities, orchestrations).await; + result.unwrap_or(false) +} - log!("pg_durable: duroxide runtime started, processing durable functions..."); +/// Check if duroxide schema or key tables are missing (used in runtime loop to detect drops) +/// Returns true if schema or tables are missing (indicating DROP EXTENSION CASCADE occurred). +/// This checks for both schema AND a key duroxide-pg table (executions) to handle the edge +/// case where DROP EXTENSION CASCADE + CREATE EXTENSION happens within the check interval. +async fn check_schema_or_tables_missing(pool: &PgPool) -> bool { + // Check if both schema AND a key table exist (duroxide.executions) + let exists: Result = sqlx::query_scalar( + "SELECT EXISTS( + SELECT 1 FROM pg_namespace n + JOIN pg_class c ON c.relnamespace = n.oid + WHERE n.nspname = 'duroxide' AND c.relname = 'executions' + )", + ) + .fetch_one(pool) + .await; - // Keep runtime alive until shutdown signal - loop { - tokio::time::sleep(Duration::from_millis(100)).await; + // Return true if missing (invert exists check) + // Default to false on error to avoid spurious shutdowns + !exists.unwrap_or(false) +} - let should_shutdown = tokio::task::spawn_blocking(is_shutdown_requested) - .await - .unwrap_or(false); +/// Initialize the duroxide runtime and return it. +/// Returns None if initialization fails. +async fn initialize_duroxide_runtime(pg_pool: Arc) -> Option> { + use crate::types::get_database_name; - if should_shutdown { - log!("pg_durable: shutdown signal received, stopping duroxide runtime..."); - break; + let pg_conn_str = postgres_connection_string(); + let database_name = get_database_name(); + + log!( + "pg_durable: connecting to PostgreSQL at {} (schema: {})", + pg_conn_str, + DUROXIDE_SCHEMA + ); + + // Create PostgreSQL store (fail fast if database doesn't exist) + let store = match PostgresProvider::new_with_schema(&pg_conn_str, Some(DUROXIDE_SCHEMA)).await { + Ok(s) => Arc::new(s), + Err(e) => { + warning!( + "pg_durable: failed to create PostgreSQL store for database '{}': {}", + database_name, + e + ); + warning!( + "pg_durable: worker will not retry. Ensure database '{}' exists and extension is created.", + database_name + ); + return None; } - } + }; + + log!( + "pg_durable: PostgreSQL store created in schema '{}'", + DUROXIDE_SCHEMA + ); + + // Create registries using the shared pool + let activities = create_activity_registry(pg_pool); + let orchestrations = create_orchestration_registry(); + + let duroxide_runtime = + runtime::Runtime::start_with_store(store.clone(), activities, orchestrations).await; - log!("pg_durable: initiating duroxide runtime shutdown..."); - duroxide_runtime.shutdown(Some(10_000)).await; - log!("pg_durable: duroxide runtime shutdown complete"); + Some(duroxide_runtime) } diff --git a/test/regress/Makefile b/test/regress/Makefile index 6a653ebe..7d070b2a 100644 --- a/test/regress/Makefile +++ b/test/regress/Makefile @@ -6,10 +6,27 @@ DATA = pg_durable--0.1.1.sql REGRESS = simple sequence variables parallel conditional # PostgreSQL configuration -PG_CONFIG ?= $(shell cargo pgrx info pg-config pg17) +# Check if PG_CONFIG is already set (e.g., from parent Makefile or environment) +# If not, try to get it from pgrx (for development) +ifndef PG_CONFIG + # Check if we have pg_config in PATH (standard install) + PG_CONFIG := $(shell which pg_config 2>/dev/null) + # If not found, try to get it from pgrx + ifeq ($(PG_CONFIG),) + PG_CONFIG := $(shell cargo pgrx info pg-config pg17 2>/dev/null) + endif +endif + PGXS := $(shell $(PG_CONFIG) --pgxs) -# Connection parameters for pgrx-managed PostgreSQL -REGRESS_OPTS = --host=localhost --port=28817 --load-extension=pg_durable +# Extract PGPORT from pg_config if not already set in environment +ifndef PGPORT + PGPORT := $(shell $(PG_CONFIG) --configure 2>/dev/null | grep -oE -- '--with-pgport=[0-9]+' | cut -d= -f2) + export PGPORT +endif + +# Connection parameters - use environment variables (PGHOST, PGPORT, PGUSER, PGDATABASE) +# Default to load-extension for pg_durable +REGRESS_OPTS = --load-extension=pg_durable include $(PGXS) diff --git a/test/regress/README.md b/test/regress/README.md index bf9a20f9..f59addc0 100644 --- a/test/regress/README.md +++ b/test/regress/README.md @@ -18,24 +18,23 @@ pg_durable has two test suites: cargo pgrx install --release --pg-config $(cargo pgrx info pg-config pg17) ``` -2. Start PostgreSQL with the test database environment variable: +2. Start PostgreSQL with the test database: ```bash - export PGDATABASE=contrib_regression - ./scripts/pg-start.sh + ./scripts/pg-start.sh contrib_regression ``` - **Important**: The `PGDATABASE` environment variable tells the background worker which database to connect to. pg_regress will create the `contrib_regression` database when tests run. + **Important**: This sets the `pg_durable.database_name` GUC to tell the background worker which database to connect to. pg_regress will create the `contrib_regression` database when tests run. ### Run all tests +As a one-liner from the repository root: ```bash -cd test/regress -make installcheck +PG_CONFIG=$(cargo pgrx info pg-config pg17) make installcheck ``` ### How Background Worker Connection Works -When PostgreSQL starts, the pg_durable background worker attempts to connect to the database specified by `PGDATABASE` (default: `postgres`). For pg_regress tests, this must be set to `contrib_regression`. +When PostgreSQL starts, the pg_durable background worker attempts to connect to the database specified by the `pg_durable.database_name` GUC (default: `postgres`). For pg_regress tests, this must be set to `contrib_regression`. **Retry logic:** If the database doesn't exist yet (common during startup), the worker retries the connection every 5 seconds until: - The database is created by pg_regress, OR diff --git a/tests/e2e/sql/25_extension_creation_security.sql b/tests/e2e/sql/25_extension_creation_security.sql index c04b1ffc..b0af4698 100644 --- a/tests/e2e/sql/25_extension_creation_security.sql +++ b/tests/e2e/sql/25_extension_creation_security.sql @@ -1,13 +1,58 @@ --- Test: Extension creation security +-- Test: Extension creation security and DROP CASCADE behavior -- Tests that: --- 1. Non-superuser cannot create the extension --- 2. Extension creation fails if 'df' schema is pre-created --- Expected: Both security conditions are enforced +-- 1. Extension exists before drop +-- 2. DROP EXTENSION CASCADE removes all schemas (df, duroxide) and objects +-- 3. Non-superuser cannot create the extension +-- 4. Extension creation fails if 'df' schema is pre-created +-- 5. Extension creation fails if 'duroxide' schema is pre-created +-- 6. Extension can be recreated after DROP CASCADE +-- 7. Background worker initializes duroxide-pg after recreation +-- 8. Worker is operational after recreation (verified by running a durable function) +-- 9. duroxide schema is owned by the extension +-- Expected: All security conditions and lifecycle operations work correctly -- Note: This test drops and recreates the extension to test installation security -- Any running instances will be lost, but E2E tests are self-contained + +-- ============================================================================ +-- Verify extension exists before drop +-- ============================================================================ + +DO $$ +BEGIN + IF NOT EXISTS (SELECT 1 FROM pg_extension WHERE extname = 'pg_durable') THEN + RAISE EXCEPTION 'TEST FAILED: Extension should exist at test start'; + END IF; + RAISE NOTICE 'PASS: Extension exists before drop'; +END $$; + +-- ============================================================================ +-- Drop extension and verify cleanup +-- ============================================================================ + DROP EXTENSION IF EXISTS pg_durable CASCADE; +-- Wait for background worker to detect schema removal and shut down gracefully +-- This prevents race conditions in CI where the worker might be mid-operation +SELECT pg_sleep(2); + +-- Verify extension and schemas (df, duroxide) are gone +DO $$ +BEGIN + IF EXISTS (SELECT 1 FROM pg_extension WHERE extname = 'pg_durable') THEN + RAISE EXCEPTION 'TEST FAILED: Extension still exists after DROP'; + END IF; + + IF EXISTS (SELECT 1 FROM pg_namespace WHERE nspname = 'df') THEN + RAISE EXCEPTION 'TEST FAILED: df schema still exists after DROP CASCADE'; + END IF; + + IF EXISTS (SELECT 1 FROM pg_namespace WHERE nspname = 'duroxide') THEN + RAISE EXCEPTION 'TEST FAILED: duroxide schema still exists after DROP CASCADE'; + END IF; + RAISE NOTICE 'PASS: Extension and df+duroxide schemas removed'; +END $$; + -- ============================================================================ -- Test 1: Non-superuser cannot create extension -- ============================================================================ @@ -77,6 +122,42 @@ END $$; -- Clean up the pre-created schema DROP SCHEMA IF EXISTS df CASCADE; +-- ============================================================================ +-- Test 3: Extension creation fails if 'duroxide' schema pre-exists +-- ============================================================================ + +-- Create the 'duroxide' schema before attempting extension creation +CREATE SCHEMA IF NOT EXISTS duroxide; + +-- Attempt to create extension with pre-existing duroxide schema (should fail) +DO $$ +DECLARE + extension_created BOOLEAN := FALSE; +BEGIN + -- This should fail because the duroxide schema already exists + BEGIN + CREATE EXTENSION pg_durable; + extension_created := TRUE; + EXCEPTION + WHEN duplicate_schema THEN + RAISE NOTICE 'TEST 3 PASSED: Extension creation correctly prevented with pre-existing duroxide schema'; + WHEN OTHERS THEN + IF SQLERRM ILIKE '%schema%' OR SQLERRM ILIKE '%already exists%' OR SQLERRM ILIKE '%duroxide%' THEN + RAISE NOTICE 'TEST 3 PASSED: Extension creation correctly prevented with pre-existing duroxide schema (%)' , SQLERRM; + ELSE + RAISE EXCEPTION 'TEST 3 FAILED: Unexpected error during extension creation: %', SQLERRM; + END IF; + END; + + -- If we get here and extension was created, that's a security failure + IF extension_created THEN + RAISE EXCEPTION 'SECURITY FAILURE: Extension created successfully even with pre-existing duroxide schema!'; + END IF; +END $$; + +-- Clean up the pre-created duroxide schema +DROP SCHEMA IF EXISTS duroxide CASCADE; + -- ============================================================================ -- Restore extension for remaining tests -- ============================================================================ @@ -84,34 +165,103 @@ DROP SCHEMA IF EXISTS df CASCADE; -- Recreate the extension properly for other tests to continue CREATE EXTENSION pg_durable; --- Wait a moment for background worker to initialize +-- Wait for background worker to initialize duroxide-pg tables +DO $$ +DECLARE + table_count INT; + attempts INT := 0; +BEGIN + LOOP + SELECT COUNT(*) INTO table_count + FROM pg_tables + WHERE schemaname = 'duroxide' + AND tablename IN ('executions', 'instances', 'history', 'orchestrator_queue', 'worker_queue'); + + EXIT WHEN table_count = 5 OR attempts > 150; + PERFORM pg_sleep(0.1); + attempts := attempts + 1; + END LOOP; + + IF table_count != 5 THEN + RAISE EXCEPTION 'TEST SETUP FAILED: Worker did not initialize duroxide-pg, found % of 5 expected tables', table_count; + END IF; + + RAISE NOTICE 'PASS: Worker initialized duroxide-pg after recreation'; +END $$; + +-- Give the worker additional time to fully complete initialization +-- This avoids race conditions with migration conflicts when client connects SELECT pg_sleep(1); --- Verify extension is properly installed +-- Verify extension schemas and ownership DO $$ DECLARE - schema_exists BOOLEAN; - extension_exists BOOLEAN; + df_exists BOOLEAN; + duroxide_exists BOOLEAN; + duroxide_owned BOOLEAN; BEGIN -- Check that df schema exists SELECT EXISTS( SELECT 1 FROM pg_namespace WHERE nspname = 'df' - ) INTO schema_exists; + ) INTO df_exists; - -- Check that extension exists + -- Check that duroxide schema exists SELECT EXISTS( - SELECT 1 FROM pg_extension WHERE extname = 'pg_durable' - ) INTO extension_exists; + SELECT 1 FROM pg_namespace WHERE nspname = 'duroxide' + ) INTO duroxide_exists; - IF NOT schema_exists THEN + -- Check that duroxide schema is owned by pg_durable extension + SELECT EXISTS( + SELECT 1 FROM pg_namespace n + JOIN pg_depend d ON d.objid = n.oid + JOIN pg_extension e ON d.refobjid = e.oid + WHERE n.nspname = 'duroxide' + AND e.extname = 'pg_durable' + AND d.deptype = 'e' + ) INTO duroxide_owned; + + IF NOT df_exists THEN RAISE EXCEPTION 'TEST SETUP FAILED: df schema not created after extension installation'; END IF; - IF NOT extension_exists THEN - RAISE EXCEPTION 'TEST SETUP FAILED: pg_durable extension not installed'; + IF NOT duroxide_exists THEN + RAISE EXCEPTION 'TEST SETUP FAILED: duroxide schema not created after extension installation'; END IF; - RAISE NOTICE 'Extension restored successfully'; + IF NOT duroxide_owned THEN + RAISE EXCEPTION 'TEST SETUP FAILED: duroxide schema not owned by pg_durable extension'; + END IF; + + RAISE NOTICE 'PASS: Extension restored with proper schema ownership'; END $$; +-- Verify worker is operational by running a simple durable function +CREATE TEMP TABLE _restore_test_state (instance_id TEXT); +INSERT INTO _restore_test_state +SELECT df.start('SELECT 1 as restore_test', 'test-extension-restoration'); + +DO $$ +DECLARE + inst_id TEXT; + status TEXT; + attempts INT := 0; +BEGIN + SELECT instance_id INTO inst_id FROM _restore_test_state; + + LOOP + SELECT s INTO status FROM df.status(inst_id) s; + EXIT WHEN lower(status) IN ('completed', 'failed', 'canceled') OR attempts > 300; + PERFORM pg_sleep(0.1); + attempts := attempts + 1; + END LOOP; + + IF lower(status) != 'completed' THEN + RAISE EXCEPTION 'TEST FAILED: Worker not operational after restoration, status = %', status; + END IF; + + RAISE NOTICE 'PASS: Worker operational after extension recreation'; +END $$; + +DROP TABLE _restore_test_state; + SELECT 'TEST PASSED' AS result; diff --git a/tests/e2e/sql/26_drop_create_loop.sql b/tests/e2e/sql/26_drop_create_loop.sql new file mode 100644 index 00000000..ebd5d632 --- /dev/null +++ b/tests/e2e/sql/26_drop_create_loop.sql @@ -0,0 +1,197 @@ +-- Test: Worker Restart After Drop +-- Tests that: +-- 1. After DROP EXTENSION CASCADE, worker waits for extension recreation +-- 2. Worker detects recreated extension and reinitializes +-- 3. System becomes operational again without PostgreSQL restart +-- 4. Multiple drop-recreate cycles work correctly + +-- This test verifies the worker's ability to handle multiple create-drop-create cycles + +-- Phase 1: Initial state - extension should exist +DO $$ +BEGIN + IF NOT EXISTS (SELECT 1 FROM pg_extension WHERE extname = 'pg_durable') THEN + RAISE EXCEPTION 'TEST FAILED: Extension should exist at test start'; + END IF; + RAISE NOTICE 'PASS: Initial extension exists'; +END $$; + +-- Phase 2: First extension drop-create cycle +DROP EXTENSION IF EXISTS pg_durable CASCADE; +CREATE EXTENSION pg_durable; +-- Wait for worker to initialize duroxide-pg tables +DO $$ +DECLARE + table_count INT; + attempts INT := 0; +BEGIN + RAISE NOTICE 'Drop-create cycle 1'; + LOOP + SELECT COUNT(*) INTO table_count + FROM pg_tables + WHERE schemaname = 'duroxide' + AND tablename IN ('executions', 'instances', 'history', 'orchestrator_queue', 'worker_queue'); + + EXIT WHEN table_count = 5 OR attempts > 150; + PERFORM pg_sleep(0.1); + attempts := attempts + 1; + END LOOP; + + IF table_count != 5 THEN + RAISE EXCEPTION 'TEST FAILED (cycle 1): Worker did not initialize duroxide-pg, found % of 5 expected tables', table_count; + END IF; + + RAISE NOTICE 'PASS: Worker initialized duroxide-pg after cycle 1'; +END $$; + +-- Give the worker additional time to fully complete initialization +-- This avoids race conditions with migration conflicts when client connects +SELECT pg_sleep(1); + +-- Verify operational with a simple durable function +CREATE TEMP TABLE _cycle1_state (instance_id TEXT); +INSERT INTO _cycle1_state +SELECT df.start('SELECT 1 as cycle1_test', 'test-worker-restart-cycle1'); + +DO $$ +DECLARE + inst_id TEXT; + status TEXT; + attempts INT := 0; +BEGIN + SELECT instance_id INTO inst_id FROM _cycle1_state; + + LOOP + SELECT s INTO status FROM df.status(inst_id) s; + EXIT WHEN lower(status) IN ('completed', 'failed', 'canceled') OR attempts > 300; + PERFORM pg_sleep(0.1); + attempts := attempts + 1; + END LOOP; + + IF lower(status) != 'completed' THEN + RAISE EXCEPTION 'TEST FAILED (cycle 1): Worker not operational, status = %', status; + END IF; + RAISE NOTICE 'PASS: Worker operational after cycle 1'; +END $$; + +DROP TABLE _cycle1_state; + +-- Phase 3: Second extension drop-create cycle +DROP EXTENSION IF EXISTS pg_durable CASCADE; +CREATE EXTENSION pg_durable; +-- Wait for worker to initialize duroxide-pg tables +DO $$ +DECLARE + table_count INT; + attempts INT := 0; +BEGIN + RAISE NOTICE 'Drop-create cycle 2'; + LOOP + SELECT COUNT(*) INTO table_count + FROM pg_tables + WHERE schemaname = 'duroxide' + AND tablename IN ('executions', 'instances', 'history', 'orchestrator_queue', 'worker_queue'); + + EXIT WHEN table_count = 5 OR attempts > 150; + PERFORM pg_sleep(0.1); + attempts := attempts + 1; + END LOOP; + + IF table_count != 5 THEN + RAISE EXCEPTION 'TEST FAILED (cycle 2): Worker did not initialize duroxide-pg, found % of 5 expected tables', table_count; + END IF; + + RAISE NOTICE 'PASS: Worker initialized duroxide-pg after cycle 2'; +END $$; + +-- Give the worker additional time to fully complete initialization +SELECT pg_sleep(1); + +-- Verify operational again +CREATE TEMP TABLE _cycle2_state (instance_id TEXT); +INSERT INTO _cycle2_state +SELECT df.start('SELECT 2 as cycle2_test', 'test-worker-restart-cycle2'); + +DO $$ +DECLARE + inst_id TEXT; + status TEXT; + attempts INT := 0; +BEGIN + SELECT instance_id INTO inst_id FROM _cycle2_state; + + LOOP + SELECT s INTO status FROM df.status(inst_id) s; + EXIT WHEN lower(status) IN ('completed', 'failed', 'canceled') OR attempts > 300; + PERFORM pg_sleep(0.1); + attempts := attempts + 1; + END LOOP; + + IF lower(status) != 'completed' THEN + RAISE EXCEPTION 'TEST FAILED (cycle 2): Worker not operational, status = %', status; + END IF; + RAISE NOTICE 'PASS: Worker operational after cycle 2'; +END $$; + +DROP TABLE _cycle2_state; + +-- Phase 4: Third extension drop-create cycle (to really prove it can handle multiple cycles) +DROP EXTENSION IF EXISTS pg_durable CASCADE; +CREATE EXTENSION pg_durable; +-- Wait for worker to initialize duroxide-pg tables +DO $$ +DECLARE + table_count INT; + attempts INT := 0; +BEGIN + RAISE NOTICE 'Drop-create cycle 3'; + LOOP + SELECT COUNT(*) INTO table_count + FROM pg_tables + WHERE schemaname = 'duroxide' + AND tablename IN ('executions', 'instances', 'history', 'orchestrator_queue', 'worker_queue'); + + EXIT WHEN table_count = 5 OR attempts > 150; + PERFORM pg_sleep(0.1); + attempts := attempts + 1; + END LOOP; + + IF table_count != 5 THEN + RAISE EXCEPTION 'TEST FAILED (cycle 3): Worker did not initialize duroxide-pg, found % of 5 expected tables', table_count; + END IF; + + RAISE NOTICE 'PASS: Worker initialized duroxide-pg after cycle 3'; +END $$; + +-- Give the worker additional time to fully complete initialization +SELECT pg_sleep(1); + +-- Verify operational one more time +CREATE TEMP TABLE _cycle3_state (instance_id TEXT); +INSERT INTO _cycle3_state +SELECT df.start('SELECT 3 as cycle3_test', 'test-worker-restart-cycle3'); + +DO $$ +DECLARE + inst_id TEXT; + status TEXT; + attempts INT := 0; +BEGIN + SELECT instance_id INTO inst_id FROM _cycle3_state; + + LOOP + SELECT s INTO status FROM df.status(inst_id) s; + EXIT WHEN lower(status) IN ('completed', 'failed', 'canceled') OR attempts > 300; + PERFORM pg_sleep(0.1); + attempts := attempts + 1; + END LOOP; + + IF lower(status) != 'completed' THEN + RAISE EXCEPTION 'TEST FAILED (cycle 3): Worker not operational, status = %', status; + END IF; + RAISE NOTICE 'PASS: Worker operational after cycle 3'; +END $$; + +DROP TABLE _cycle3_state; + +SELECT 'TEST PASSED: Worker restart after multiple drop-create cycles verified' AS result;