diff --git a/CHANGELOG.md b/CHANGELOG.md index 483db67c..d860dceb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,6 +20,8 @@ Pre-1.0 note: while `pg_durable` is in major version `0`, minor releases may inc `continue_on_failure` syntax is experimental and may change in future releases. +- **`pg_durable.http_allowed_domains` (#375):** a restart-only GUC that replaces the HTTP and multipart domain allow-list with exact hostnames and `*.domain` patterns. Existing build-dependent defaults are preserved; an explicit empty list denies all domains in restricted builds. Other HTTP feature gates and safeguards are unchanged. + ### Changed - **Loop lifetime:** raises the loop-iteration backstop from 100,000 to diff --git a/README.md b/README.md index 8731924f..5c0c8600 100644 --- a/README.md +++ b/README.md @@ -107,7 +107,7 @@ Tagged releases publish Debian packages for PostgreSQL 17 and 18 on amd64 from t Tagged releases also publish a ready-to-run Docker image (`linux/amd64`) for PostgreSQL 17 and 18 to GitHub Container Registry: `ghcr.io/microsoft/pg_durable`. The image installs the released Debian package on top of the official `postgres` image. Each release publishes immutable `X.Y.Z-pg` and `vX.Y.Z-pg` tags (for example `0.2.2-pg17`, `0.2.2-pg18`); the highest stable release additionally updates the floating `pg` tags, and the default major (`pg17`) also updates `latest`. The PG major version is part of every tag so multiple PostgreSQL versions can be published alongside each other. Browse all published images and tags at . -> **Warning:** The published Docker image is intended for **evaluating and learning pg_durable only — do not use it in production.** It enables superuser durable instances for a frictionless out-of-the-box demo. Its HTTP egress policy is whatever the released Debian package was built with (`http-allow-azure-domains` — Azure domains only). Multi-arch (`linux/arm64`) images are not published yet; they will follow once arm64 Debian packages are available. +> **Warning:** The published Docker image is intended for **evaluating and learning pg_durable only — do not use it in production.** It enables superuser durable instances for a frictionless out-of-the-box demo. Its HTTP egress policy uses the released Debian package's `http-allow-azure-domains` tier, defaulting to Azure service subdomains and `api.github.com`. See [HTTP allowed domains](USER_GUIDE.md#http-allowed-domains) for configuration in v0.2.8+. Multi-arch (`linux/arm64`) images are not published yet; they will follow once arm64 Debian packages are available. Run the published image — PostgreSQL 17 and 18 can run side by side on different host ports: diff --git a/USER_GUIDE.md b/USER_GUIDE.md index 8dc3ce00..a5f17b19 100644 --- a/USER_GUIDE.md +++ b/USER_GUIDE.md @@ -24,10 +24,11 @@ pg_durable is a PostgreSQL extension that brings durable, fault-tolerant functio 14. [Monitoring](#monitoring) 15. [User Isolation & Privileges](#user-isolation--privileges) 16. [Connection Host](#connection-host) -17. [Connection Limits](#connection-limits) -18. [Troubleshooting](#troubleshooting) -19. [Quick Reference Card](#quick-reference-card) -20. [Appendix: Test Data Setup](#appendix-test-data-setup) +17. [HTTP Allowed Domains](#http-allowed-domains) +18. [Connection Limits](#connection-limits) +19. [Troubleshooting](#troubleshooting) +20. [Quick Reference Card](#quick-reference-card) +21. [Appendix: Test Data Setup](#appendix-test-data-setup) --- @@ -2168,12 +2169,24 @@ If the user who submitted a function is dropped **before execution**: #### HTTP Requests -HTTP requests (`df.http()`) currently execute with the **background worker's privileges**, not the submitting user's privileges: - -- All users can make HTTP requests to the same endpoints -- No user-specific URL allowlists - -**Security model:** For pg_durable's built-in `df.http()` activity, outbound HTTP is controlled by compile-time Cargo features and is off by default. When enabled, a hardcoded SSRF IP blocklist and domain allow-list are enforced — all `df.http()` requests to private/reserved IP ranges are blocked and only approved Azure service domains are permitted (e.g. `*.blob.core.windows.net`, `*.openai.azure.com`). These `df.http()` restrictions cannot be bypassed by any database user, including superusers. They do not restrict arbitrary SQL functions, user-defined functions, or third-party Postgres extensions that a workflow role can execute from SQL nodes; administrators must manage extension installation, function privileges, and network egress separately. See `docs/http-security.md` for the full security model and feature flag reference. +`df.http()` and `df.http_multipart()` share a server-wide destination policy. +Permission to use each function is checked for the submitting role, but there +are no per-role domain allowlists. + +**Security model:** Outbound HTTP availability and its security tier are +controlled by compile-time Cargo features; HTTP is off when no HTTP feature is +enabled. Restricted builds enforce a hardcoded SSRF IP blocklist and the +[HTTP domain allow-list](#http-allowed-domains), which defaults to Azure service +subdomains and `api.github.com`. Administrators can replace the domain list +with `pg_durable.http_allowed_domains` and restart PostgreSQL. This does not +relax the other restrictions or exempt superuser requests. Only the +development-only `http-allow-all` build bypasses domain and IP restrictions. + +These protections apply to the built-in HTTP activities, not arbitrary SQL +functions, user-defined functions, or third-party Postgres extensions that a +workflow role can execute from SQL nodes. Administrators must manage extension +installation, function privileges, and network egress separately. See +[HTTP security](docs/http-security.md) for the full security model. **Future:** Per-user HTTP isolation and URL allowlists are planned. @@ -2361,6 +2374,43 @@ This postmaster setting requires a PostgreSQL restart. When it is empty or unset --- +## HTTP Allowed Domains + +Since v0.2.8, administrators can replace the destination allow-list for +`df.http()` and `df.http_multipart()` in restricted HTTP builds: + +```ini +# postgresql.conf +pg_durable.http_allowed_domains = 'api.github.com, *.blob.core.windows.net' +``` + +This server-wide **Postmaster-context** setting requires a PostgreSQL restart. +It can also be configured through an authorized `ALTER SYSTEM SET`; a reload +alone does not apply it. All users can inspect the active list with +`SHOW pg_durable.http_allowed_domains`, but sessions and roles cannot override it. + +Use comma-separated hostnames. `api.example.com` matches only that host; +`*.example.com` matches subdomains at any depth, but not `example.com` itself. +Whitespace around entries is ignored, matching is case-insensitive, and +internationalized hostnames can use UTF-8 or ASCII/Punycode. Do not include +URLs, ports, IP addresses, or trailing dots. + +**The configured list replaces all defaults.** Without an override, +`http-allow-azure-domains` permits the existing Azure service subdomains and +`api.github.com`; `http-allow-test-domains` also permits `httpbingo.org`. +An empty list (`''`) denies every domain in restricted builds. Malformed lists +are rejected as a whole; an invalid startup value prevents PostgreSQL from +starting rather than silently restoring defaults. + +This setting does not enable HTTP in a build without HTTP support, and +`http-allow-all` continues to bypass it even when it is empty. The HTTPS +requirement, IP blocklist, proxy and redirect restrictions, and HTTP function +privileges are unchanged. After restart, pending requests and retries use the +new list. See [HTTP security](docs/http-security.md#5-layer-2-endpoint-allow-list) +for the full syntax and default domain list. + +--- + ## Connection Limits pg_durable uses multiple PostgreSQL connections for different purposes. Four GUCs let you control the connection budget to match your deployment's resources. diff --git a/docs/api-reference.md b/docs/api-reference.md index 325c5f50..794cb926 100644 --- a/docs/api-reference.md +++ b/docs/api-reference.md @@ -287,6 +287,10 @@ df.wait_for_signal('approval', 3600) -- 1 hour timeout Makes an HTTP request. +In restricted builds, the destination must be permitted by +[`pg_durable.http_allowed_domains`](#pg_durablehttp_allowed_domains). +The same policy applies to `df.http_multipart()`. + | Parameter | Type | Auto-wrap | Description | |-----------|------|-----------|-------------| | `url` | TEXT | ❌ Literal | Request URL (supports `$var` substitution) | @@ -734,6 +738,46 @@ These settings are configured via `ALTER SYSTEM SET` or `postgresql.conf`. See e --- +### pg_durable.http_allowed_domains + +The complete destination allow-list for `df.http()` and `df.http_multipart()` +in restricted builds. Available since v0.2.8. + +| Property | Value | +|----------|-------| +| Type | `string` | +| Default | Azure subdomain patterns and `api.github.com` with `http-allow-azure-domains`; also `httpbingo.org` with `http-allow-test-domains`; empty otherwise | +| Context | `POSTMASTER` (requires a PostgreSQL restart, not just a reload) | +| Visibility | All users can read the active value | + +```ini +# postgresql.conf +pg_durable.http_allowed_domains = 'api.github.com, *.blob.core.windows.net' +``` + +Entries are comma-separated exact hostnames or `*.domain` patterns, with +optional surrounding whitespace. A pattern permits subdomains at any depth, +not the apex itself. Matching is case-insensitive and uses IDNA/Punycode +normalization. Use UTF-8 internationalized names or ASCII/Punycode. + +An explicit value **replaces all defaults**, including test domains. An empty +or whitespace-only value denies all domains in restricted builds. Malformed +entries reject the whole setting; a malformed startup value prevents server +startup. URLs, ports, IPs, CIDRs, percent escapes, trailing dots, standalone `*`, +and empty entries within a nonempty list are not accepted. + +Session, role, and database settings cannot override this policy. An +authorized `ALTER SYSTEM SET` can change the startup configuration, but +PostgreSQL must restart before requests use it. + +The GUC does not override Cargo feature gates: HTTP remains disabled in builds +without an HTTP feature, and `http-allow-all` bypasses the list even when it is +empty. Other HTTP safeguards are unchanged. See +[HTTP security](http-security.md#5-layer-2-endpoint-allow-list) for the default +domains and execution-time behavior. + +--- + ### pg_durable.enable_superuser_instances Controls whether pg_durable allows durable function instances whose `submitted_by` role is a PostgreSQL superuser. diff --git a/docs/http-security.md b/docs/http-security.md index f4f7f9d5..10228bee 100644 --- a/docs/http-security.md +++ b/docs/http-security.md @@ -1,8 +1,8 @@ # HTTP Security in pg_durable -This document describes the security model for `df.http()` — the durable HTTP -activity that lets workflows make outbound HTTP(S) requests from within the -PostgreSQL background worker. +This document describes the security model for `df.http()` and +`df.http_multipart()`, which make outbound HTTP(S) requests from within the +PostgreSQL background worker. Both use the same destination policy. --- @@ -22,22 +22,23 @@ PostgreSQL background worker. ## 1. Feature Flags -Outbound HTTP access is controlled entirely by Cargo features at build time. -The database cannot override these choices — they cannot be changed with GUCs -or SQL. +Cargo features select the outbound HTTP security tier at build time. +In restricted builds, administrators can replace the domain allow-list with +`pg_durable.http_allowed_domains` and restart PostgreSQL. That setting cannot +enable HTTP in a disabled build or change the other protections. | Feature | What is allowed | Use case | |---------|-----------------|----------| | *(none)* | Nothing — `df.http()` errors immediately at DSL time **and** at execution time | Deployments that don't need HTTP | -| `http-allow-azure-domains` | HTTPS to subdomains of the Azure allow-list plus `api.github.com`; bare IPs blocked; redirects blocked | Production | -| `http-allow-test-domains` | HTTPS to everything in `http-allow-azure-domains` **plus** `httpbingo.org` | E2E testing; implies `http-allow-azure-domains` | +| `http-allow-azure-domains` | HTTPS to configured domains, defaulting to Azure subdomains plus `api.github.com`; bare IPs blocked; redirects blocked | Production | +| `http-allow-test-domains` | Same restrictions; the default list also includes `httpbingo.org` | E2E testing; implies `http-allow-azure-domains` | | `http-allow-all` | HTTP and HTTPS to all URLs; SSRF IP blocklist and allow-list are both disabled | Local development only | The scripts and CI use `http-allow-test-domains` so that the HTTP E2E tests pass — this includes the source-built `Dockerfile` used for local dev and CI. The released Debian packages are built with `http-allow-azure-domains`, so the published Docker image (`Dockerfile.release`, which installs that package) -inherits the `http-allow-azure-domains` policy. +inherits the `http-allow-azure-domains` tier and defaults. ### When no feature is set @@ -84,8 +85,10 @@ block is enforced again at execution time inside `execute_http.rs` via └──────────────────────────────────────────────────────────┘ ``` -All three layers run inside `execute_http.rs` before the request is sent. -There is no GUC, no table override, no superuser bypass for Layers 1 and 2. +Both HTTP activities enforce these layers before sending a request. +The domain allow-list is configurable at server startup; the IP blocklist is +not. Neither has a per-session or per-role override, and superuser HTTP requests +are subject to the same destination policy. --- @@ -239,42 +242,89 @@ would create false positives without any security benefit. Bare IP literals in URLs (e.g. `http://169.254.169.254/...`) bypass DNS entirely — `reqwest` connects directly without calling the resolver. -`validate_allowlist` blocks all bare IPs unconditionally, so these never -reach the resolver. +`validate_allowlist` blocks all bare IPs in restricted builds, so these never +reach the resolver. Only the development-only `http-allow-all` feature bypasses +this rule. --- ## 5. Layer 2: Endpoint Allow-List -### 5.1 Azure domains (always present with `http-allow-azure-domains`) +### 5.1 Configuring allowed domains -Only subdomains of the following suffixes are permitted. Apex domains (e.g. -`blob.core.windows.net` without a subdomain label) are rejected. +Since v0.2.8, `pg_durable.http_allowed_domains` is the complete allow-list for +both HTTP activities in restricted builds. For example, in `postgresql.conf`: -| Suffix | Service | +```ini +pg_durable.http_allowed_domains = 'api.github.com, *.blob.core.windows.net' +``` + +An exact hostname allows only that host. `*.example.com` allows subdomains at +any depth, such as `a.example.com` and `a.b.example.com`, but not `example.com` +itself. Add a separate exact entry to allow the apex. Matching is +case-insensitive and uses the same IDNA/Punycode representation as request URL +parsing. Use UTF-8 internationalized names or their ASCII/Punycode spelling. + +Separate entries with commas; whitespace around entries is ignored. Entries +must be DNS hostnames, optionally prefixed with `*.`. URLs, ports, IP addresses, +CIDRs, percent escapes, trailing dots, other wildcard forms, and empty entries +inside a nonempty list are rejected. A malformed value rejects the whole +setting, not just the offending entry. A malformed startup value prevents +PostgreSQL from starting rather than falling back to a potentially broader +default. + +**An explicit value replaces all defaults.** It does not implicitly retain +Azure, GitHub, or test domains. An empty or whitespace-only value denies all +domains in restricted builds: + +```ini +pg_durable.http_allowed_domains = '' +``` + +This is a **Postmaster-context** setting. Set it in `postgresql.conf` or through +an authorized `ALTER SYSTEM SET`, then restart PostgreSQL; a reload alone is +not enough. `SET`, `SET LOCAL`, and role/database settings cannot override it. +All users can inspect the active value with +`SHOW pg_durable.http_allowed_domains`. + +The restarted worker applies the new policy to requests it executes, including +pending activities and retries. Previously recorded activity results replay +normally. There is no per-workflow snapshot of the old policy. + +The setting does not relax HTTPS, IP blocking, proxy restrictions, redirects, +or function privileges. A hostname that resolves to a blocked IP is still +blocked. Builds without an HTTP feature remain disabled regardless of the +list; `http-allow-all` bypasses it, even when it is empty. + +### 5.2 Default Azure domains (`http-allow-azure-domains`) + +With no override, the following subdomain patterns are allowed. Apex domains +(e.g. `blob.core.windows.net`) require a separate exact entry. + +| Pattern | Service | |--------|---------| -| `.blob.core.windows.net` | Azure Blob Storage | -| `.blob.storage.azure.net` | Azure Blob Storage (secondary) | -| `.queue.core.windows.net` | Azure Queue Storage | -| `.table.core.windows.net` | Azure Table Storage | -| `.file.core.windows.net` | Azure Files | -| `.azurewebsites.net` | Azure App Service | -| `.azure-api.net` | Azure API Management | -| `.documents.azure.com` | Azure Cosmos DB | -| `.servicebus.windows.net` | Azure Service Bus | -| `.openai.azure.com` | Azure OpenAI | -| `.cognitiveservices.azure.com` | Azure Cognitive Services | -| `.vault.azure.net` | Azure Key Vault | -| `.redis.cache.windows.net` | Azure Cache for Redis | -| `.database.windows.net` | Azure SQL Database | -| `.kusto.windows.net` | Azure Data Explorer | -| `.azurefd.net` | Azure Front Door | -| `.azureedge.net` | Azure CDN | -| `.azure-devices.net` | Azure IoT Hub | -| `.trafficmanager.net` | Azure Traffic Manager | -| `.cloudapp.azure.com` | Azure Cloud App | - -### 5.2 Exact-match domains (always present with `http-allow-azure-domains`) +| `*.blob.core.windows.net` | Azure Blob Storage | +| `*.blob.storage.azure.net` | Azure Blob Storage (secondary) | +| `*.queue.core.windows.net` | Azure Queue Storage | +| `*.table.core.windows.net` | Azure Table Storage | +| `*.file.core.windows.net` | Azure Files | +| `*.azurewebsites.net` | Azure App Service | +| `*.azure-api.net` | Azure API Management | +| `*.documents.azure.com` | Azure Cosmos DB | +| `*.servicebus.windows.net` | Azure Service Bus | +| `*.openai.azure.com` | Azure OpenAI | +| `*.cognitiveservices.azure.com` | Azure Cognitive Services | +| `*.vault.azure.net` | Azure Key Vault | +| `*.redis.cache.windows.net` | Azure Cache for Redis | +| `*.database.windows.net` | Azure SQL Database | +| `*.kusto.windows.net` | Azure Data Explorer | +| `*.azurefd.net` | Azure Front Door | +| `*.azureedge.net` | Azure CDN | +| `*.azure-devices.net` | Azure IoT Hub | +| `*.trafficmanager.net` | Azure Traffic Manager | +| `*.cloudapp.azure.com` | Azure Cloud App | + +### 5.3 Default exact-match domains (`http-allow-azure-domains`) Matched exactly — subdomains and lookalikes are rejected. @@ -282,20 +332,20 @@ Matched exactly — subdomains and lookalikes are rejected. |--------|---------| | `api.github.com` | GitHub API | -### 5.3 Test domains (additional with `http-allow-test-domains`) +### 5.4 Additional default test domains (`http-allow-test-domains`) | Domain | Purpose | |--------|---------| | `httpbingo.org` | HTTP echo service (used in HTTP E2E tests) | -### 5.4 Bare IP rejection +### 5.5 Bare IP rejection -All bare IPv4 and IPv6 addresses are rejected by `validate_allowlist` -regardless of feature flag — even under `http-allow-azure-domains`. +All bare IPv4 and IPv6 addresses are rejected by `validate_allowlist` in +restricted builds, regardless of the configured domain list. Because the allowlist blocks all bare IPs, there is no separate IP-literal check; the allowlist is the definitive gate for IP-literal URLs. -### 5.4 One parse, one URL +### 5.6 One parse, one URL The host judged by the allow-list is read from the `url::Url` that is then handed to `reqwest`, never from the caller's string. Comparing a separately @@ -394,8 +444,9 @@ endpoint can echo them in its response. | HTTP disabled (no feature) | `Blocked: outbound HTTP requests are disabled. Rebuild with the 'http-allow-azure-domains' Cargo feature to enable them.` | | Plaintext HTTP in a restricted build | `Blocked: plaintext HTTP is not permitted in restricted builds. HTTPS is required.` | | Unsupported scheme | `Blocked: unsupported URL scheme. Only {allowed} is allowed.` where `{allowed}` is `https` in restricted builds or `http and https` with `http-allow-all` | -| Bare IP address | `Blocked: requests to bare IP addresses are not permitted. Use an approved Azure service hostname instead.` | -| Non-allowed domain | `Blocked: '{host}' is not in the allowed endpoint list. Only requests to approved Azure service domains are permitted.` | +| Bare IP address | `Blocked: requests to bare IP addresses are not permitted. Use an approved service hostname instead.` | +| Non-allowed domain | `Blocked: '{host}' is not in the allowed endpoint list. Configure pg_durable.http_allowed_domains to allow this hostname.` | +| Invalid domain-list configuration | `invalid value for parameter "pg_durable.http_allowed_domains"` with the offending entry and reason | | Blocked IP (literal or DNS) | `Blocked: the resolved IP address for '{host}' is in a restricted range. df.http() cannot access private or internal network addresses.` | | DSL-time (no feature) | `df.http() is disabled. Rebuild with the 'http-allow-azure-domains' Cargo feature to enable outbound HTTP requests.` | diff --git a/docs/upgrade-testing.md b/docs/upgrade-testing.md index 116d5c90..dfe2e3e3 100644 --- a/docs/upgrade-testing.md +++ b/docs/upgrade-testing.md @@ -205,6 +205,8 @@ what the upgrade script handles, and any backward compatibility considerations. ### 0.2.8 +#### Failure-isolated loops and loop lifetime + - `sql/pg_durable--0.2.7--0.2.8.sql` renames `df.loop(text, text)` to `df._loop_legacy(text, text)`, preserving its function OID and dependent objects, then creates the single public @@ -233,6 +235,13 @@ what the upgrade script handles, and any backward compatibility considerations. binary, which continues toward the higher backstop instead. Drain such long-running loops before upgrade when continuity is required. +#### Configurable HTTP domains (#375) +- **Runtime change (no DDL):** `pg_durable.http_allowed_domains` is a Postmaster-context string GUC that replaces the domain allow-list for both HTTP activities in restricted builds. The defaults preserve the existing Azure/GitHub policy, including `httpbingo.org` in test builds. Disabled and `http-allow-all` build behavior is unchanged. +- **Configuration migration:** None is required to retain existing behavior. Administrators can configure exact hostnames and `*.domain` patterns, then restart PostgreSQL. An explicit value replaces all defaults; an empty list denies all domains in restricted builds. Malformed values are rejected, including at server startup. +- **Scenario A/B2 considerations:** No upgrade-script DDL, schema changes, or data migration. Existing graphs, activity names, and serialized activity inputs are unchanged. +- **Scenario B1 considerations:** The new `.so` works against all previous supported schemas without `ALTER EXTENSION UPDATE` or runtime schema detection. The policy is read from the GUC, not extension tables. +- **Replay compatibility:** The policy is evaluated only when an HTTP activity executes, not by orchestration code. Pending requests and retries use the new list after restart; already-recorded activity results replay normally. + ### v0.2.6 → v0.2.7 #### Transaction-aware graph admission diff --git a/scripts/test-e2e-docker.sh b/scripts/test-e2e-docker.sh index 6bffdbf8..a29f490a 100755 --- a/scripts/test-e2e-docker.sh +++ b/scripts/test-e2e-docker.sh @@ -35,6 +35,8 @@ SKIP_TESTS=( "46_connection_limit_startup_validation" "66_new_transaction_launch_limit" "67_host_guc" + "69_http_allowed_domains" + "70_http_allowed_domains_empty" "47_http_dsl_disabled" "48_http_allow_all" # Needs the "reconcile" phase GUCs (reconcile_interval=2, retention_days=0) @@ -212,7 +214,7 @@ for run in $(seq 1 $REPEAT_COUNT); do fi # Skip tests that require a different PostgreSQL startup mode or - # restart-sensitive connection-limit GUC changes. + # restart-sensitive GUC changes. skip=false for skip_test in "${SKIP_TESTS[@]}"; do if [[ "$test_name" == "$skip_test" ]]; then diff --git a/scripts/test-e2e-local.sh b/scripts/test-e2e-local.sh index e90d83ed..345b5ea7 100755 --- a/scripts/test-e2e-local.sh +++ b/scripts/test-e2e-local.sh @@ -26,6 +26,7 @@ # ./scripts/test-e2e-local.sh --default-build-phases # ./scripts/test-e2e-local.sh 00_requires_shared_preload # ./scripts/test-e2e-local.sh 45_connection_limit_timeout +# ./scripts/test-e2e-local.sh http_allowed_domains # ./scripts/test-e2e-local.sh --http-disabled 47_http_dsl_disabled # ./scripts/test-e2e-local.sh --http-allow-all # END_USAGE @@ -63,6 +64,8 @@ DEFAULT_BUILD_PHASES=( "new-start-limit" "connlimit-startup" "reconcile" + "http-custom-domains" + "http-empty-domains" ) ALL_PHASES=( @@ -75,6 +78,8 @@ ALL_PHASES=( "new-start-limit" "connlimit-startup" "reconcile" + "http-custom-domains" + "http-empty-domains" "http-disabled" "http-allow-all" ) @@ -157,6 +162,12 @@ phase_label() { reconcile) echo "reconcile orphans" ;; + http-custom-domains) + echo "HTTP custom domain allowlist" + ;; + http-empty-domains) + echo "HTTP empty domain allowlist (deny all)" + ;; http-disabled) echo "HTTP disabled (no http Cargo feature)" ;; @@ -195,6 +206,12 @@ phase_for_test() { 54_reconcile_orphans) echo "reconcile" ;; + 69_http_allowed_domains) + echo "http-custom-domains" + ;; + 70_http_allowed_domains_empty) + echo "http-empty-domains" + ;; 47_http_dsl_disabled) echo "http-disabled" ;; @@ -366,6 +383,68 @@ restart_server() { wait_for_server } +assert_http_domains_startup_rejected() ( + # Scope the restoration trap to this probe, leaving the runner's EXIT trap intact. + config_backup=$(mktemp "$DATA_DIR/http-domains-startup.XXXXXX") || exit 1 + if ! cp "$CONF_FILE" "$config_backup"; then + rm -f -- "$config_backup" + exit 1 + fi + startup_log="$config_backup.log" + + restore_startup_config() { + result=$? + stop_server + if "$PG_CTL" status -D "$DATA_DIR" >/dev/null 2>&1; then + echo "TEST FAILED: could not stop PostgreSQL after the invalid-config probe" + result=1 + fi + if ! cp "$config_backup" "$CONF_FILE"; then + echo "TEST FAILED: could not restore $CONF_FILE; backup retained at $config_backup" + exit 1 + fi + rm -f -- "$config_backup" "$startup_log" || result=1 + exit "$result" + } + + trap restore_startup_config EXIT + trap 'exit 130' INT + trap 'exit 143' TERM + + stop_server + if "$PG_CTL" status -D "$DATA_DIR" >/dev/null 2>&1; then + echo "TEST FAILED: PostgreSQL must be stopped before the invalid-config probe" + exit 1 + fi + + # The last assignment wins. This must reach the preload check hook rather + # than ALTER SYSTEM validation or a postgresql.conf syntax error. + printf "\npg_durable.http_allowed_domains = 'example.com,https://api.github.com'\n" \ + >> "$CONF_FILE" || exit 1 + + failed=false + if startup_output=$("$PG_CTL" -D "$DATA_DIR" -l "$startup_log" -w -t 15 start 2>&1); then + echo "TEST FAILED: PostgreSQL accepted a malformed startup domain allowlist" + failed=true + fi + if "$PG_CTL" status -D "$DATA_DIR" >/dev/null 2>&1; then + echo "TEST FAILED: PostgreSQL is still running after the invalid-config startup" + failed=true + fi + if ! grep -Eq '(ERROR|FATAL):[[:space:]]+invalid value for parameter "pg_durable[.]http_allowed_domains"' "$startup_log"; then + echo "TEST FAILED: startup did not report the expected domain GUC error" + failed=true + fi + + if [ "$failed" = true ] || [ "$VERBOSE" = true ]; then + printf '%s\n' "$startup_output" + if [ -f "$startup_log" ]; then + tail -40 "$startup_log" + fi + fi + [ "$failed" = false ] +) + build_extension() { echo "Building and installing extension..." cd "$PROJECT_DIR" @@ -477,6 +556,7 @@ configure_phase() { clear_connlimit_gucs remove_conf_key "log_connections" remove_conf_key "pg_durable.host" + remove_conf_key "pg_durable.http_allowed_domains" # Match scripts/pg-common.sh so the shared pgrx cluster keeps a usable socket # directory for `make installcheck` after an E2E run. set_conf_line "unix_socket_directories" "'$PGRX_HOME'" @@ -548,17 +628,19 @@ configure_phase() { set_conf_line "pg_durable.reconcile_interval" "2" set_conf_line "pg_durable.retention_days" "0" ;; - http-disabled) + http-custom-domains|http-disabled) set_conf_line "shared_preload_libraries" "'pg_durable'" set_conf_line "pg_durable.worker_role" "'postgres'" set_conf_line "pg_durable.database" "'postgres'" set_conf_line "pg_durable.enable_superuser_instances" "on" + set_conf_line "pg_durable.http_allowed_domains" "'example.com'" ;; - http-allow-all) + http-empty-domains|http-allow-all) set_conf_line "shared_preload_libraries" "'pg_durable'" set_conf_line "pg_durable.worker_role" "'postgres'" set_conf_line "pg_durable.database" "'postgres'" set_conf_line "pg_durable.enable_superuser_instances" "on" + set_conf_line "pg_durable.http_allowed_domains" "''" ;; esac } @@ -575,7 +657,7 @@ prepare_phase() { http-allow-all) build_extension_http_allow_all ;; - no-preload|standard|host-guc|superuser-guc-off|connlimit-backpressure|connlimit-timeout|connlimit-startup|reconcile) + no-preload|standard|host-guc|superuser-guc-off|connlimit-backpressure|connlimit-timeout|connlimit-startup|reconcile|http-custom-domains|http-empty-domains) # Rebuild if previous phase changed the Cargo features if [ "$CURRENT_FEATURES" != "http-allow-test-domains" ]; then build_extension @@ -598,6 +680,11 @@ prepare_phase() { -c "DROP EXTENSION IF EXISTS pg_durable CASCADE; DROP SCHEMA IF EXISTS duroxide CASCADE;" \ >/dev/null 2>&1 || true + if [ "$phase" = "http-custom-domains" ]; then + echo "Checking malformed HTTP allowlist startup rejection..." + assert_http_domains_startup_rejected || exit 1 + fi + if [ -f "$LOG_FILE" ]; then PHASE_LOG_MARK=$(wc -l < "$LOG_FILE") else @@ -648,7 +735,7 @@ prepare_phase() { reconcile) wait_for_worker_ready ;; - http-disabled|http-allow-all) + http-custom-domains|http-empty-domains|http-disabled|http-allow-all) ensure_e2e_role wait_for_worker_ready ;; diff --git a/scripts/test-upgrade.sh b/scripts/test-upgrade.sh index bd04f63f..61fbf200 100755 --- a/scripts/test-upgrade.sh +++ b/scripts/test-upgrade.sh @@ -301,8 +301,9 @@ if [ -f "$DATA_DIR/postgresql.conf" ]; then # postgresql.conf. Without this, the connlimit-* phases' max_duroxide_connections=1 # causes the BGW to refuse to start (breaking the B1 wait-for-readiness check), # and the reconcile phase's aggressive reconcile_interval/retention_days would - # remove terminal instances mid-test and make df.result() flaky. - sed -i.bak '/^[#[:space:]]*pg_durable\.max_/d; /^[#[:space:]]*pg_durable\.execution_/d; /^[#[:space:]]*pg_durable\.reconcile_/d; /^[#[:space:]]*pg_durable\.retention_/d' "$DATA_DIR/postgresql.conf" + # remove terminal instances mid-test and make df.result() flaky. HTTP phases + # can leave a custom or empty domain list instead of the build's defaults. + sed -i.bak '/^[#[:space:]]*pg_durable\.max_/d; /^[#[:space:]]*pg_durable\.execution_/d; /^[#[:space:]]*pg_durable\.reconcile_/d; /^[#[:space:]]*pg_durable\.retention_/d; /^[#[:space:]]*pg_durable\.http_allowed_domains[[:space:]]*=/d' "$DATA_DIR/postgresql.conf" fi # If the server is already running, restart it so both the freshly installed diff --git a/src/activities/execute_http.rs b/src/activities/execute_http.rs index 874d1511..63a068d6 100644 --- a/src/activities/execute_http.rs +++ b/src/activities/execute_http.rs @@ -3,10 +3,10 @@ //! ExecuteHTTP activity - makes HTTP requests //! -//! Cargo features control what outbound HTTP(S) is allowed: -//! - `http-allow-azure-domains`: Azure endpoints + api.github.com only -//! (+ IP blocklist, no redirects). -//! - `http-allow-test-domains`: same + httpbingo.org. +//! Cargo features control the outbound HTTP(S) security tier: +//! - `http-allow-azure-domains`: configurable domains, defaulting to Azure +//! endpoints + api.github.com (+ IP blocklist, no redirects). +//! - `http-allow-test-domains`: same, also defaulting to allow httpbingo.org. //! - `http-allow-all`: no restrictions (development only). //! - *(none)*: all HTTP calls fail at execution time. //! @@ -18,6 +18,7 @@ use std::time::Duration; use sqlx::PgPool; +use crate::ssrf::DomainAllowlist; use crate::types::HttpConfig; /// Activity name for registration and scheduling @@ -86,6 +87,7 @@ pub(crate) fn build_client(timeout: Duration) -> Result pub async fn execute( ctx: ActivityContext, pool: Arc, + allowed_domains: Arc, config_json: String, ) -> Result { let config: HttpConfig = @@ -111,7 +113,7 @@ pub async fn execute( // bypass path where a user crafts raw Durofut JSON and passes // it to df.start() without going through the DSL guard. // 1. Scheme: blocks file://, gopher://, etc. - // 2. Allowlist: blocks ALL bare IPs (public and private) + non-Azure + // 2. Allowlist: blocks ALL bare IPs (public and private) + unlisted // domains. Fails-closed on malformed URLs. Because bare IPs // bypass the DNS resolver entirely in reqwest, this is the // definitive gate for IP-literal URLs. @@ -144,8 +146,8 @@ pub async fn execute( )); })?; - // --- Azure endpoint allow-list (blocks all bare IPs + non-Azure domains) --- - crate::ssrf::validate_allowlist(&request_url).inspect_err(|_| { + // --- Endpoint allow-list (blocks all bare IPs + unlisted domains) --- + crate::ssrf::validate_allowlist(&request_url, &allowed_domains).inspect_err(|_| { ctx.trace_info(format!( "HTTP BLOCKED (allowlist) url={safe_url} submitted_by={audit_user}" )); diff --git a/src/activities/execute_multipart.rs b/src/activities/execute_multipart.rs index 7e4edb33..9c79ce3d 100644 --- a/src/activities/execute_multipart.rs +++ b/src/activities/execute_multipart.rs @@ -4,7 +4,7 @@ //! ExecuteMultipart activity - makes multipart/form-data HTTP requests. //! //! This is the file-upload / form-post counterpart to `execute_http`. It shares -//! the same security model (privilege check, scheme validation, Azure +//! the same security model (privilege check, scheme validation, domain //! allow-list, SSRF-safe DNS resolver, no redirects) and reuses //! `execute_http::build_client` so the two paths cannot drift on client //! configuration. The only differences are the body construction (a @@ -22,6 +22,7 @@ use std::time::Duration; use sqlx::PgPool; use crate::activities::execute_http::build_client; +use crate::ssrf::DomainAllowlist; use crate::types::MultipartConfig; /// Activity name for registration and scheduling @@ -80,6 +81,7 @@ fn decode_part_data(data_b64: &str) -> Result, base64::DecodeError> { pub async fn execute( ctx: ActivityContext, pool: Arc, + allowed_domains: Arc, config_json: String, ) -> Result { let config: MultipartConfig = serde_json::from_str(&config_json) @@ -97,7 +99,7 @@ pub async fn execute( // Validation chain — order is security-critical and mirrors execute_http: // 0. Privilege: submitted_by must hold EXECUTE on df.http_multipart(). // 1. Scheme: blocks file://, gopher://, etc. - // 2. Allowlist: blocks ALL bare IPs (public and private) + non-Azure + // 2. Allowlist: blocks ALL bare IPs (public and private) + unlisted // domains. Fails-closed on malformed URLs. // 3. DNS resolver (SsrfSafeResolver): catches DNS rebinding. @@ -123,8 +125,8 @@ pub async fn execute( )); })?; - // --- Azure endpoint allow-list --- - crate::ssrf::validate_allowlist(&request_url).inspect_err(|_| { + // --- Endpoint allow-list --- + crate::ssrf::validate_allowlist(&request_url, &allowed_domains).inspect_err(|_| { ctx.trace_info(format!( "HTTP_MULTIPART BLOCKED (allowlist) url={safe_url} submitted_by={audit_user}" )); diff --git a/src/lib.rs b/src/lib.rs index e9125d94..4e1f654b 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -22,6 +22,9 @@ pub static DATABASE: GucSetting> = pub static HOST: GucSetting> = GucSetting::>::new(Some(c"")); +pub static HTTP_ALLOWED_DOMAINS: GucSetting> = + GucSetting::>::new(Some(ssrf::DEFAULT_HTTP_ALLOWED_DOMAINS)); + pub static MAX_MANAGEMENT_CONNECTIONS: GucSetting = GucSetting::::new(6); pub static MAX_DUROXIDE_CONNECTIONS: GucSetting = GucSetting::::new(10); pub static MAX_USER_CONNECTIONS: GucSetting = GucSetting::::new(10); @@ -126,6 +129,21 @@ pub extern "C-unwind" fn _PG_init() { GucFlags::default(), ); + // The callback is pg_guard-protected and only validates the proposed value. + unsafe { + GucRegistry::define_string_guc_with_hooks( + c"pg_durable.http_allowed_domains", + c"Hostnames allowed for outbound HTTP requests in restricted builds", + c"Comma-separated exact hostnames or *.domain subdomain patterns. Replaces the build's default list; an empty list denies all domains. Does not override HTTP feature gates or IP restrictions. Requires a server restart to change.", + &HTTP_ALLOWED_DOMAINS, + GucContext::Postmaster, + GucFlags::default(), + Some(ssrf::check_http_allowed_domains), + None, + None, + ); + } + GucRegistry::define_int_guc( c"pg_durable.max_management_connections", c"Maximum number of connections in the background worker management pool (lifecycle, graph loading, status updates)", diff --git a/src/registry.rs b/src/registry.rs index 1eee1517..6ecab30b 100644 --- a/src/registry.rs +++ b/src/registry.rs @@ -11,9 +11,14 @@ use tokio::sync::Semaphore; use crate::activities; use crate::orchestrations; +use crate::ssrf::DomainAllowlist; /// Create the activity registry with all registered activities -pub fn create_activity_registry(pool: Arc, semaphore: Arc) -> ActivityRegistry { +pub fn create_activity_registry( + pool: Arc, + semaphore: Arc, + http_allowed_domains: Arc, +) -> ActivityRegistry { let sql_semaphore = semaphore; let graph_pool = pool.clone(); let transaction_graph_pool = pool.clone(); @@ -21,6 +26,7 @@ pub fn create_activity_registry(pool: Arc, semaphore: Arc) -> let node_status_pool = pool.clone(); let http_pool = pool.clone(); let multipart_pool = pool.clone(); + let multipart_allowed_domains = http_allowed_domains.clone(); ActivityRegistry::builder() .register(activities::execute_sql::NAME, move |ctx: ActivityContext, input_json: String| { @@ -50,11 +56,13 @@ pub fn create_activity_registry(pool: Arc, semaphore: Arc) -> }) .register(activities::execute_http::NAME, move |ctx: ActivityContext, config_json: String| { let pool = http_pool.clone(); - async move { activities::execute_http::execute(ctx, pool, config_json).await } + let allowed_domains = http_allowed_domains.clone(); + async move { activities::execute_http::execute(ctx, pool, allowed_domains, config_json).await } }) .register(activities::execute_multipart::NAME, move |ctx: ActivityContext, config_json: String| { let pool = multipart_pool.clone(); - async move { activities::execute_multipart::execute(ctx, pool, config_json).await } + let allowed_domains = multipart_allowed_domains.clone(); + async move { activities::execute_multipart::execute(ctx, pool, allowed_domains, config_json).await } }) .build() } diff --git a/src/ssrf.rs b/src/ssrf.rs index 2c326cf5..f3b5e8e7 100644 --- a/src/ssrf.rs +++ b/src/ssrf.rs @@ -9,22 +9,24 @@ //! | Feature | Behaviour | //! |---------|-----------| //! | *(none)* | **All** outbound HTTP is blocked — at DSL time and at execution time. | -//! | `http-allow-azure-domains` | SSRF IP blocklist active, bare IPs blocked, redirects blocked, only Azure suffixes plus `api.github.com` allowed. | -//! | `http-allow-test-domains` | Same as `http-allow-azure-domains` **plus** `httpbingo.org` (for E2E tests). Implies `http-allow-azure-domains`. | +//! | `http-allow-azure-domains` | SSRF IP blocklist active, bare IPs blocked, redirects blocked, GUC allow-list defaults to Azure suffixes plus `api.github.com`. | +//! | `http-allow-test-domains` | Same as `http-allow-azure-domains`, defaulting to **also** allow `httpbingo.org`. Implies `http-allow-azure-domains`. | //! | `http-allow-all` | All SSRF protections disabled — any URL is allowed (development only). | //! -//! The blocklist and allow-list are hardcoded and cannot be bypassed by any -//! database user, including superusers. See docs/http-security.md for details. +//! `pg_durable.http_allowed_domains` replaces the restricted builds' domain +//! allow-list at server startup. It cannot disable the hardcoded IP blocklist +//! or override the HTTP feature gates. See docs/http-security.md for details. //! //! Every check that inspects a URL runs on the [`Url`] produced by //! [`parse_request_url`], and that same value is handed to reqwest. A second, //! independent parser would reintroduce the differential described there. use reqwest::Url; +use std::ffi::CStr; use std::net::{IpAddr, Ipv4Addr, Ipv6Addr}; // --------------------------------------------------------------------------- -// Endpoint allow-list — compile-time constant +// Endpoint allow-list configuration // --------------------------------------------------------------------------- /// Returns `true` when *any* HTTP feature is enabled (azure, test, or all). @@ -36,55 +38,173 @@ pub const fn http_enabled() -> bool { )) } -/// Hard-coded Azure endpoint allow-list (data-plane only). -/// -/// Populated when `http-allow-azure-domains` (or `http-allow-test-domains`, -/// which implies it) is enabled. When `http-allow-all` is set the allow-list -/// is bypassed entirely so its contents don't matter. -/// -/// Each entry starts with `.` so that a simple `ends_with` check naturally -/// requires at least one subdomain label (the apex domain itself never matches). -#[cfg(any( - feature = "http-allow-azure-domains", - feature = "http-allow-test-domains" -))] -pub(crate) const AZURE_DOMAIN_SUFFIXES: &[&str] = &[ - ".blob.core.windows.net", - ".blob.storage.azure.net", - ".queue.core.windows.net", - ".table.core.windows.net", - ".file.core.windows.net", - ".azurewebsites.net", - ".azure-api.net", - ".documents.azure.com", - ".servicebus.windows.net", - ".openai.azure.com", - ".cognitiveservices.azure.com", - ".vault.azure.net", - ".redis.cache.windows.net", - ".database.windows.net", - ".kusto.windows.net", - ".azurefd.net", - ".azureedge.net", - ".azure-devices.net", - ".trafficmanager.net", - ".cloudapp.azure.com", -]; - -/// Fully-qualified non-Azure domains allowed alongside the Azure suffixes -/// (exact match, not suffix). -/// -/// Available whenever `http-allow-azure-domains` (or `http-allow-test-domains`, -/// which implies it) is enabled. -#[cfg(any( - feature = "http-allow-azure-domains", - feature = "http-allow-test-domains" -))] -pub(crate) const EXACT_DOMAINS: &[&str] = &["api.github.com"]; - -/// Fully-qualified test domains (exact match, not suffix). -#[cfg(feature = "http-allow-test-domains")] -pub(crate) const TEST_EXACT_DOMAINS: &[&str] = &["httpbingo.org"]; +// Keep the production and test defaults sourced from the same list. +macro_rules! azure_domain_defaults { + ($extra:literal) => { + concat!( + "*.blob.core.windows.net,", + "*.blob.storage.azure.net,", + "*.queue.core.windows.net,", + "*.table.core.windows.net,", + "*.file.core.windows.net,", + "*.azurewebsites.net,", + "*.azure-api.net,", + "*.documents.azure.com,", + "*.servicebus.windows.net,", + "*.openai.azure.com,", + "*.cognitiveservices.azure.com,", + "*.vault.azure.net,", + "*.redis.cache.windows.net,", + "*.database.windows.net,", + "*.kusto.windows.net,", + "*.azurefd.net,", + "*.azureedge.net,", + "*.azure-devices.net,", + "*.trafficmanager.net,", + "*.cloudapp.azure.com,", + "api.github.com", + $extra, + "\0" + ) + .as_bytes() + }; +} + +pub(crate) const DEFAULT_HTTP_ALLOWED_DOMAINS: &CStr = + match CStr::from_bytes_with_nul(if cfg!(feature = "http-allow-test-domains") { + azure_domain_defaults!(",httpbingo.org") + } else if cfg!(feature = "http-allow-azure-domains") { + azure_domain_defaults!("") + } else { + b"\0" + }) { + Ok(value) => value, + Err(_) => panic!("HTTP domain defaults must form a C string"), + }; + +/// An immutable, canonically parsed hostname policy. Empty means deny all. +#[derive(Debug, Default)] +pub struct DomainAllowlist { + exact_domains: Vec, + domain_suffixes: Vec, +} + +impl DomainAllowlist { + pub fn parse(value: &str) -> Result { + let mut allowlist = Self::default(); + if value.trim().is_empty() { + return Ok(allowlist); + } + + for (index, entry) in value.split(',').enumerate() { + let entry = entry.trim(); + let invalid = |reason: &str| format!("entry {} ({entry:?}): {reason}", index + 1); + let subdomains = entry.strip_prefix("*."); + let domain = subdomains.unwrap_or(entry); + if domain.contains('%') { + return Err(invalid("percent-encoded hostnames are not permitted")); + } + + let host = match url::Host::parse(domain) + .map_err(|_| invalid("expected a hostname or *.hostname"))? + { + url::Host::Domain(host) => host, + url::Host::Ipv4(_) | url::Host::Ipv6(_) => { + return Err(invalid("IP addresses are not permitted")); + } + }; + + if host.len() > 253 + || host.split('.').any(|label| { + label.is_empty() + || label.len() > 63 + || label.starts_with('-') + || label.ends_with('-') + || !label + .bytes() + .all(|byte| byte.is_ascii_alphanumeric() || byte == b'-') + }) + { + return Err(invalid("expected a DNS hostname without a trailing dot")); + } + + if subdomains.is_some() { + allowlist.domain_suffixes.push(format!(".{host}")); + } else { + allowlist.exact_domains.push(host); + } + } + Ok(allowlist) + } + + fn validate(&self, url: &Url) -> Result<(), String> { + // WHATWG canonicalises decimal, octal and IPv4-mapped forms into IP + // variants. They must not bypass the resolver's IP protection. + let host = match url.host() { + Some(url::Host::Domain(domain)) => domain, + Some(url::Host::Ipv4(_)) | Some(url::Host::Ipv6(_)) => { + return Err("Blocked: requests to bare IP addresses are not permitted. \ + Use an approved service hostname instead." + .to_string()); + } + None => return Err("Blocked: unable to extract hostname from URL.".to_string()), + }; + + if self.exact_domains.iter().any(|domain| host == domain) + || self + .domain_suffixes + .iter() + .any(|suffix| host.len() > suffix.len() && host.ends_with(suffix)) + { + return Ok(()); + } + + Err(format!( + "Blocked: '{host}' is not in the allowed endpoint list. \ + Configure pg_durable.http_allowed_domains to allow this hostname." + )) + } +} + +impl TryFrom<&CStr> for DomainAllowlist { + type Error = String; + + fn try_from(value: &CStr) -> Result { + Self::parse(value.to_str().map_err(|_| { + "value must be valid UTF-8; ASCII/punycode hostnames are always supported".to_string() + })?) + } +} + +#[pgrx::pg_guard] +pub(crate) unsafe extern "C-unwind" fn check_http_allowed_domains( + newval: *mut *mut std::ffi::c_char, + _extra: *mut *mut std::ffi::c_void, + _source: pgrx::pg_sys::GucSource::Type, +) -> bool { + let result = if unsafe { (*newval).is_null() } { + Err("value must not be NULL".to_string()) + } else { + DomainAllowlist::try_from(unsafe { CStr::from_ptr(*newval) }) + }; + match result { + Ok(_) => true, + Err(error) => { + // During preload, PostgreSQL otherwise only warns and restores the + // default when rejecting a placeholder, potentially widening policy. + if unsafe { pgrx::pg_sys::process_shared_preload_libraries_in_progress } { + pgrx::error!( + "invalid value for parameter \"pg_durable.http_allowed_domains\": {error}" + ); + } + unsafe { + pgrx::pg_sys::GUC_check_errdetail_string = + pgrx::PgMemoryContexts::ErrorContext.pstrdup(&error); + } + false + } + } +} // --------------------------------------------------------------------------- // IP blocklist @@ -228,83 +348,19 @@ pub fn validate_scheme(url: &Url) -> Result<(), String> { /// Behaviour depends on Cargo features (most to least restrictive): /// /// * *(none)* — all requests blocked, regardless of domain. -/// * `http-allow-azure-domains` — bare IPs blocked; only Azure suffixes and -/// `api.github.com` allowed. -/// * `http-allow-test-domains` — same as above **plus** `httpbingo.org` -/// (for E2E tests). +/// * `http-allow-azure-domains` / `http-allow-test-domains` — bare IPs blocked; +/// the worker's `pg_durable.http_allowed_domains` snapshot is enforced. /// * `http-allow-all` — allow-list check is skipped entirely; all domains pass. -pub fn validate_allowlist(url: &Url) -> Result<(), String> { - // http-allow-all: skip all domain checks. - #[cfg(feature = "http-allow-all")] - { - let _ = url; - Ok(()) +pub fn validate_allowlist(url: &Url, allowlist: &DomainAllowlist) -> Result<(), String> { + if cfg!(feature = "http-allow-all") { + return Ok(()); } - - #[cfg(not(feature = "http-allow-all"))] - { - // No http feature at all — block everything. - #[cfg(not(any( - feature = "http-allow-azure-domains", - feature = "http-allow-test-domains", - )))] - { - let _ = url; - Err("Blocked: outbound HTTP requests are disabled. \ - Rebuild with the 'http-allow-azure-domains' Cargo feature to enable them." - .to_string()) - } - - // http-allow-azure-domains or http-allow-test-domains: enforce allow-list. - #[cfg(any( - feature = "http-allow-azure-domains", - feature = "http-allow-test-domains", - ))] - { - // Matching on Host (rather than inspecting the host string) keeps - // the IP-literal case total: WHATWG canonicalises decimal, octal and - // IPv4-mapped forms into these variants before we ever see them. - let host = match url.host() { - Some(url::Host::Domain(domain)) => domain, - Some(url::Host::Ipv4(_)) | Some(url::Host::Ipv6(_)) => { - return Err("Blocked: requests to bare IP addresses are not permitted. \ - Use an approved service hostname instead." - .to_string()); - } - None => return Err("Blocked: unable to extract hostname from URL.".to_string()), - }; - - let host_lower = host.to_ascii_lowercase(); - - // Check Azure suffixes (always present when either azure or test feature is on). - for suffix in AZURE_DOMAIN_SUFFIXES { - if host_lower.ends_with(suffix) { - return Ok(()); - } - } - - // Exact-match non-Azure domains allowed in the same tier. - for exact in EXACT_DOMAINS { - if host_lower == *exact { - return Ok(()); - } - } - - // Additional test domains (only with http-allow-test-domains). - #[cfg(feature = "http-allow-test-domains")] - for exact in TEST_EXACT_DOMAINS { - if host_lower == *exact { - return Ok(()); - } - } - - Err(format!( - "Blocked: '{}' is not in the allowed endpoint list. \ - Only requests to approved Azure service domains are permitted.", - host - )) - } + if !http_enabled() { + return Err("Blocked: outbound HTTP requests are disabled. \ + Rebuild with the 'http-allow-azure-domains' Cargo feature to enable them." + .to_string()); } + allowlist.validate(url) } // Keep this marker in sync with the error message in SsrfSafeResolver::resolve(). @@ -637,11 +693,10 @@ mod tests { // --- Canonical URL parsing --- - // Tests drive the allow-list through the same parse-then-validate path the - // activities use, so an unparseable URL is a rejection like any other. - #[cfg(not(feature = "http-allow-all"))] + // Exercise domain matching independently of feature gates; their precedence + // is covered separately with both custom and empty lists. fn validate_url_allowlist(url: &str) -> Result<(), String> { - validate_allowlist(&parse_request_url(url)?) + DomainAllowlist::try_from(DEFAULT_HTTP_ALLOWED_DOMAINS)?.validate(&parse_request_url(url)?) } #[test] @@ -674,6 +729,219 @@ mod tests { // --- Endpoint allow-list validation --- + #[test] + fn allowlist_defaults_match_build() { + assert_eq!( + validate_url_allowlist("https://api.github.com/").is_ok(), + cfg!(feature = "http-allow-azure-domains") + ); + assert_eq!( + validate_url_allowlist("https://account.blob.core.windows.net/").is_ok(), + cfg!(feature = "http-allow-azure-domains") + ); + assert_eq!( + validate_url_allowlist("https://httpbingo.org/").is_ok(), + cfg!(feature = "http-allow-test-domains") + ); + } + + #[test] + fn configured_allowlist_matches_exact_names_and_subdomain_boundaries() { + let allowlist = DomainAllowlist::parse(" API.GitHub.COM ,\n *.Example.COM \t").unwrap(); + for url in [ + "https://api.github.com/", + "https://API.GITHUB.COM/", + "https://a.example.com/", + "https://a.b.c.example.com:8443/", + "https://a%2Eexample%2Ecom/", + ] { + assert!( + allowlist.validate(&parse_request_url(url).unwrap()).is_ok(), + "{url}" + ); + } + for url in [ + "https://github.com/", + "https://evil.api.github.com/", + "https://evilapi.github.com/", + "https://example.com/", + "https://.example.com/", + "https://a.example.com.evil.net/", + "https://api.github.com./", + "https://a.example.com./", + "https://evil.net?@a.example.com/", + r"https://evil.net\@a.example.com/", + "https://8.8.8.8/", + "https://[2001:4860:4860::8888]/", + ] { + assert!( + parse_request_url(url) + .and_then(|url| allowlist.validate(&url)) + .is_err(), + "{url}" + ); + } + } + + #[test] + fn configured_allowlist_replaces_all_default_domains() { + let allowlist = DomainAllowlist::parse("example.com").unwrap(); + assert!(allowlist + .validate(&parse_request_url("https://example.com/").unwrap()) + .is_ok()); + for url in [ + "https://api.github.com/", + "https://httpbingo.org/", + "https://account.blob.core.windows.net/", + ] { + let error = allowlist + .validate(&parse_request_url(url).unwrap()) + .unwrap_err(); + assert!(error.contains("pg_durable.http_allowed_domains"), "{error}"); + } + } + + #[test] + fn configured_allowlist_normalizes_idna_without_homograph_matches() { + let allowlist = + DomainAllowlist::parse("B\u{dc}CHER.example, *.ma\u{f1}ana.example").unwrap(); + for url in [ + "https://b\u{fc}cher.example/", + "https://xn--bcher-kva.example/", + "https://a.b.ma\u{f1}ana.example/", + "https://a.xn--maana-pta.example/", + ] { + assert!( + allowlist.validate(&parse_request_url(url).unwrap()).is_ok(), + "{url}" + ); + } + for url in [ + "https://bucher.example/", + "https://a.manana.example/", + "https://ma\u{f1}ana.example/", + "https://xn--bcher-kva.example./", + ] { + assert!( + allowlist + .validate(&parse_request_url(url).unwrap()) + .is_err(), + "{url}" + ); + } + } + + #[test] + fn empty_configured_allowlist_denies_all_domains() { + for value in ["", " \t\r\n "] { + let allowlist = DomainAllowlist::parse(value).unwrap(); + for url in [ + "https://api.github.com/", + "https://httpbingo.org/", + "https://account.blob.core.windows.net/", + ] { + assert!(allowlist + .validate(&parse_request_url(url).unwrap()) + .is_err()); + } + } + } + + #[test] + fn malformed_domain_configuration_is_rejected_as_a_whole() { + for value in [ + ",", + ",example.com", + "example.com,", + "example.com,,example.net", + "*", + "*.", + "**.example.com", + "api.*.example.com", + ".example.com", + "example.com.", + "example..com", + "-api.example.com", + "api-.example.com", + "api_example.com", + "https://example.com", + "example.com:443", + "example.com/path", + "user@example.com", + "example.com?query", + "example.com#fragment", + r"example.com\path", + "\"example.com\"", + "'example.com'", + "exa mple.com", + "example%2ecom", + "8.8.8.8", + "0x7f.1", + "2130706433", + "[::1]", + "::ffff:127.0.0.1", + "*.127.0.0.1", + "10.0.0.0/8", + ] { + let error = DomainAllowlist::parse(value).unwrap_err(); + assert!(error.contains("entry "), "{value:?}: {error}"); + } + let error = DomainAllowlist::parse("example.com, https://example.net").unwrap_err(); + assert!(error.contains("entry 2"), "{error}"); + } + + #[test] + fn configured_allowlist_enforces_dns_lengths_not_sql_identifier_lengths() { + let long_hostname = format!( + "{}.{}.{}.{}", + "a".repeat(63), + "b".repeat(63), + "c".repeat(63), + "d".repeat(61) + ); + let allowlist = DomainAllowlist::parse(&long_hostname).unwrap(); + assert!(allowlist + .validate(&parse_request_url(&format!("https://{long_hostname}/")).unwrap()) + .is_ok()); + assert!(DomainAllowlist::parse(&format!("{long_hostname}d")).is_err()); + assert!(DomainAllowlist::parse(&format!("{}.example.com", "a".repeat(64))).is_err()); + } + + #[test] + fn configured_allowlist_rejects_non_utf8_without_lossy_conversion() { + let value = c"\xff.example"; + assert!(DomainAllowlist::try_from(value) + .unwrap_err() + .contains("UTF-8")); + } + + #[test] + fn http_feature_gates_take_precedence_over_configured_domains() { + let allowed = parse_request_url("https://example.com/").unwrap(); + let unlisted = parse_request_url("https://example.net/").unwrap(); + let ip = parse_request_url("https://8.8.8.8/").unwrap(); + let custom = DomainAllowlist::parse("example.com").unwrap(); + let empty = DomainAllowlist::parse("").unwrap(); + if cfg!(feature = "http-allow-all") { + for allowlist in [&custom, &empty] { + for url in [&allowed, &unlisted, &ip] { + assert!(validate_allowlist(url, allowlist).is_ok()); + } + } + } else if http_enabled() { + assert!(validate_allowlist(&allowed, &custom).is_ok()); + assert!(validate_allowlist(&unlisted, &custom).is_err()); + assert!(validate_allowlist(&ip, &custom).is_err()); + assert!(validate_allowlist(&allowed, &empty).is_err()); + } else { + for allowlist in [&custom, &empty] { + assert!(validate_allowlist(&allowed, allowlist) + .unwrap_err() + .contains("outbound HTTP requests are disabled")); + } + } + } + // These "blocks_*" tests are only meaningful when some http feature is // enabled (otherwise the no-feature path blocks everything anyway). #[cfg(any( diff --git a/src/worker.rs b/src/worker.rs index 1e5dd55c..fa4928f4 100644 --- a/src/worker.rs +++ b/src/worker.rs @@ -17,6 +17,7 @@ use duroxide_pg::PostgresProvider; use tracing_subscriber::EnvFilter; use crate::registry::{create_activity_registry, create_orchestration_registry}; +use crate::ssrf::DomainAllowlist; use crate::types::{ get_max_duroxide_connections, get_max_management_connections, get_max_user_connections, get_reconcile_interval, get_retention_days, postgres_connection_string, @@ -98,6 +99,15 @@ pub extern "C-unwind" fn duroxide_worker_main(_arg: pg_sys::Datum) { log!("pg_durable: duroxide background worker starting..."); + let configured_domains = crate::HTTP_ALLOWED_DOMAINS + .get() + .unwrap_or_else(|| error!("pg_durable.http_allowed_domains must not be NULL")); + let http_allowed_domains = Arc::new( + DomainAllowlist::try_from(configured_domains.as_c_str()).unwrap_or_else(|err| { + error!("invalid value for parameter \"pg_durable.http_allowed_domains\": {err}") + }), + ); + let rt = match tokio::runtime::Builder::new_current_thread() .enable_all() .build() @@ -110,7 +120,7 @@ pub extern "C-unwind" fn duroxide_worker_main(_arg: pg_sys::Datum) { }; rt.block_on(async { - run_duroxide_runtime().await; + run_duroxide_runtime(http_allowed_domains).await; }); // All async cleanup (pool closes, runtime shutdown) is performed inside @@ -124,7 +134,7 @@ pub extern "C-unwind" fn duroxide_worker_main(_arg: pg_sys::Datum) { } /// Run the duroxide runtime with proper shutdown handling -async fn run_duroxide_runtime() { +async fn run_duroxide_runtime(http_allowed_domains: Arc) { const WAIT_FOR_EXTENSION_POLL_INTERVAL: Duration = Duration::from_secs(5); const EXTENSION_DROP_POLL_INTERVAL: Duration = Duration::from_secs(5); const INIT_RETRY_INTERVAL: Duration = Duration::from_secs(1); @@ -288,6 +298,7 @@ async fn run_duroxide_runtime() { INIT_RETRY_INTERVAL, &mgmt_pool, &duroxide_schema, + &http_allowed_domains, ) .await else { @@ -671,6 +682,7 @@ async fn initialize_duroxide_runtime( retry_interval: Duration, mgmt_pool: &sqlx::PgPool, schema_name: &str, + http_allowed_domains: &Arc, ) -> Option<(Arc, Arc)> { log!("pg_durable: initializing duroxide runtime..."); @@ -759,8 +771,11 @@ async fn initialize_duroxide_runtime( // Reuse the management pool for activities (graph loading, status updates). // The former dedicated activity pool with its df.in_workflow hook is no // longer needed — connect_as_user() sets that flag independently. - let activities = - create_activity_registry(Arc::new(mgmt_pool.clone()), user_semaphore.clone()); + let activities = create_activity_registry( + Arc::new(mgmt_pool.clone()), + user_semaphore.clone(), + http_allowed_domains.clone(), + ); let orchestrations = create_orchestration_registry(); let store_for_client = store.clone(); diff --git a/tests/e2e/sql/47_http_dsl_disabled.sql b/tests/e2e/sql/47_http_dsl_disabled.sql index 2c7283f6..184df3a5 100644 --- a/tests/e2e/sql/47_http_dsl_disabled.sql +++ b/tests/e2e/sql/47_http_dsl_disabled.sql @@ -7,6 +7,15 @@ -- This test runs in the "http-disabled" phase, which builds pg_durable without any -- http-allow-* features. df.http() must raise immediately at SQL call time (before -- df.start() is ever called), not just at execution time. +-- The requested hostname is explicitly allowed by the GUC: a configured list +-- must not enable HTTP in a build without an HTTP feature. + +DO $$ +BEGIN + IF current_setting('pg_durable.http_allowed_domains') IS DISTINCT FROM 'example.com' THEN + RAISE EXCEPTION 'TEST FAILED: http-disabled phase requires the example.com allowlist'; + END IF; +END $$; -- ============================================================================ -- Test 1: df.http() raises at DSL construction time when HTTP is disabled diff --git a/tests/e2e/sql/48_http_allow_all.sql b/tests/e2e/sql/48_http_allow_all.sql index 64e7290e..4048ed81 100644 --- a/tests/e2e/sql/48_http_allow_all.sql +++ b/tests/e2e/sql/48_http_allow_all.sql @@ -7,6 +7,14 @@ -- http-allow-all Cargo feature. Domains that are normally blocked by the Azure -- allow-list (e.g. example.com) must be reachable (or at least not rejected by -- the allow-list — network/DNS failure is fine). +-- An explicitly empty GUC must not restrict this build. + +DO $$ +BEGIN + IF current_setting('pg_durable.http_allowed_domains') IS DISTINCT FROM '' THEN + RAISE EXCEPTION 'TEST FAILED: http-allow-all phase requires an empty allowlist'; + END IF; +END $$; -- ============================================================================ -- Test 1: Non-Azure domain passes allow-list when http-allow-all is set @@ -15,25 +23,31 @@ CREATE TEMP TABLE _test_allowall1 (instance_id TEXT); INSERT INTO _test_allowall1 SELECT df.start( - df.http('http://example.com/', 'GET'), + df.http('http://example.com/', 'GET', NULL, NULL, 5), 'test-http-allow-all-non-azure' ); DO $$ DECLARE inst_id TEXT; + status TEXT; node_result TEXT; BEGIN SELECT instance_id INTO inst_id FROM _test_allowall1; RAISE NOTICE 'Testing non-Azure domain allowed under http-allow-all: %', inst_id; - PERFORM df.await_instance(inst_id); + SELECT df.await_instance(inst_id, 30) INTO status; -- Must NOT fail due to allow-list; network/DNS failure is acceptable SELECT result::text INTO node_result FROM df.nodes WHERE instance_id = inst_id AND node_type = 'HTTP'; + IF status IS NULL OR status NOT IN ('completed', 'failed') OR node_result IS NULL THEN + RAISE EXCEPTION 'TEST FAILED: allow-all request did not finish: status = %, result = %', + status, node_result; + END IF; + IF node_result ILIKE '%not in the allowed%' THEN RAISE EXCEPTION 'TEST FAILED: allow-list should be bypassed under http-allow-all, got: %', node_result; END IF; @@ -59,24 +73,30 @@ DROP TABLE _test_allowall1; CREATE TEMP TABLE _test_allowall2 (instance_id TEXT); INSERT INTO _test_allowall2 SELECT df.start( - df.http('https://8.8.8.8/', 'GET'), + df.http('https://8.8.8.8/', 'GET', NULL, NULL, 5), 'test-http-allow-all-bare-ip' ); DO $$ DECLARE inst_id TEXT; + status TEXT; node_result TEXT; BEGIN SELECT instance_id INTO inst_id FROM _test_allowall2; RAISE NOTICE 'Testing bare public IP allowed under http-allow-all: %', inst_id; - PERFORM df.await_instance(inst_id); + SELECT df.await_instance(inst_id, 30) INTO status; SELECT result::text INTO node_result FROM df.nodes WHERE instance_id = inst_id AND node_type = 'HTTP'; + IF status IS NULL OR status NOT IN ('completed', 'failed') OR node_result IS NULL THEN + RAISE EXCEPTION 'TEST FAILED: allow-all IP request did not finish: status = %, result = %', + status, node_result; + END IF; + -- Under http-allow-all the allow-list is entirely bypassed — no "bare IP" rejection IF node_result ILIKE '%bare IP%' THEN RAISE EXCEPTION 'TEST FAILED: bare IP check should be bypassed under http-allow-all, got: %', node_result; diff --git a/tests/e2e/sql/69_http_allowed_domains.sql b/tests/e2e/sql/69_http_allowed_domains.sql new file mode 100644 index 00000000..324c6c71 --- /dev/null +++ b/tests/e2e/sql/69_http_allowed_domains.sql @@ -0,0 +1,177 @@ +-- Copyright (c) Microsoft Corporation. +-- Licensed under the PostgreSQL License. + +-- Issue #375: the "http-custom-domains" phase restarts PostgreSQL with +-- pg_durable.http_allowed_domains = 'example.com' and http-allow-test-domains. +-- The local runner first checks rejection of a malformed postgresql.conf value, +-- restores the configuration, and starts PostgreSQL with this valid allowlist. +-- Allowed requests must reach HTTP transport; example.com need not provide a +-- working POST endpoint. All other requests must fail before DNS or networking. + +SET SESSION AUTHORIZATION df_e2e_user; + +DO $$ +BEGIN + IF current_setting('pg_durable.http_allowed_domains') IS DISTINCT FROM 'example.com' THEN + RAISE EXCEPTION 'TEST FAILED: custom allowlist is not visible to an ordinary user'; + END IF; + + IF NOT EXISTS ( + SELECT 1 FROM pg_settings + WHERE name = 'pg_durable.http_allowed_domains' + AND setting = 'example.com' + AND context = 'postmaster' + AND vartype = 'string' + AND source = 'configuration file' + AND NOT pending_restart + ) THEN + RAISE EXCEPTION 'TEST FAILED: expected a readable string GUC applied at server startup'; + END IF; +END $$; + +RESET SESSION AUTHORIZATION; + +DO $$ +DECLARE + statement TEXT; +BEGIN + FOREACH statement IN ARRAY ARRAY[ + 'SET pg_durable.http_allowed_domains = ''api.github.com''', + 'SET LOCAL pg_durable.http_allowed_domains = ''api.github.com''', + 'ALTER ROLE df_e2e_user SET pg_durable.http_allowed_domains = ''api.github.com''', + format('ALTER DATABASE %I SET pg_durable.http_allowed_domains = ''api.github.com''', + current_database()), + format('ALTER ROLE df_e2e_user IN DATABASE %I SET pg_durable.http_allowed_domains = ''api.github.com''', + current_database()) + ] LOOP + BEGIN + EXECUTE statement; + RAISE EXCEPTION 'TEST FAILED: accepted a runtime override of a postmaster GUC: %', statement; + EXCEPTION WHEN cant_change_runtime_param THEN + NULL; + END; + END LOOP; + + IF current_setting('pg_durable.http_allowed_domains') IS DISTINCT FROM 'example.com' THEN + RAISE EXCEPTION 'TEST FAILED: runtime overrides changed the allowlist'; + END IF; +END $$; + +-- ALTER SYSTEM must run outside a transaction block. Save its diagnostics +-- before issuing another query, and clean up even if it unexpectedly succeeds. +\set ON_ERROR_STOP off +ALTER SYSTEM SET pg_durable.http_allowed_domains = 'example.com,https://api.github.com'; +\set invalid_domains_sqlstate :SQLSTATE +\set invalid_domains_message :LAST_ERROR_MESSAGE +\set ON_ERROR_STOP on + +CREATE TEMP TABLE _test_invalid_domains AS +SELECT :'invalid_domains_sqlstate'::text AS sqlstate, + :'invalid_domains_message'::text AS message, + EXISTS ( + SELECT 1 FROM pg_file_settings + WHERE name = 'pg_durable.http_allowed_domains' + AND sourcefile LIKE '%/postgresql.auto.conf' + ) AS override_written; + +ALTER SYSTEM RESET pg_durable.http_allowed_domains; + +DO $$ +DECLARE + rejected RECORD; +BEGIN + SELECT * INTO STRICT rejected FROM _test_invalid_domains; + + IF rejected.sqlstate IS DISTINCT FROM '22023' + OR rejected.message IS NULL + OR position('invalid value for parameter "pg_durable.http_allowed_domains"' IN rejected.message) = 0 + OR rejected.override_written THEN + RAISE EXCEPTION 'TEST FAILED: invalid ALTER SYSTEM value was not rejected atomically: %', rejected; + END IF; + + IF current_setting('pg_durable.http_allowed_domains') IS DISTINCT FROM 'example.com' THEN + RAISE EXCEPTION 'TEST FAILED: invalid ALTER SYSTEM value changed the running allowlist'; + END IF; +END $$; + +DROP TABLE _test_invalid_domains; + +SET SESSION AUTHORIZATION df_e2e_user; + +CREATE TEMP TABLE _test_http_allowed_domains ( + instance_id TEXT, + hostname TEXT, + node_type TEXT, + allowed BOOLEAN +); + +INSERT INTO _test_http_allowed_domains +SELECT df.start( + CASE node_type + WHEN 'HTTP' THEN df.http('https://' || hostname || '/', 'GET', NULL, NULL, 10) + ELSE df.http_multipart( + 'https://' || hostname || '/', 'POST', + '[{"name":"field","data_b64":"dmFsdWU="}]'::jsonb, NULL, 10 + ) + END, + 'custom-domains-' || node_type || '-' || hostname +), hostname, node_type, allowed +FROM (VALUES + ('example.com', true), + ('pg-durable-test-nonexistent.blob.core.windows.net', false), + ('api.github.com', false), + ('httpbingo.org', false) +) AS endpoints(hostname, allowed) +CROSS JOIN (VALUES ('HTTP'), ('HTTP_MULTIPART')) AS node_types(node_type); + +DO $$ +DECLARE + test_case RECORD; + status TEXT; + node_result TEXT; + http_status INT; +BEGIN + FOR test_case IN SELECT * FROM _test_http_allowed_domains ORDER BY hostname, node_type LOOP + SELECT df.await_instance(test_case.instance_id, 30) INTO status; + SELECT result::text INTO node_result + FROM df.nodes + WHERE instance_id = test_case.instance_id AND node_type = test_case.node_type; + + IF test_case.allowed THEN + IF status IS NULL OR status NOT IN ('completed', 'failed') OR node_result IS NULL THEN + RAISE EXCEPTION 'TEST FAILED: allowed % request to %: status = %, result = %', + test_case.node_type, test_case.hostname, status, node_result; + END IF; + + IF status = 'completed' THEN + http_status := (node_result::jsonb->>'status')::int; + IF http_status IS NULL OR http_status NOT BETWEEN 100 AND 599 THEN + RAISE EXCEPTION 'TEST FAILED: expected an HTTP response for %: %', + test_case.node_type, node_result; + END IF; + -- Only errors from after the policy checks prove admission. + ELSIF NOT (node_result LIKE ANY (ARRAY[ + '%HTTP connection failed%', + '%HTTP request failed%', + '%HTTP timeout after%', + '% returned 5__:%' + ])) THEN + RAISE EXCEPTION 'TEST FAILED: allowed request did not reach HTTP transport for %: %', + test_case.node_type, node_result; + END IF; + ELSE + IF status IS DISTINCT FROM 'failed' + OR node_result IS NULL + OR node_result NOT LIKE '%is not in the allowed endpoint list%' + OR position('pg_durable.http_allowed_domains' IN node_result) = 0 THEN + RAISE EXCEPTION 'TEST FAILED: built-in endpoint % was not denied for %: status = %, result = %', + test_case.hostname, test_case.node_type, status, node_result; + END IF; + END IF; + END LOOP; +END $$; + +DROP TABLE _test_http_allowed_domains; +RESET SESSION AUTHORIZATION; + +SELECT 'TEST PASSED' AS result; diff --git a/tests/e2e/sql/70_http_allowed_domains_empty.sql b/tests/e2e/sql/70_http_allowed_domains_empty.sql new file mode 100644 index 00000000..b0b1c882 --- /dev/null +++ b/tests/e2e/sql/70_http_allowed_domains_empty.sql @@ -0,0 +1,79 @@ +-- Copyright (c) Microsoft Corporation. +-- Licensed under the PostgreSQL License. + +-- Issue #375: the "http-empty-domains" phase restarts PostgreSQL with an +-- explicitly empty pg_durable.http_allowed_domains and http-allow-test-domains. +-- The previous phase's custom hostname and the built-in defaults must all be +-- denied, not restored as a fallback. No request needs a live endpoint. + +SET SESSION AUTHORIZATION df_e2e_user; + +DO $$ +BEGIN + IF current_setting('pg_durable.http_allowed_domains') IS DISTINCT FROM '' THEN + RAISE EXCEPTION 'TEST FAILED: expected an explicitly empty allowlist'; + END IF; + + IF NOT EXISTS ( + SELECT 1 FROM pg_settings + WHERE name = 'pg_durable.http_allowed_domains' + AND setting = '' + AND context = 'postmaster' + AND source = 'configuration file' + AND NOT pending_restart + ) THEN + RAISE EXCEPTION 'TEST FAILED: empty allowlist was not applied at server startup'; + END IF; +END $$; + +CREATE TEMP TABLE _test_http_empty_domains ( + instance_id TEXT, + hostname TEXT, + node_type TEXT +); + +INSERT INTO _test_http_empty_domains +SELECT df.start( + CASE node_type + WHEN 'HTTP' THEN df.http('https://' || hostname || '/', 'GET', NULL, NULL, 5) + ELSE df.http_multipart( + 'https://' || hostname || '/', 'POST', + '[{"name":"field","data_b64":"dmFsdWU="}]'::jsonb, NULL, 5 + ) + END, + 'empty-domains-' || node_type || '-' || hostname +), hostname, node_type +FROM (VALUES + ('example.com'), + ('pg-durable-test-nonexistent.blob.core.windows.net'), + ('api.github.com'), + ('httpbingo.org') +) AS endpoints(hostname) +CROSS JOIN (VALUES ('HTTP'), ('HTTP_MULTIPART')) AS node_types(node_type); + +DO $$ +DECLARE + test_case RECORD; + status TEXT; + node_result TEXT; +BEGIN + FOR test_case IN SELECT * FROM _test_http_empty_domains ORDER BY hostname, node_type LOOP + SELECT df.await_instance(test_case.instance_id, 30) INTO status; + SELECT result::text INTO node_result + FROM df.nodes + WHERE instance_id = test_case.instance_id AND node_type = test_case.node_type; + + IF status IS DISTINCT FROM 'failed' + OR node_result IS NULL + OR node_result NOT LIKE '%is not in the allowed endpoint list%' + OR position('pg_durable.http_allowed_domains' IN node_result) = 0 THEN + RAISE EXCEPTION 'TEST FAILED: empty allowlist did not deny % for %: status = %, result = %', + test_case.hostname, test_case.node_type, status, node_result; + END IF; + END LOOP; +END $$; + +DROP TABLE _test_http_empty_domains; +RESET SESSION AUTHORIZATION; + +SELECT 'TEST PASSED' AS result;