fix: fix #17 fix #18 fix #20, ref #16

Author: spwoodcock

.env.example

@@ -0,0 +1,14 @@
+# At least one webhook URL is required
+CENTRAL_WEBHOOK_NEW_SUBMISSION_URL=https://your.domain/webhook/new-submission
+CENTRAL_WEBHOOK_REVIEW_SUBMISSION_URL=https://your.domain/webhook/review-submission
+CENTRAL_WEBHOOK_UPDATE_ENTITY_URL=https://your.domain/webhook/update-entity
+
+# Optional authentication header
+CENTRAL_WEBHOOK_API_KEY=
+
+# Optional log level: INFO or DEBUG
+CENTRAL_WEBHOOK_LOG_LEVEL=INFO
+
+# Optional image tags used by compose.webhook.yml
+CENTRAL_WEBHOOK_TAG=latest
+POSTGRES_MAJOR=14

.github/workflows/test.yml

@@ -18,5 +18,3 @@ jobs:
       tag_override: ci
       compose_file: compose.yml
       compose_service: webhook
-      cache_extra_imgs: |
-        "docker.io/postgis/postgis:17-3.5-alpine"

.gitignore

@@ -22,3 +22,4 @@ go.work.sum
 
 # env file
 .env
+.env.webhook

Dockerfile

@@ -12,7 +12,7 @@ RUN CGO_ENABLED=0 GOOS=linux go build -o /app/centralwebhook
 
 # Run the tests in the container
 FROM build AS run-test-stage
-RUN go test -v ./...
+RUN go test -p 1 -v ./...
 
 
 # Add a non-root user to passwd file

README.md

@@ -37,12 +37,60 @@ The `centralwebhook` tool is a simple CLI that installs or uninstalls database t
 
 > [!NOTE]
 > **Using our helper images**: We provide PostgreSQL images with the `pgsql-http` extension pre-installed:
-> - `ghcr.io/hotosm/postgres:18-http` (based on vanilla PostgreSQL 18 images)
+> - `ghcr.io/hotosm/postgres:14-http` (recommended default for current ODK Central installs)
+> - `ghcr.io/hotosm/postgres:15-http`
+> - `ghcr.io/hotosm/postgres:16-http`
+> - `ghcr.io/hotosm/postgres:17-http`
+> - `ghcr.io/hotosm/postgres:18-http`
 >
 > These images are drop-in replacements for standard PostgreSQL images and simply add the extension.
 >
 > **Installing manually**: If you don't wish to use these images, you must install the `pgsql-http` extension yourself. The extension may require superuser privileges to install. If you cannot install it yourself, ask your database administrator.
 
+## ODK Central Quick Start (Recommended)
+
+For most users running a mostly-vanilla ODK Central setup, the easiest approach is to layer this repo's compose override on top of the official Central compose file.
+
+1. In your ODK Central repo/environment, copy the example file and set webhook variables:
+
+```bash
+cp /path/to/central-webhook/.env.example .env.webhook
+```
+
+Then merge the values you need into your Central `.env` (or export them in your shell):
+
+```dotenv
+# Required: provide at least one webhook URL
+CENTRAL_WEBHOOK_NEW_SUBMISSION_URL=https://your.domain/webhook/new-submission
+CENTRAL_WEBHOOK_REVIEW_SUBMISSION_URL=https://your.domain/webhook/review-submission
+CENTRAL_WEBHOOK_UPDATE_ENTITY_URL=https://your.domain/webhook/update-entity
+
+# Optional
+CENTRAL_WEBHOOK_API_KEY=your-secret-api-key
+CENTRAL_WEBHOOK_LOG_LEVEL=INFO
+
+# Optional image tags (defaults shown)
+CENTRAL_WEBHOOK_TAG=latest
+POSTGRES_MAJOR=14
+```
+
+2. Start Central + webhook trigger installer:
+
+```bash
+docker compose -f docker-compose.yml -f /path/to/central-webhook/compose.webhook.yml up -d
+```
+
+3. Verify installation:
+
+```bash
+docker compose logs webhook
+```
+
+You should see a successful `install` message after Central and PostgreSQL are healthy.
+
+> [!IMPORTANT]
+> Use the helper image major version that matches your Central database major version. Do not point a newer PostgreSQL container at an existing older data directory without a proper PostgreSQL upgrade process.
+
 ## Usage
 
 The `centralwebhook` tool is a CLI that installs or uninstalls database triggers. After installation, the triggers run automatically whenever audit events occur in the database.
@@ -233,6 +281,10 @@ The tool installs PostgreSQL triggers on the `audits` table that:
 
 The triggers run automatically after installation - no long-running service is needed.
 
+For `submission.update` events, if `instanceId` is missing from audit details in your
+Central version, the trigger resolves it from `submission_defs` using
+`submissionDefId`/`submissionId`.
+
 ## Development
 
 - This package uses the standard library and a Postgres driver.

compose.webhook.yml

@@ -14,7 +14,7 @@ services:
 
   # Override the image used to contain the `pgsql-http` extension and a healthcheck
   postgres14:
-    image: "ghcr.io/hotosm/postgres:14-http"
+    image: "ghcr.io/hotosm/postgres:${POSTGRES_MAJOR:-14}-http"
 
     healthcheck:
       test: pg_isready -U ${DB_USER:-odk} || exit 1
@@ -25,9 +25,9 @@ services:
 
   # The main webhook service
   webhook:
-    image: "ghcr.io/hotosm/central-webhook:1.0.2"
+    image: "ghcr.io/hotosm/central-webhook:${CENTRAL_WEBHOOK_TAG:-latest}"
     environment:
-      CENTRAL_WEBHOOK_DB_URI: postgresql://odk:odk@postgres14:5432/odk?sslmode=disable
+      CENTRAL_WEBHOOK_DB_URI: postgresql://${DB_USER:-odk}:${DB_PASSWORD:-odk}@postgres14:5432/${DB_NAME:-odk}?sslmode=disable
       CENTRAL_WEBHOOK_UPDATE_ENTITY_URL: ${CENTRAL_WEBHOOK_UPDATE_ENTITY_URL}
       CENTRAL_WEBHOOK_REVIEW_SUBMISSION_URL: ${CENTRAL_WEBHOOK_REVIEW_SUBMISSION_URL}
       CENTRAL_WEBHOOK_NEW_SUBMISSION_URL: ${CENTRAL_WEBHOOK_NEW_SUBMISSION_URL}

compose.yml

@@ -31,7 +31,8 @@ services:
     extra_hosts:
       - host.docker.internal:host-gateway
     restart: "no"
-    entrypoint: go test -timeout=2m -v ./...
+    # Use -p 1 to avoid race condition when testing
+    entrypoint: go test -p 1 -timeout=2m -v ./...
 
   db:
     image: "ghcr.io/hotosm/postgres:18-http"

db/trigger.go

@@ -44,10 +44,6 @@ func CreateTrigger(ctx context.Context, dbPool *pgxpool.Pool, tableName string,
 		url := quoteSQLString(*opts.UpdateEntityURL)
 		caseStatements += fmt.Sprintf(`
 			WHEN 'entity.update.version' THEN
-			DECLARE
-				entity_data jsonb;
-				entity_payload jsonb;
-			BEGIN
 				-- Deduplicate: only fire for the first audit row for this entity UUID
 				IF EXISTS (
 					SELECT 1
@@ -78,17 +74,16 @@ func CreateTrigger(ctx context.Context, dbPool *pgxpool.Pool, tableName string,
 					'data', entity_data
 				);
 
-				PERFORM http((
+				PERFORM (http((
 					'POST',
 					%s,
 					http_headers(%s),
 					'application/json',
 					entity_payload::text
-				)::http_request);
+				)::http_request)).status;
 
 				-- Mark as processed
 				NEW.details := NEW.details || '{"webhook_sent": true}'::jsonb;
-			END;
 		`, tableName, url, headersSQL)
 	}
 
@@ -99,16 +94,33 @@ func CreateTrigger(ctx context.Context, dbPool *pgxpool.Pool, tableName string,
 		url := quoteSQLString(*opts.NewSubmissionURL)
 		caseStatements += fmt.Sprintf(`
 			WHEN 'submission.create' THEN
-			DECLARE
-				submission_data jsonb;
-				submission_payload jsonb;
-			BEGIN
+				resolved_instance_id := NEW.details->>'instanceId';
+				IF (resolved_instance_id IS NULL OR resolved_instance_id = '')
+					AND (NEW.details->>'submissionDefId') IS NOT NULL THEN
+					SELECT submission_defs."instanceId"::text
+					INTO resolved_instance_id
+					FROM submission_defs
+					WHERE submission_defs.id = (NEW.details->>'submissionDefId')::int;
+				END IF;
+
+				IF resolved_instance_id IS NULL OR resolved_instance_id = '' THEN
+					RAISE WARNING 'Instance ID missing for submission.create event';
+					NEW.details := NEW.details || '{"webhook_sent": true}'::jsonb;
+					RETURN NEW;
+				END IF;
+
 				-- Deduplicate by instanceId
 				IF EXISTS (
 					SELECT 1
 					FROM %s a
 					WHERE a.action = 'submission.create'
-					AND a.details->>'instanceId' = NEW.details->>'instanceId'
+					AND (
+						a.details->>'instanceId' = resolved_instance_id
+						OR (
+							(NEW.details->>'submissionDefId') IS NOT NULL
+							AND a.details->>'submissionDefId' = NEW.details->>'submissionDefId'
+						)
+					)
 					AND a.details ? 'webhook_sent'
 				) THEN
 					-- Already processed
@@ -129,21 +141,20 @@ func CreateTrigger(ctx context.Context, dbPool *pgxpool.Pool, tableName string,
 
 				submission_payload := jsonb_build_object(
 					'type', 'submission.create',
-					'id', (NEW.details->>'instanceId'),
+					'id', resolved_instance_id,
 					'data', submission_data
 				);
 
-				PERFORM http((
+				PERFORM (http((
 					'POST',
 					%s,
 					http_headers(%s),
 					'application/json',
 					submission_payload::text
-				)::http_request);
+				)::http_request)).status;
 
 				-- Mark as processed
 				NEW.details := NEW.details || '{"webhook_sent": true}'::jsonb;
-			END;
 		`, tableName, url, headersSQL)
 	}
 
@@ -154,16 +165,50 @@ func CreateTrigger(ctx context.Context, dbPool *pgxpool.Pool, tableName string,
 		url := quoteSQLString(*opts.ReviewSubmissionURL)
 		caseStatements += fmt.Sprintf(`
 			WHEN 'submission.update' THEN
-			DECLARE
-				review_payload jsonb;
-			BEGIN
+				resolved_instance_id := NEW.details->>'instanceId';
+				resolved_review_state := COALESCE(NEW.details->>'reviewState', '');
+
+				-- Fallback for Central audit rows that omit instanceId
+				IF (resolved_instance_id IS NULL OR resolved_instance_id = '')
+					AND (NEW.details->>'submissionDefId') IS NOT NULL THEN
+					SELECT submission_defs."instanceId"::text
+					INTO resolved_instance_id
+					FROM submission_defs
+					WHERE submission_defs.id = (NEW.details->>'submissionDefId')::int;
+				END IF;
+				IF (resolved_instance_id IS NULL OR resolved_instance_id = '')
+					AND (NEW.details->>'submissionId') IS NOT NULL THEN
+					SELECT submission_defs."instanceId"::text
+					INTO resolved_instance_id
+					FROM submission_defs
+					WHERE submission_defs."submissionId" = (NEW.details->>'submissionId')::int
+					ORDER BY submission_defs.id DESC
+					LIMIT 1;
+				END IF;
+
+				IF resolved_instance_id IS NULL OR resolved_instance_id = '' THEN
+					RAISE WARNING 'Instance ID missing for submission.update event';
+					NEW.details := NEW.details || '{"webhook_sent": true}'::jsonb;
+					RETURN NEW;
+				END IF;
+
 				-- Deduplicate by instanceId + reviewState
 				IF EXISTS (
 					SELECT 1
 					FROM %s a
 					WHERE a.action = 'submission.update'
-					AND a.details->>'instanceId' = NEW.details->>'instanceId'
-					AND a.details->>'reviewState' = NEW.details->>'reviewState'
+					AND (
+						a.details->>'instanceId' = resolved_instance_id
+						OR (
+							(NEW.details->>'submissionDefId') IS NOT NULL
+							AND a.details->>'submissionDefId' = NEW.details->>'submissionDefId'
+						)
+						OR (
+							(NEW.details->>'submissionId') IS NOT NULL
+							AND a.details->>'submissionId' = NEW.details->>'submissionId'
+						)
+					)
+					AND COALESCE(a.details->>'reviewState', '') = resolved_review_state
 					AND a.details ? 'webhook_sent'
 				) THEN
 					-- Already processed
@@ -173,23 +218,22 @@ func CreateTrigger(ctx context.Context, dbPool *pgxpool.Pool, tableName string,
 
 				review_payload := jsonb_build_object(
 					'type', 'submission.update',
-					'id', (NEW.details->>'instanceId'),
+					'id', resolved_instance_id,
 					'data', jsonb_build_object(
 						'reviewState', NEW.details->>'reviewState'
 					)
 				);
 
-				PERFORM http((
+				PERFORM (http((
 					'POST',
 					%s,
 					http_headers(%s),
 					'application/json',
 					review_payload::text
-				)::http_request);
+				)::http_request)).status;
 
 				-- Mark as processed
 				NEW.details := NEW.details || '{"webhook_sent": true}'::jsonb;
-			END;
 		`, tableName, url, headersSQL)
 	}
 
@@ -204,6 +248,13 @@ func CreateTrigger(ctx context.Context, dbPool *pgxpool.Pool, tableName string,
 		$$
 		DECLARE
 			action_type text;
+			entity_data jsonb;
+			entity_payload jsonb;
+			submission_data jsonb;
+			submission_payload jsonb;
+			review_payload jsonb;
+			resolved_instance_id text;
+			resolved_review_state text;
 		BEGIN
 			action_type := NEW.action;
 
@@ -358,6 +409,16 @@ func RemoveTrigger(ctx context.Context, dbPool *pgxpool.Pool, tableName string)
 		}
 	}
 
+	// Verify function removal. This catches rare cases where a DROP is accepted
+	// due to a transient error but the function still exists.
+	functionExists, err := checkFunctionExists(ctx, conn)
+	if err != nil {
+		return fmt.Errorf("failed to verify function removal: %w", err)
+	}
+	if functionExists {
+		return fmt.Errorf("failed to remove function new_audit_log")
+	}
+
 	return nil
 }