fix: match consultations schema exactly - drop id, loaded_at

The MariaDB consultations table now matches the original schema:
source, name, description, status, agency, tags, region, url,
publishdate, expirydate. Removes the extraneous id and loaded_at
columns that were added since the Python-to-SQL migration. Also enable
setting target table name via helm chart.

Author: adonm

.github/workflows/ci-nightly.yaml

@@ -1,6 +1,8 @@
 name: Nightly CI
 
 on:
+  push:
+    branches: [main]
   schedule:
     - cron: '0 14 * * *'  # 10 PM AWST
   workflow_dispatch:

chart/Chart.yaml

@@ -2,5 +2,5 @@ apiVersion: v2
 name: harvest-consultations
 description: DuckDB harvest pipeline for WA government consultation data
 type: application
-version: 0.5.0
+version: 0.5.3
 appVersion: "1.5.2"

chart/harvest.sql

@@ -218,17 +218,15 @@ FROM engagementhq_raw;
 CREATE OR REPLACE TABLE consultations_final AS
 SELECT
     source,
-    id,
     name,
     description,
     status,
-    tags,
     agency,
+    tags,
     region,
     url,
     publishdate,
-    expirydate,
-    CURRENT_TIMESTAMP AS loaded_at
+    expirydate
 FROM (
     SELECT * FROM engagementhq_std
     UNION ALL BY NAME
@@ -246,5 +244,5 @@ ORDER BY source, status;
 --    Matches old/harvest.py export behaviour: replace the whole output table.
 -- ============================================================================
 ATTACH '' AS mysqldb (TYPE mysql);
-CREATE OR REPLACE TABLE mysqldb.consultations AS
+CREATE OR REPLACE TABLE mysqldb.{{ .Values.mysql.table }} AS
 SELECT * FROM consultations_final;

chart/templates/configmap.yaml

@@ -7,4 +7,4 @@ metadata:
     {{- include "harvest-consultations.labels" . | nindent 4 }}
 data:
   harvest.sql: |
-{{ .Files.Get "harvest.sql" | indent 4 }}
+{{ tpl (.Files.Get "harvest.sql") . | indent 4 }}

chart/values.yaml

@@ -1,6 +1,7 @@
 mysql:
   host: mariadb
   database: harvest
+  table: consultations
 
 mariadb:
   # rootPassword is only used for MariaDB initialization; the application and

justfile

@@ -1,5 +1,6 @@
 ns := "harvest-consultations"
-helmHost := "mariadb"
+mysqlHost := "mariadb"
+table := "consultations"
 
 default:
   just --choose
@@ -13,16 +14,13 @@ kind-up:
   kind get clusters | grep -q harvest || kind create cluster --name harvest
   helm upgrade --install harvest chart \
     --namespace {{ns}} --create-namespace \
-    --set mysql.host={{helmHost}}
+    --set mysql.host={{mysqlHost}} \
+    --set mysql.table={{table}}
 
 # Forward mariadb from k8s cluster
 mariadb-svc: kind-up
   ss -ltpn | grep 3306 || kubectl port-forward service/mariadb 3306:3306 -n {{ns}} & sleep 1
 
-# Run the DuckDB pipeline locally
-run:
-  duckdb -c ".read chart/harvest.sql"
-
 # Create a one-off test job in the cluster
 test: kind-up
   kubectl delete job test -n {{ns}} --ignore-not-found
@@ -32,7 +30,8 @@ test: kind-up
 helm-install:
   helm upgrade --install harvest chart \
     --namespace {{ns}} --create-namespace \
-    --set mysql.host={{helmHost}}
+    --set mysql.host={{mysqlHost}} \
+    --set mysql.table={{table}}
 
 # Package helm chart
 helm-package:
@@ -52,7 +51,8 @@ ci-test:
   echo "=== CI: installing helm chart ==="
   helm upgrade --install harvest chart \
     --namespace {{ns}} --create-namespace \
-    --set mysql.host={{helmHost}}
+    --set mysql.host={{mysqlHost}} \
+    --set mysql.table={{table}}
 
   echo "=== CI: waiting for MariaDB (up to 5 min) ==="
   kubectl rollout status statefulset/mariadb -n {{ns}} --timeout=300s
@@ -68,16 +68,16 @@ ci-test:
     exit 1
   }
 
-  echo "=== CI: dumping consultations table ==="
+  echo "=== CI: dumping {{table}} table ==="
   POD=$(kubectl get pod -l app=mariadb -n {{ns}} -o jsonpath='{.items[0].metadata.name}')
   mkdir -p dist
   kubectl exec -n {{ns}} "$POD" -- \
-    sh -c 'MYSQL_PWD="$MARIADB_PASSWORD" exec mariadb-dump -u"$MARIADB_USER" -h 127.0.0.1 harvest consultations' \
-    | gzip > dist/consultations.sql.gz
+    sh -c 'MYSQL_PWD="$MARIADB_PASSWORD" exec mariadb-dump -u"$MARIADB_USER" -h 127.0.0.1 harvest {{table}}' \
+    | gzip > dist/{{table}}.sql.gz
 
   echo "=== CI: validating dump ==="
-  gunzip -c dist/consultations.sql.gz | head -20 || true
-  ROWS=$(gunzip -c dist/consultations.sql.gz | grep -c 'INSERT INTO' || echo 0)
+  gunzip -c dist/{{table}}.sql.gz | head -20 || true
+  ROWS=$(gunzip -c dist/{{table}}.sql.gz | grep -c 'INSERT INTO' || echo 0)
   echo "Rows found: $ROWS"
   if [ "$ROWS" -eq 0 ]; then
     echo "ERROR: dump contains no INSERT statements"