fix: run ops psql as trakrf-migrate, add psql-super and a QUERY argument (TRA-1105)

.claude/csw-validate.sh

@@ -0,0 +1,63 @@
+#!/usr/bin/env bash
+# Local mirror of .github/workflows/ci.yml — run before opening a PR so CI
+# failures surface here instead of on the PR. Lives in .claude/ because this
+# repo gitignores all of .claude/; promoting it to a `just validate` recipe
+# (matching trakrf/platform) would need its own PR.
+set -euo pipefail
+cd "$(git rev-parse --show-toplevel)"
+
+fail=0
+run() { printf '\n\033[1m==> %s\033[0m\n' "$*"; "$@" || { fail=1; printf '\033[31mFAILED: %s\033[0m\n' "$*"; }; }
+
+# --- ops-lib unit tests (job: ops-lib) ---
+run ./scripts/test-ops-lib.sh
+
+# --- tofu fmt (job: tofu-fmt) ---
+run tofu fmt -check -recursive terraform
+
+# --- tofu validate (job: tofu-validate, matrix dir) ---
+#
+# `tofu init` records provider hashes for the current platform into the
+# tracked .terraform.lock.hcl files. CI does this in a throwaway checkout and
+# never notices; run locally it leaves unrelated lockfile churn staged into
+# whatever you commit next. Validation must not mutate the repo, so note
+# which lockfiles are clean going in and restore exactly those afterwards —
+# a lockfile you had already edited on purpose is left alone.
+clean_locks=()
+while IFS= read -r f; do
+  [ -n "$f" ] || continue
+  git diff --quiet -- "$f" 2>/dev/null && clean_locks+=("$f")
+done < <(git ls-files 'terraform/*/.terraform.lock.hcl')
+
+for d in aws azure cloudflare bootstrap gcp; do
+  run tofu -chdir="terraform/$d" init -backend=false -input=false
+  run tofu -chdir="terraform/$d" validate
+done
+
+if [ "${#clean_locks[@]}" -gt 0 ]; then
+  git checkout -- "${clean_locks[@]}"
+fi
+
+# --- helm lint + template (job: helm, matrix chart x cluster) ---
+for c in cert-manager-config traefik-config trakrf-backend trakrf-db; do
+  for k in eks aks; do
+    run helm lint "helm/$c" -f "helm/$c/values.yaml" -f "helm/$c/values-$k.yaml"
+    run helm template "helm/$c" -f "helm/$c/values.yaml" -f "helm/$c/values-$k.yaml"
+  done
+done
+
+# --- helm-mosquitto (job: helm-mosquitto, GKE-only) ---
+run helm lint helm/trakrf-mosquitto -f helm/trakrf-mosquitto/values.yaml -f helm/trakrf-mosquitto/values-gke.yaml
+run helm template helm/trakrf-mosquitto -f helm/trakrf-mosquitto/values.yaml -f helm/trakrf-mosquitto/values-gke.yaml \
+  --set hostname=mqtt.preview.gke.trakrf.id --set loadBalancerIP=1.2.3.4
+
+# --- argocd-root (job: argocd-root, matrix cluster) ---
+for k in eks aks; do
+  run helm template trakrf-root argocd/root --set cluster="$k" \
+    --set certManagerIdentityClientId=fake --set tenantId=fake --set subscriptionId=fake \
+    --set dnsZoneResourceGroup=fake --set traefikLbIp=1.2.3.4 --set mainResourceGroupName=fake
+done
+
+printf '\n'
+if [ "$fail" -ne 0 ]; then printf '\033[31mvalidate: FAILED\033[0m\n'; exit 1; fi
+printf '\033[32mvalidate: OK\033[0m\n'

.claude/csw.json

@@ -0,0 +1,9 @@
+{
+  "ticketPrefix": "TRA",
+  "tracker": "linear",
+  "baseBranch": "main",
+  "defaultType": "feat",
+  "validate": ".claude/csw-validate.sh",
+  "worktreeDir": ".claude/worktrees",
+  "branchPattern": "<type>/<ticket>-<slug>"
+}

.github/workflows/ci.yml

@@ -6,6 +6,16 @@ on:
     branches: [main]
 
 jobs:
+  ops-lib:
+    # Unit tests for scripts/ops-lib.sh, the shared helper library behind the
+    # per-env ops recipes in the justfile. Pure bash with stubbed kubectl —
+    # no cluster, no credentials.
+    name: ops-lib tests
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - run: ./scripts/test-ops-lib.sh
+
   tofu-fmt:
     name: tofu fmt
     runs-on: ubuntu-latest

.gitignore

@@ -43,4 +43,11 @@ backend.conf
 
 # Ignore .terraform.lock.hcl file (uncomment if you prefer not to version this file)
 # .terraform.lock.hcl
-.claude/
+# Git worktrees — Claude Code's native worktree tool writes here; this is the
+# canonical location (gitignored). Narrow ignore: the rest of .claude/ stays
+# tracked, so shared agent config (csw.json) is versioned with the repo.
+.claude/worktrees/
+
+# Claude Code local settings and per-session scratch
+.claude/settings.local.json
+.claude/*.lock

CLAUDE.md

@@ -25,7 +25,7 @@
 - Merge PRs with `--merge` (never `--squash` or `--rebase`)
 
 ## Worktrees
-- **Canonical location**: worktrees go in `.claude/worktrees/<name>/` (repo-relative), one per branch — gitignored as part of the `.claude/` ignore, and the same convention across all trakrf repos (docs/platform/infra). Since this repo ignores all of `.claude/`, this CLAUDE.md line is the only in-repo signal of the convention — keep it explicit.
+- **Canonical location**: worktrees go in `.claude/worktrees/<name>/` (repo-relative), one per branch — gitignored by the narrow `.claude/worktrees/` rule, and the same convention across all trakrf repos (docs/platform/infra). The ignore is deliberately narrow (matching trakrf/docs) so the rest of `.claude/` stays tracked and shared agent config is versioned: `.claude/csw.json` (csw workflow config — tracker, base branch, validate command) and `.claude/csw-validate.sh` (local mirror of `.github/workflows/ci.yml`; run it before opening a PR).
 - **Create** with the native `EnterWorktree` tool (writes to `.claude/worktrees/<name>`, auto-creates branch `worktree-<name>` — rename to a `feat/...`/`fix/...` branch after if desired). Do NOT use manual `git worktree add`, and do NOT create a `.worktrees/` dir or a `.claude/worktrees -> ../.worktrees` symlink (fresh-clone footgun). Manual `git worktree` is only the superpowers fallback for harnesses with no native tool — not us.
 - **Cleanup**: `git worktree list` is authoritative (empty leftover dirs are not worktrees); use `ExitWorktree` (or `git worktree remove`) to leave/remove.
 

docs/ops.md

@@ -160,8 +160,9 @@ to a namespace that does not exist.
 **Guard rule.** Unguarded recipes (`pods`, `logs`, `rollout`, `db-status`,
 `psql`, `mqtt-logs`, `mqtt-sub`, `argo-status`) run against both
 environments with no confirmation prompt. That is not the same as
-read-only — `psql` in particular opens a superuser session that can write;
-see the note in §5. Mutating recipes (`backend-restart`, `set-log-level`)
+read-only — `psql` in particular opens a session that owns the schema and
+can write; see the note in §5. Mutating recipes (`backend-restart`,
+`set-log-level`, `psql-super`)
 prompt before touching prod: they print what they are about to do and
 require you to type `prod`. `argo-sync` prompts for every app **except**
 `*-preview` ones — that includes `*-prod` apps, but also the cluster-scoped
@@ -185,33 +186,88 @@ metacharacters.
 
 ## 5. Database
 
-### Interactive psql
+### psql
+
+Interactive with no `QUERY`, a one-shot `psql -c` with one:
 
 ```sh
 just psql preview
-just psql prod
+just psql prod "SELECT version, dirty FROM trakrf.schema_migrations;"
 ```
 
 Raw equivalent:
 
 ```sh
-kubectl -n trakrf-prod exec -it "$(kubectl -n trakrf-prod get pod -l cnpg.io/instanceRole=primary -o jsonpath='{.items[0].metadata.name}')" -c postgres -- psql -U postgres -d trakrf
+kubectl -n trakrf-prod exec -it "$(kubectl -n trakrf-prod get pod -l cnpg.io/instanceRole=primary -o jsonpath='{.items[0].metadata.name}')" -c postgres -- env PGOPTIONS='-c role=trakrf-migrate' psql -U postgres -d trakrf
 ```
 
 The primary is resolved by the `cnpg.io/instanceRole=primary` label rather
 than a fixed pod name, so it follows a failover automatically — you always
-land on whichever instance is currently primary. Auth is superuser via
-in-pod peer auth over the unix socket, so no password is involved. The
-database name is `trakrf` in both namespaces (the namespace is what
-separates the environments, not the database name).
-
-> **`just psql prod` is not read-only.** It drops you into a superuser
-> session — `psql -U postgres -d trakrf` — that can insert, update, or drop
-> anything in the database. It runs unguarded, with no `confirm_prod`
-> prompt, because it opens a shell rather than performing one describable
-> operation `confirm_prod` could gate. The lack of a prompt reflects that,
-> not that the recipe is safe. Treat every statement you type inside it as
-> if it were already committed against production.
+land on whichever instance is currently primary. The database name is
+`trakrf` in both namespaces (the namespace is what separates the
+environments, not the database name).
+
+**The session runs as `trakrf-migrate`, not as a superuser.** Connection auth
+is still in-pod peer auth as `postgres` over the unix socket — that is the
+only credential available without threading a password in — but `PGOPTIONS`
+applies the equivalent of `SET ROLE "trakrf-migrate"` at connect time, so
+anything you CREATE is owned by the same role migrations run as.
+
+This matters because a `postgres`-owned object in the `trakrf` schema is
+permanently un-replaceable by a later migration: `CREATE OR REPLACE`, `DROP`
+and `ALTER … OWNER TO` all require ownership. The migration aborts partway,
+golang-migrate leaves the ledger dirty, and because the migrate Job is an
+ArgoCD PreSync hook the Deployment is never updated — the old pod keeps
+serving while ArgoCD reports Healthy and CI stays green. That is exactly how
+preview wedged in TRA-1104.
+
+It is a guardrail, not a security boundary: `session_user` is still the
+`postgres` superuser, so `SET ROLE postgres` escapes it deliberately. Plain
+`RESET ROLE` does **not** — the role arrives in the connection's startup
+packet, so it becomes the session default that `RESET` returns to. What
+changed is the default, so drift is no longer created by accident.
+
+When the output is going to be piped or read by a script, note that the
+`→ namespace/pod` banner is written to **stderr**, so stdout carries only
+psql's output. A one-shot query runs with `ON_ERROR_STOP=1`, so a failing
+statement exits non-zero.
+
+`QUERY` may span multiple lines and contain quotes, `$` and backticks — it is
+passed through as a single argument. That is what makes a real audit runnable
+without a hand-rolled `kubectl exec`, for example the ownership sweep from
+platform's `findOwnershipDrift` (`backend/internal/cmd/migrate/ownership.go`):
+
+```sh
+just psql preview "$(cat drift.sql)"
+```
+
+Run that one as `psql`, **not** `psql-super`: it filters on
+`pg_has_role(CURRENT_USER, ...)`, and a superuser is implicitly a member of
+every role, so it would report a false clean.
+
+> **`just psql prod` is not read-only.** `trakrf-migrate` owns the schema:
+> it can still insert, update, and drop. It runs unguarded, with no
+> `confirm_prod` prompt, because it opens a shell rather than performing one
+> describable operation `confirm_prod` could gate. The lack of a prompt
+> reflects that, not that the recipe is safe. Treat every statement you type
+> inside it as if it were already committed against production.
+
+### psql-super — superuser, deliberate opt-in
+
+```sh
+just psql-super preview
+YES=1 just psql-super prod 'ALTER FUNCTION f() OWNER TO "trakrf-migrate"'
+```
+
+Identical to `psql` but with no `SET ROLE` — a raw `postgres` superuser
+session. Reach for it only for what genuinely needs superuser: repairing
+ownership drift, and role or extension management. Anything you CREATE here
+is owned by `postgres` and becomes the next wedge.
+
+Unlike `psql`, this one **prompts before prod** (`confirm_prod`) and fails
+closed without a tty, so nothing scripted falls through it — set `YES=1` to
+proceed deliberately. It also prints a warning banner on stderr in both
+environments.
 
 ### Cluster health
 
@@ -474,7 +530,7 @@ Two ❌ auth lines above the unreachable namespaces means this is an auth
 failure wearing a network error's clothes. Unreachable namespaces with the
 first two lines green is the real transient.
 
-### ``Recipe `psql` got 0 arguments but takes 1``
+### ``Recipe `psql` got 0 arguments but takes at least 1``
 
 `ENV` is required by design on every per-env recipe — there is no default
 that could silently mean prod. Name the environment explicitly:

justfile

@@ -1095,18 +1095,58 @@ db-restore-pitr-test ENV TARGET_TIME="":
     scratch_applied=0
     echo "PITR restore proof complete for {{ ENV }} (source ${src_cluster})."
 
-# Interactive psql on the CNPG primary. Superuser via in-pod peer auth.
+# The session runs as trakrf-migrate, NOT the postgres superuser, so that
+# hand-run DDL is owned by the same role migrations run as. A postgres-owned
+# object in the trakrf schema is permanently un-replaceable by a later
+# migration (CREATE OR REPLACE, DROP and ALTER .. OWNER TO all need
+# ownership), which is what wedged preview deploys in TRA-1104. Use
+# `just psql-super` to opt into superuser deliberately.
+#
+# Interactive with no QUERY, one-shot `psql -c` with one.
 #   just psql preview
-#   just psql prod
-psql ENV:
+#   just psql prod "SELECT version, dirty FROM trakrf.schema_migrations;"
+psql ENV QUERY="":
+    #!/usr/bin/env bash
+    set -euo pipefail
+    source scripts/ops-lib.sh
+    require_env "{{ ENV }}"
+    ns="trakrf-{{ ENV }}"
+    pod=$(cnpg_primary_pod "$ns")
+    # quote() rather than interpolating QUERY into a bare double-quoted
+    # assignment: just substitutes into the recipe body textually, so SQL
+    # containing a double quote (a quoted identifier such as
+    # "trakrf-migrate") would otherwise terminate the string and mangle the
+    # statement. quote() emits a properly single-quoted shell word, so
+    # embedded quotes, $ and backticks all survive verbatim.
+    #
+    # Do NOT write an interpolation of QUERY inside a comment here. just
+    # expands them in comments too, and a multi-line value then spills past
+    # the leading # and executes as shell.
+    query={{ quote(QUERY) }}
+    # Banner on stderr so `just psql ENV "SELECT .."` stays pipeable.
+    echo "→ $ns/$pod (database: trakrf, role: trakrf-migrate)" >&2
+    db_psql "$ns" "$pod" trakrf-migrate "$query"
+
+# Prefer `just psql` for everything except what genuinely needs superuser:
+# repairing ownership drift, and role/extension management. Any object you
+# CREATE in this session is owned by postgres and will block a future
+# migration — see the note on `psql` above. Prompts before prod.
+#
+# Superuser psql on the CNPG primary — deliberate opt-in.
+#   just psql-super preview
+#   YES=1 just psql-super prod 'ALTER FUNCTION f() OWNER TO "trakrf-migrate"'
+psql-super ENV QUERY="":
     #!/usr/bin/env bash
     set -euo pipefail
     source scripts/ops-lib.sh
     require_env "{{ ENV }}"
     ns="trakrf-{{ ENV }}"
     pod=$(cnpg_primary_pod "$ns")
-    echo "→ $ns/$pod (database: trakrf)"
-    kubectl -n "$ns" exec -it "$pod" -c postgres -- psql -U postgres -d trakrf
+    query={{ quote(QUERY) }}
+    confirm_prod "{{ ENV }}" "open a SUPERUSER psql session (objects you create here are owned by postgres)"
+    echo "⚠️  SUPERUSER session on $ns/$pod — anything you CREATE is owned by" >&2
+    echo "   postgres and will block a future migration. Prefer 'just psql {{ ENV }}'." >&2
+    db_psql "$ns" "$pod" postgres "$query"
 
 # CNPG cluster health plus its instance pods.
 #   just db-status prod

scripts/ops-lib.sh

@@ -91,3 +91,64 @@ cnpg_primary_pod() {
     fi
     echo "$pod"
 }
+
+# db_psql <namespace> <pod> <role> [query]
+# Run psql against the CNPG primary's `trakrf` database.
+#
+# Auth is always in-pod peer auth as the `postgres` superuser over the unix
+# socket — that is the only credential available without threading a password
+# in. What <role> controls is the role the SESSION then runs as:
+#
+#   trakrf-migrate  ->  PGOPTIONS='-c role=trakrf-migrate' makes the backend
+#                       apply the equivalent of SET ROLE at connect time, so
+#                       DDL typed in the session is owned by trakrf-migrate —
+#                       the role migrations run as, and therefore the only
+#                       owner that keeps an object replaceable by a later
+#                       migration (TRA-1105, after the TRA-1104 wedge).
+#   postgres        ->  no PGOPTIONS; a raw superuser session.
+#
+# This is a guardrail, not a security boundary: session_user is still the
+# postgres superuser, so `SET ROLE postgres` escapes it. Note that plain
+# `RESET ROLE` does NOT — the role arrived in the startup packet, so it is
+# the session default that RESET returns to. The point is that the DEFAULT
+# stops silently minting postgres-owned objects, not that escape is
+# impossible.
+#
+# An empty <query> opens an interactive shell (-it). A non-empty one runs
+# `psql -c` with ON_ERROR_STOP=1 and no tty (-i), so the output is clean
+# enough to pipe and a failing statement sets a non-zero exit status.
+db_psql() {
+    local ns="${1:-}" pod="${2:-}" role="${3:-}" query="${4:-}"
+
+    if [ -z "$ns" ]; then
+        echo "ERROR: db_psql requires a namespace" >&2
+        return 1
+    fi
+    if [ -z "$pod" ]; then
+        echo "ERROR: db_psql requires a pod" >&2
+        return 1
+    fi
+    if [ -z "$role" ]; then
+        echo "ERROR: db_psql requires a role" >&2
+        return 1
+    fi
+
+    # kubectl exec cannot set an environment variable on the remote process,
+    # so PGOPTIONS is applied by exec'ing through env(1) inside the container.
+    local -a role_env=()
+    if [ "$role" != "postgres" ]; then
+        role_env=(env "PGOPTIONS=-c role=$role")
+    fi
+
+    # ${arr[@]+"${arr[@]}"} — expanding an empty array as plain "${arr[@]}"
+    # is an unbound-variable error under `set -u` on bash before 4.4.
+    if [ -n "$query" ]; then
+        kubectl -n "$ns" exec -i "$pod" -c postgres -- \
+            ${role_env[@]+"${role_env[@]}"} \
+            psql -U postgres -d trakrf -v ON_ERROR_STOP=1 -c "$query"
+    else
+        kubectl -n "$ns" exec -it "$pod" -c postgres -- \
+            ${role_env[@]+"${role_env[@]}"} \
+            psql -U postgres -d trakrf
+    fi
+}