PMM-15331 Gate health on SEP provisioning

.env.example

@@ -43,6 +43,65 @@ PMM_PORT_HTTPS=443
 # PMM_POSTGRES_SSL_CERT_PATH=/tmp/certs/pmm-managed.crt
 # PMM_DISABLE_BUILTIN_POSTGRES=1
 
+# Expose the built-in PostgreSQL to SEP running in a side container on the same bridge
+# network. PMM creates a dedicated `sep` database owned by a non-superuser `sep` role and
+# accepts connections for it from the container's Docker subnets only; nothing is
+# published on the host. Both variables are required to enable it.
+# PMM_ENABLE_SEP=1
+# PMM_SEP_POSTGRES_PASSWORD=<password>
+
+# With PMM_ENABLE_SEP set, PMM also provisions the Grafana service account SEP
+# authenticates users against, and writes its token to /srv/sep-secrets — the
+# `pmm-sep-secrets` volume, which the SEP container mounts read-only with SECRETS_DIR
+# pointing at it. The same token value is written to two files, one per canonical SEP
+# settings name:
+#   AUTH__PROVIDER__GRAFANA__SERVICE_ACCOUNT_TOKEN
+#   PMM__API_KEY
+# The directory is setgid mode 2770 and the files 0640, so both files belong to group 0
+# whichever uid PMM runs as. PMM cannot chgrp to SEP's own group, so the SEP container
+# has to join group 0 — `group_add: ["0"]`, or `user: "1001:0"` — or it cannot read them.
+# Leave both settings names above unset on the SEP container rather than empty: an empty
+# value counts as supplied there and outranks the file, which then goes unused.
+# The token is revalidated against Grafana on every PMM start, and replaced when the
+# file is missing or empty or Grafana rejects it, so restarting PMM does not revoke the
+# token a running SEP holds.
+# Unsetting PMM_ENABLE_SEP removes both files on the next start; the Grafana service
+# account is left in place.
+# With PMM_ENABLE_SEP set, the container reports healthy only once the current start has
+# published both files above, so gate the SEP container on
+# `depends_on: {pmm-server: {condition: service_healthy}}` — it reads them only at process
+# start. Under PMM_HA_ENABLE or PMM_DISABLE_BUILTIN_POSTGRES, where PMM_ENABLE_SEP is
+# ignored outright, the gate does not apply.
+# With an external Grafana database — GF_DATABASE_URL or GF_DATABASE_HOST, without either flag
+# above — the container never reports healthy, because PMM cannot provision the service
+# account in a database it does not own. Point SEP at its own Grafana, or leave
+# PMM_ENABLE_SEP unset.
+
+# PMM writes SEP's remaining deployment secrets to the same directory, with the same mode
+# and group, one file per canonical SEP settings name:
+#   SECRET_KEY                      generated once on first start and persisted on the
+#                                   /srv volume, so restarting PMM keeps every SEP
+#                                   session valid
+#   SEP__DATABASE__PASSWORD         PMM_SEP_POSTGRES_PASSWORD verbatim, rewritten on
+#   INVENTORY__DATABASE__PASSWORD   every start so rotating it takes effect on the next
+#   TASKS__DATABASE__PASSWORD       restart
+# SECRET_KEY is persisted outside the secrets directory, at /srv/.sep_secret_key mode 0600,
+# and copied from there on every start. A fresh /srv volume mints a new one; an existing one
+# is never overwritten, so back /srv up to keep SEP's sessions across a redeployment.
+# Do not pass SECRET_KEY to the SEP container — an environment value outranks the file, and
+# a PMM that later mints its own key then diverges from it silently. As with the token names
+# above, leave the three canonical *__DATABASE__PASSWORD names unset there rather than empty.
+# Give PMM_SEP_POSTGRES_PASSWORD no leading or trailing whitespace: PostgreSQL keeps it, SEP
+# strips it when reading the file, and the two then disagree.
+# Rotating PMM_SEP_POSTGRES_PASSWORD takes two restarts: restart PMM to move the database and
+# rewrite the files, then restart the SEP container, which reads them only at process start.
+# Once SEP is actually in use, PMM Server stops rather than degrading SEP alone: an empty
+# PMM_SEP_POSTGRES_PASSWORD, or a /srv/sep-secrets it cannot write to, is fatal. Neither
+# check applies under PMM_HA_ENABLE or PMM_DISABLE_BUILTIN_POSTGRES, which ignore
+# PMM_ENABLE_SEP outright and leave no SEP database to hold a password for.
+# Unsetting PMM_ENABLE_SEP removes all four files on the next start; the persisted
+# SECRET_KEY is left in place, so re-enabling SEP keeps existing sessions valid.
+
 # Use SSL certificates for PMM Server's internal database connection (PostgreSQL)
 # GF_DATABASE_SSL_MODE=verify-full
 # GF_DATABASE_CA_CERT_PATH=/tmp/certs/root.crt

api-tests/server/logs_test.go

@@ -68,6 +68,7 @@ func TestDownloadLogs(t *testing.T) {
 		"prometheus.base.yml",
 		"qan-api2.ini",
 		"qan-api2.log",
+		"sep-provision.log",
 		"supervisorctl_status.log",
 		"supervisord.conf",
 		"supervisord.log",

build/AGENTS.md

@@ -37,6 +37,7 @@ Source code (Go, TypeScript)
 | `grafana` | Install Grafana, provision datasources and dashboards |
 | `nginx` | Configure Nginx as reverse proxy (SSL termination, routing) |
 | `postgres` | Install and configure PostgreSQL for pmm-managed |
+| `sep` | SEP side-car integration helpers copied into the image and invoked from the entrypoint (no `tasks/`); publishes SEP's secrets when `PMM_ENABLE_SEP` is set |
 | `supervisord` | Configure Supervisord for process management |
 | `dashboards` | Provision PMM Grafana dashboards |
 | `initialization` | PMM Server first-run setup |
@@ -84,6 +85,7 @@ make rpmbuild-el9         # Build RPM build environment image
 
 - `build/docker/server/Dockerfile.el9` — PMM Server Docker image definition
 - `build/docker/server/entrypoint.sh` — Server container entrypoint
+- `build/docker/server/healthcheck.sh` — Server container healthcheck
 - `build/ansible/pmm-docker/main.yml` — Docker provisioning playbook
 - `build/ansible/roles/` — All Ansible roles for server components
 - `build/packages/rpm/server/SPECS/` — RPM spec files for server components

build/ansible/pmm-docker/post-build.yml

@@ -67,6 +67,20 @@
         - directory
       no_log: true
 
+    # Must exist in the image: a named volume mounted at a path the image lacks is
+    # created root:root, which pmm-server cannot write to. Group-writable like the
+    # other /srv directories, so an arbitrary runtime uid in group 0 can still write.
+    # Setgid so the token files written here belong to group 0 whatever that uid's
+    # primary group is - the SEP container joins group 0 to read them. Docker carries
+    # the bit into a fresh named volume mounted at this path.
+    - name: Create the SEP secrets mountpoint
+      file:
+        path: /srv/sep-secrets
+        state: directory
+        owner: pmm
+        group: root
+        mode: 02770
+
     - name: Create empty log directory for nginx
       file:
         path: /var/log/nginx

build/ansible/roles/grafana/files/grafana-sep

@@ -0,0 +1,286 @@
+#!/bin/bash
+#
+# Provisions the Grafana service account SEP authenticates with, and publishes its
+# token to a secrets directory the SEP container mounts, one file per canonical SEP
+# settings name. The files are 0640 and take group 0 from the setgid directory
+# post-build.yml ships - nothing here sets their group, so that bit must stay.
+#
+# Everything below is a no-op unless PMM_ENABLE_SEP is set. When it is unset again,
+# the token files this script wrote are removed on the next start; the Grafana
+# service account itself is left in place.
+#
+# Runs under supervisord rather than from the entrypoint because Grafana is neither
+# running nor migrated - on a first boot its schema does not exist - until long after
+# the entrypoint has exec'd supervisord.
+
+set -o errexit
+set -o pipefail
+
+declare SECRETS_DIR="${PMM_SEP_SECRETS_DIR:-/srv/sep-secrets}"
+declare GRAFANA_URL="http://127.0.0.1:3000"
+declare POSTGRES_BIN_DIR="${POSTGRES_BIN_DIR:-/usr/pgsql-14/bin}"
+declare SA_NAME="pmm-sep"
+declare TOKEN_NAME="pmm-sep"
+# Generous because a failed run is not retried: the one-shot cannot bound its own
+# restarts under supervisord. The image's HEALTHCHECK start-period is sized on this.
+declare READY_TIMEOUT=600
+declare READY_INTERVAL=5
+declare -a TOKEN_FILES=(AUTH__PROVIDER__GRAFANA__SERVICE_ACCOUNT_TOKEN PMM__API_KEY)
+declare TMP_FILE=""
+declare SEP_MARKER="/srv/.sep_provisioned"
+declare MARK_RUN=1
+
+# status must be captured first: the TMP_FILE test below would overwrite $?.
+cleanup() {
+    local status=$?
+
+    [ -n "$TMP_FILE" ] && rm -f "$TMP_FILE"
+    if [ "$status" -eq 0 ] && [ "$MARK_RUN" -eq 1 ] && ! : > "$SEP_MARKER"; then
+        echo "WARNING: could not write $SEP_MARKER, the container will not report healthy." >&2
+    fi
+    return 0
+}
+trap cleanup EXIT
+
+is_enabled() { [ "$1" = "1" ] || [ "$1" = "true" ]; }
+
+psql_grafana() {
+    "$POSTGRES_BIN_DIR/psql" -X -q -v ON_ERROR_STOP=1 -U postgres -h /run/postgresql -d grafana "$@"
+}
+
+# Same readiness signal pmm-managed waits on: /api/health reporting the database as ok.
+wait_for_grafana() {
+    local waited=0
+
+    while [ "$waited" -lt "$READY_TIMEOUT" ]; do
+        if curl -sS --max-time 5 "$GRAFANA_URL/api/health" 2> /dev/null |
+            grep -q '"database": *"ok"'; then
+            return 0
+        fi
+        sleep "$READY_INTERVAL"
+        waited=$((waited + READY_INTERVAL))
+    done
+
+    return 1
+}
+
+# Prints the status of an authenticated call to the access-control endpoint SEP's
+# org-role lookups go through, or 000 when the call could not be made at all. Fed on
+# stdin rather than -H so the token never appears in any process's argv.
+#
+# Unreachable is kept distinct from rejected, and retried, because a transient failure
+# is not a revoked token: reading one as the other re-mints on the reuse path below.
+grafana_auth_status() {
+    local code attempt=0
+
+    while :; do
+        code=$(printf 'header = "Authorization: Bearer %s"\n' "$1" |
+            curl -sS -K - --max-time 10 -o /dev/null -w '%{http_code}' \
+                "$GRAFANA_URL/api/access-control/user/permissions") || code="000"
+
+        attempt=$((attempt + 1))
+        if [ "$code" != "000" ] || [ "$attempt" -ge 3 ]; then
+            break
+        fi
+        sleep "$READY_INTERVAL"
+    done
+
+    printf '%s' "$code"
+}
+
+# The validator endpoint answers 200 for any role, so a service account demoted in the
+# Grafana UI would keep authenticating while SEP's org-role lookups silently degrade.
+ensure_admin_role() {
+    psql_grafana -v sa_name="$SA_NAME" <<'SQL'
+UPDATE org_user ou
+SET role = 'Admin', updated = now()
+FROM "user" u
+WHERE u.id = ou.user_id
+  AND u.login = 'sa-' || ou.org_id || '-' || :'sa_name'
+  AND ou.role <> 'Admin';
+SQL
+}
+
+# Prints, one per line and in the order the caller unpacks them: the token value, its
+# api_key.key hash, and the user row's salt, rands and uid. The encoding reproduces
+# Grafana's own - crc32("glsa_" + secret) little-endian as the checksum,
+# util.EncodePassword(secret, checksum) as the stored hash.
+generate_token_material() {
+    python3 - <<'PY'
+import binascii
+import hashlib
+import secrets
+import string
+import zlib
+
+alnum = string.ascii_letters + string.digits
+secret = "".join(secrets.choice(alnum) for _ in range(32))
+checksum = binascii.hexlify(
+    zlib.crc32(("glsa_" + secret).encode()).to_bytes(4, "little")
+).decode()
+
+print(f"glsa_{secret}_{checksum}")
+print(hashlib.pbkdf2_hmac("sha256", secret.encode(), checksum.encode(), 10000, 50).hex())
+print("".join(secrets.choice(alnum) for _ in range(10)))
+print("".join(secrets.choice(alnum) for _ in range(10)))
+print("".join(secrets.choice(string.ascii_lowercase + string.digits) for _ in range(14)))
+PY
+}
+
+# Administers Grafana through the database it owns, the way postgres-sep provisions the
+# sep role as the local superuser: Grafana exposes no credential-free path to create a
+# service account, and once an operator changes the admin password pmm-server holds none.
+#
+# psql only interpolates :'var' when reading a script, hence stdin rather than -c. The
+# ON CONFLICT clauses target UQE_user_login and UQE_org_user_org_id_user_id, and DO
+# UPDATE rather than DO NOTHING is what makes RETURNING fire on re-provisioning.
+provision_service_account() {
+    local hashed="$1" salt="$2" rands="$3" uid="$4"
+
+    psql_grafana -v sa_name="$SA_NAME" -v token_name="$TOKEN_NAME" -v hashed="$hashed" \
+        -v salt="$salt" -v rands="$rands" -v uid="$uid" <<'SQL'
+BEGIN;
+
+SELECT id AS org_id FROM org ORDER BY id LIMIT 1
+\gset
+
+INSERT INTO "user" (version, login, email, name, password, salt, rands, company, org_id,
+                    is_admin, email_verified, theme, created, updated, help_flags1,
+                    is_disabled, is_service_account, uid, is_provisioned)
+VALUES (0, 'sa-' || :'org_id' || '-' || :'sa_name', 'sa-' || :'org_id' || '-' || :'sa_name',
+        :'sa_name', '', :'salt', :'rands', '', :'org_id',
+        false, false, '', now(), now(), 0, false, true, :'uid', false)
+ON CONFLICT (login) DO UPDATE SET updated = now(), is_disabled = false
+RETURNING id AS sa_id
+\gset
+
+INSERT INTO org_user (org_id, user_id, role, created, updated)
+VALUES (:'org_id', :'sa_id', 'Admin', now(), now())
+ON CONFLICT (org_id, user_id) DO UPDATE SET role = 'Admin', updated = now();
+
+-- Every token on the account, plus any token elsewhere in the org already holding the
+-- name, which would otherwise collide with UQE_api_key_org_id_name below.
+DELETE FROM api_key
+WHERE service_account_id = :'sa_id' OR (org_id = :'org_id' AND name = :'token_name');
+
+-- Grafana stores Viewer in api_key.role for every service-account token whatever role
+-- the request asked for; the effective authority is org_user.role above.
+INSERT INTO api_key (org_id, name, key, role, created, updated, service_account_id, is_revoked)
+VALUES (:'org_id', :'token_name', :'hashed', 'Viewer', now(), now(), :'sa_id', false);
+
+COMMIT;
+SQL
+}
+
+# Written through a temporary file so SEP never reads a half-written secret.
+publish_token() {
+    local token="$1" name
+
+    for name in "${TOKEN_FILES[@]}"; do
+        TMP_FILE=$(mktemp "$SECRETS_DIR/.${name}.XXXXXX")
+        printf '%s' "$token" > "$TMP_FILE"
+        chmod 0640 "$TMP_FILE"
+        mv "$TMP_FILE" "$SECRETS_DIR/$name"
+        TMP_FILE=""
+    done
+}
+
+# Never fatal, so the disabled path cannot fail a start over a directory this uid cannot
+# write - but a token left behind is still readable by SEP, so it is reported.
+remove_token_files() {
+    local name path
+
+    for name in "${TOKEN_FILES[@]}"; do
+        path="${SECRETS_DIR:?}/$name"
+        rm -f "$path" 2> /dev/null || true
+        if [ -e "$path" ]; then
+            echo "WARNING: could not remove $path, SEP can still read this token." >&2
+        fi
+    done
+}
+
+if ! is_enabled "$PMM_ENABLE_SEP"; then
+    MARK_RUN=0
+    remove_token_files
+    exit 0
+fi
+
+if is_enabled "$PMM_HA_ENABLE" || is_enabled "$PMM_DISABLE_BUILTIN_POSTGRES" ||
+    [ -n "$GF_DATABASE_URL" ] || [ -n "$GF_DATABASE_HOST" ]; then
+    echo "WARNING: ignoring PMM_ENABLE_SEP, Grafana is not using the embedded PostgreSQL." >&2
+    MARK_RUN=0
+    remove_token_files
+    exit 0
+fi
+
+# Created only when absent: install -d on an existing directory also chmods it, which a
+# uid that does not own the mountpoint cannot do.
+if [ ! -d "$SECRETS_DIR" ]; then
+    install -d -m 2770 "$SECRETS_DIR" 2> /dev/null || true
+fi
+
+if [ ! -w "$SECRETS_DIR" ]; then
+    echo "FATAL: $SECRETS_DIR is not writable for uid $(id -u)." >&2
+    echo "Please make sure the volume mounted there is owned by uid $(id -u) and gid $(id -g) and try again." >&2
+    exit 1
+fi
+
+if ! wait_for_grafana; then
+    echo "FATAL: Grafana did not become ready within ${READY_TIMEOUT}s, cannot provision the SEP service account." >&2
+    exit 1
+fi
+
+declare TOKEN=""
+if [ -s "$SECRETS_DIR/${TOKEN_FILES[0]}" ]; then
+    TOKEN=$(< "$SECRETS_DIR/${TOKEN_FILES[0]}")
+fi
+
+declare STATUS=""
+if [ -n "$TOKEN" ]; then
+    STATUS=$(grafana_auth_status "$TOKEN")
+fi
+
+case "$STATUS" in
+    200)
+        echo "The Grafana service account token for SEP is still valid, keeping it."
+        # Ordered so a token already proven valid is on disk even if the role repair below
+        # fails under errexit.
+        publish_token "$TOKEN"
+        ensure_admin_role
+        exit 0
+        ;;
+    "" | 401 | 403)
+        # No token yet, or Grafana genuinely rejected it: fall through and mint.
+        ;;
+    *)
+        echo "FATAL: could not verify the existing Grafana token for SEP, the check answered $STATUS." >&2
+        echo "Please retry once Grafana answers again; replacing the token now would revoke the one SEP is using." >&2
+        exit 1
+        ;;
+esac
+
+echo "Provisioning the Grafana service account for SEP..."
+
+declare MATERIAL
+MATERIAL=$(generate_token_material)
+
+declare -a PARTS
+mapfile -t PARTS <<< "$MATERIAL"
+if [ ${#PARTS[@]} -ne 5 ]; then
+    echo "FATAL: could not generate the Grafana service account token." >&2
+    exit 1
+fi
+
+TOKEN="${PARTS[0]}"
+provision_service_account "${PARTS[1]}" "${PARTS[2]}" "${PARTS[3]}" "${PARTS[4]}"
+
+# Validated before publishing so a half-working credential never reaches SEP, and so a
+# future change to Grafana's schema or token encoding fails loudly here instead of
+# silently breaking SEP's authentication.
+if [ "$(grafana_auth_status "$TOKEN")" != "200" ]; then
+    echo "FATAL: Grafana rejected the service account token just provisioned for SEP." >&2
+    exit 1
+fi
+
+publish_token "$TOKEN"
+echo "Published the Grafana service account token for SEP to $SECRETS_DIR."