PMM-15205 Ship SEP embedded in PMM (Tech Preview)

.devcontainer/setup.sh

@@ -56,6 +56,8 @@ install_node() {
     # corepack's own package ships yarn/yarnpkg shims too; --force lets the real yarn
     # binary below win over those.
     npm install -g yarn@1.22.22 --force
+    # pnpm is not pinned here: corepack resolves it from the `packageManager`
+    # field in ui/package.json on first use.
     corepack enable pnpm
     node --version
     yarn --version

.env.example

@@ -49,9 +49,50 @@ PMM_PORT_HTTPS=443
 # Expose the built-in PostgreSQL to SEP running in a side container on the same bridge
 # network. PMM creates a dedicated `sep` database owned by a non-superuser `sep` role and
 # accepts connections for it from the container's Docker subnets only; nothing is
-# published on the host. Both variables are required to enable it.
+# published on the host. PMM_ENABLE_SEP alone is enough: PMM generates the role's password
+# on first start and persists it at /srv/.sep_postgres_password mode 0600, so a plain
+# `docker compose up` needs no secret chosen in advance. Set PMM_SEP_POSTGRES_PASSWORD to
+# supply your own instead; it is used verbatim and is not persisted, so clearing it later
+# returns to the generated one.
+#
+# The same flag makes nginx reverse-proxy /sep/ to the side-car. PMM_SEP_ADDRESS
+# overrides where it is reached and defaults to sep:9000, so it only needs setting
+# when the side-car's service name or port differs.
 # PMM_ENABLE_SEP=1
 # PMM_SEP_POSTGRES_PASSWORD=<password>
+# PMM_SEP_ADDRESS=sep:9000
+
+# With PMM_ENABLE_SEP set, PMM writes SEP's deployment secrets to /srv/sep - the pmm-sep
+# volume, which the SEP container mounts read-only with SECRETS_DIR pointing at it.
+# The directory is setgid mode 2770 and the files 0640, so every file belongs to group 0
+# whichever uid PMM runs as. PMM cannot chgrp to SEP's own group, so the SEP container
+# has to join group 0 - `group_add: ["0"]`, or `user: "1001:0"` - or it cannot read them.
+# One file per canonical SEP settings name:
+#   SECRET_KEY                      generated once on first start and persisted on the
+#                                   /srv volume, so restarting PMM keeps every SEP
+#                                   session valid
+#   SEP__DATABASE__PASSWORD         the role's password, generated or supplied, rewritten
+#   INVENTORY__DATABASE__PASSWORD   on every start so rotating it takes effect on the next
+#   TASKS__DATABASE__PASSWORD       restart
+# SECRET_KEY is persisted outside the secrets directory, at /srv/.sep_secret_key mode 0600,
+# and copied from there on every start. A fresh /srv volume mints a new one; an existing one
+# is never overwritten, so back /srv up to keep SEP's sessions across a redeployment.
+# Do not pass SECRET_KEY to the SEP container — an environment value outranks the file, and
+# a PMM that later mints its own key then diverges from it silently. Leave the three canonical
+# *__DATABASE__PASSWORD names unset on the SEP container rather than empty: an empty value
+# counts as supplied there and outranks the file, which then goes unused.
+# Give PMM_SEP_POSTGRES_PASSWORD no leading or trailing whitespace: PostgreSQL keeps it, SEP
+# strips it when reading the file, and the two then disagree.
+# Rotating PMM_SEP_POSTGRES_PASSWORD takes two restarts: restart PMM to move the database and
+# rewrite the files, then restart the SEP container, which reads them only at process start.
+# Once SEP is actually in use, PMM Server stops rather than degrading SEP alone: a /srv/sep
+# it cannot write to is fatal, and so is one that is not setgid group 0 with group r-x,
+# since the files would otherwise be published in a group SEP cannot read, or behind a
+# directory SEP cannot search. Neither check applies under
+# PMM_HA_ENABLE or PMM_DISABLE_BUILTIN_POSTGRES, which ignore PMM_ENABLE_SEP outright and
+# leave no SEP database to hold a password for - so nothing is generated or written at all.
+# Unsetting PMM_ENABLE_SEP removes all four files on the next start; the persisted
+# SECRET_KEY is left in place, so re-enabling SEP keeps existing sessions valid.
 
 # Use SSL certificates for PMM Server's internal database connection (PostgreSQL)
 # GF_DATABASE_SSL_MODE=verify-full

build/AGENTS.md

@@ -37,6 +37,7 @@ Source code (Go, TypeScript)
 | `grafana` | Install Grafana, provision datasources and dashboards |
 | `nginx` | Configure Nginx as reverse proxy (SSL termination, routing) |
 | `postgres` | Install and configure PostgreSQL for pmm-managed |
+| `sep` | SEP side-car integration helpers copied into the image and invoked from the entrypoint (no `tasks/`); publishes SEP's secrets when `PMM_ENABLE_SEP` is set |
 | `supervisord` | Configure Supervisord for process management |
 | `dashboards` | Provision PMM Grafana dashboards |
 | `initialization` | PMM Server first-run setup |

build/ansible/pmm-docker/post-build.yml

@@ -82,3 +82,17 @@
         owner: pmm
         group: root
         mode: 0664
+
+    # Must exist in the image: a named volume mounted at a path the image lacks is
+    # created root:root, which pmm-server cannot write to. Group-writable like the
+    # other /srv directories, so an arbitrary runtime uid in group 0 can still write.
+    # Setgid so the files written here belong to group 0 whatever that uid's primary
+    # group is - the SEP container joins group 0 to read them. Docker carries the bit
+    # into a fresh named volume mounted at this path.
+    - name: Create the SEP directory
+      file:
+        path: /srv/sep
+        state: directory
+        owner: pmm
+        group: root
+        mode: 02770

build/ansible/roles/nginx/files/conf.d/pmm.conf

@@ -143,6 +143,10 @@
       return 403 '{"code":7,"error":"Access denied","message":"Access denied"}';
     }
 
+    # SEP side-car locations, installed by the entrypoint when PMM_ENABLE_SEP is
+    # enabled. The glob is a no-op when the directory is empty or absent.
+    include /etc/nginx/sep.d/*.conf;
+
     # PMM UI
     location /pmm-ui {
       # Will redirect on FE to login page if user is not authenticated

build/ansible/roles/nginx/files/sep/sep.conf.template

@@ -0,0 +1,57 @@
+# Reverse-proxy locations for the SEP side-car. Installed into
+# /etc/nginx/sep.d/ by build/docker/server/entrypoint.sh when PMM_ENABLE_SEP is
+# enabled; removed on the next start when it is not.
+
+location @sep_unavailable {
+  auth_request off;
+  default_type application/json;
+
+  # An explicit body bypasses the server-level error_page 503 @maintenance,
+  # which would otherwise turn this into PMM's maintenance page.
+  return 503 '{"code":14,"error":"SEP unavailable","message":"SEP unavailable"}';
+}
+
+location /sep/ {
+  # SEP authenticates its own bearer; pmm-managed knows nothing about it.
+  auth_request off;
+
+  # The http-level resolver is public and cannot answer for a container name.
+  # The address is read from /etc/resolv.conf at start so this works under both
+  # Docker and Podman. It stays inside this location on purpose: this file is
+  # included in the server block, so at file scope it would also redirect DNS
+  # for every other request-time lookup, /percona-blog/feed among them.
+  # valid= caps how long a restarted side-car stays unreachable.
+  resolver __SEP_RESOLVER__ valid=10s;
+  resolver_timeout 5s;
+
+  # A variable defers resolution to request time, so this config loads with the
+  # side-car absent. It must carry no URI component: a trailing / would replace
+  # the request URI rather than strip the prefix.
+  set $sep_upstream "__SEP_ADDRESS__";
+  proxy_pass http://$sep_upstream;
+
+  # Covers nginx-generated DNS and connect failures only; proxy_intercept_errors
+  # is off, so a 502 returned by SEP itself passes through unchanged. Like
+  # proxy_set_header below, this is not additive -- it drops the inherited
+  # 401/403/503 handlers, which is only safe while auth_request is off.
+  error_page 502 504 = @sep_unavailable;
+
+  # proxy_set_header is not additive: declaring any header here discards the
+  # server-level X-Forwarded-For, so it is re-declared.
+  # Host drives SEP's url_for output; without it SEP emits upstream-addressed URLs.
+  proxy_set_header Host $http_host;
+  proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+  proxy_set_header X-Forwarded-Proto $scheme;
+
+  # Buffering stays on. SEP marks its own streaming responses with
+  # X-Accel-Buffering: no, which nginx honours per response -- keeping stream
+  # knowledge in SEP instead of encoding routes here. SEP applies that header
+  # from one frozen constant shared by every streaming route, so a new stream
+  # cannot reach this proxy without it.
+  proxy_http_version 1.1;
+  proxy_set_header Upgrade $http_upgrade;
+  proxy_set_header Connection $connection_upgrade;
+  proxy_read_timeout 3600;
+
+  client_max_body_size 100m;
+}

build/ansible/roles/nginx/tasks/main.yml

@@ -14,6 +14,7 @@
     - /srv/nginx/tmp/uwsgi/
     - /srv/nginx/tmp/scgi/
     - /etc/nginx/conf.d/
+    - /etc/nginx/sep.d/
     - /etc/nginx/ssl/
 
 - name: Enable nginx 1.26 module stream

build/ansible/roles/postgres/files/postgres-sep

@@ -97,7 +97,7 @@ fi
 
 if [ -z "$PMM_SEP_POSTGRES_PASSWORD" ]; then
     echo "FATAL: PMM_ENABLE_SEP is set but PMM_SEP_POSTGRES_PASSWORD is empty." >&2
-    echo "Please set PMM_SEP_POSTGRES_PASSWORD to the password SEP will connect with and try again." >&2
+    echo "The entrypoint generates one when it is unset, so reaching here means /srv could not be written; check its ownership and try again." >&2
     exit 1
 fi
 

build/ansible/roles/sep/files/sep-secrets

@@ -0,0 +1,215 @@
+#!/bin/bash
+#
+# Publishes SEP's SECRET_KEY and database credentials to a secrets directory the SEP
+# container mounts, one file per canonical SEP settings name. The files are 0640 and
+# take group 0 from the setgid directory post-build.yml ships - nothing here sets their
+# group, so that bit must stay.
+#
+# The two lifecycles differ deliberately. SECRET_KEY is generated once and persisted on
+# /srv: re-minting it signs out every SEP session and changes the SEP_INTERNAL_TOKEN
+# derived from it by HMAC. The database credential is rewritten on every start so
+# rotating PMM_SEP_POSTGRES_PASSWORD takes effect on the next restart.
+#
+# Everything below is a no-op unless PMM_ENABLE_SEP is set. When it is unset again, the
+# files this script wrote are removed on the next start; the persisted key is left in
+# place so re-enabling SEP does not invalidate the sessions of a deployment that toggled
+# the flag.
+#
+# Runs from the entrypoint rather than under supervisord because neither value depends on
+# a running service: the key is generated here and the password comes from the environment.
+
+set -o errexit
+set -o pipefail
+
+declare SECRETS_DIR="${PMM_DEV_SEP_SECRETS_DIR:-/srv/sep}"
+declare SECRET_KEY_FILE="/srv/.sep_secret_key"
+declare -a DB_PASSWORD_FILES=(
+    SEP__DATABASE__PASSWORD
+    INVENTORY__DATABASE__PASSWORD
+    TASKS__DATABASE__PASSWORD
+)
+declare -a MANAGED_FILES=(SECRET_KEY "${DB_PASSWORD_FILES[@]}")
+declare -a STAGED_TMP=() STAGED_DEST=()
+declare TMP_FILE=""
+
+cleanup() {
+    local tmp
+
+    [ -n "$TMP_FILE" ] && rm -f "$TMP_FILE"
+    for tmp in "${STAGED_TMP[@]}"; do
+        rm -f "$tmp"
+    done
+    return 0
+}
+trap cleanup EXIT
+
+is_enabled() { [ "$1" = "1" ] || [ "$1" = "true" ]; }
+
+# Every fallible step - mktemp, the write, the chmod - runs here, before commit_staged
+# renames anything, so the failure that actually happens to this operation (a write dying
+# on a full volume) leaves SEP the previous complete set rather than a mix of new and
+# stale credentials. Interleaving write and rename per file is what makes that mix
+# reachable; the rename itself, within one filesystem and after a complete write, is not
+# the fallible part.
+#
+# printf '%s' so no trailing newline reaches a reader that strips rather than chomps.
+stage() {
+    local name="$1" value="$2" tmp
+
+    tmp=$(mktemp "$SECRETS_DIR/.${name}.XXXXXX")
+    # Recorded before the write, not after: errexit abandons the rest of this function on
+    # a failed write, and a temporary created but not yet recorded is one cleanup cannot
+    # find.
+    STAGED_TMP+=("$tmp")
+    STAGED_DEST+=("$SECRETS_DIR/$name")
+    printf '%s' "$value" > "$tmp"
+    chmod 0640 "$tmp"
+}
+
+# Each rename replaces a whole file with one already written in full, so SEP never reads a
+# half-written secret. The set is still four renames rather than one operation: a failure
+# between them leaves the earlier files updated, which is the residue this cannot remove
+# without a directory swap the mountpoint contract rules out.
+commit_staged() {
+    local i
+
+    for i in "${!STAGED_TMP[@]}"; do
+        mv "${STAGED_TMP[$i]}" "${STAGED_DEST[$i]}"
+    done
+    STAGED_TMP=()
+    STAGED_DEST=()
+}
+
+# Never fatal, so the disabled path cannot fail a start over a directory this uid cannot
+# write - but a secret left behind is still readable by SEP, so it is reported.
+remove_managed_files() {
+    local name path
+
+    for name in "${MANAGED_FILES[@]}"; do
+        path="${SECRETS_DIR:?}/$name"
+        # The staging temporaries go too. cleanup removes them on any exit bash runs a trap
+        # for, so what survives to reach here is the SIGKILL and grace-period-expiry
+        # residue - a readable .$name.XXXXXX the canonical names alone never collect.
+        rm -f "$path" "${SECRETS_DIR:?}/.$name".?????? 2> /dev/null || true
+        if [ -e "$path" ]; then
+            echo "WARNING: could not remove $path, SEP can still read this secret." >&2
+        fi
+    done
+}
+
+# Non-empty rather than merely present: SEP's settings classes reject a blank SECRET_KEY,
+# and the side-car's own gate treats a mounted-but-empty key file as a failure rather than
+# as a supplied value.
+#
+# Generated into a temporary file and renamed, never written at the final path: a direct
+# redirect that dies mid-write leaves a short but non-empty key there, which the -s test
+# would then accept and republish forever. errexit aborts before the rename, so an
+# interrupted generation leaves the previous key - or no key - untouched.
+ensure_secret_key() {
+    local mode
+
+    if [ ! -s "$SECRET_KEY_FILE" ]; then
+        echo "Generating SEP's SECRET_KEY..."
+        TMP_FILE=$(mktemp "$SECRET_KEY_FILE.XXXXXX")
+        openssl rand -hex 32 > "$TMP_FILE"
+        mv "$TMP_FILE" "$SECRET_KEY_FILE"
+        TMP_FILE=""
+    fi
+    # Unconditional so a key whose mode was widened by hand is narrowed back without being
+    # replaced. A failed chmod is not by itself fatal: it needs ownership rather than write
+    # permission, and the arbitrary-uid path lets a later start run as a uid that does not
+    # own what an earlier one persisted. A mode that still leaves the key readable beyond
+    # its owner is fatal, because this is the value that signs every SEP session - the same
+    # rule entrypoint.sh applies to the password it persists next to it.
+    if ! chmod 600 "$SECRET_KEY_FILE" 2> /dev/null; then
+        mode=$(stat -c '%a' "$SECRET_KEY_FILE")
+        if [ $((8#$mode & 8#077)) -ne 0 ]; then
+            echo "FATAL: $SECRET_KEY_FILE is mode $mode and could not be narrowed to 600." >&2
+            echo "Please make sure it is owned by uid $(id -u), or narrow it by hand, and try again." >&2
+            exit 1
+        fi
+    fi
+}
+
+if ! is_enabled "$PMM_ENABLE_SEP"; then
+    remove_managed_files
+    exit 0
+fi
+
+# Deliberately not bailing on GF_DATABASE_URL/GF_DATABASE_HOST: nothing here touches
+# Grafana, and postgres-sep still provisions the sep role and database with an external
+# Grafana, so SEP still needs its password files.
+#
+# The entrypoint has already warned about the combination below; bail quietly rather than
+# print a second copy of the same line.
+if is_enabled "$PMM_HA_ENABLE" || is_enabled "$PMM_DISABLE_BUILTIN_POSTGRES"; then
+    remove_managed_files
+    exit 0
+fi
+
+if [ -z "$PMM_SEP_POSTGRES_PASSWORD" ]; then
+    echo "FATAL: PMM_ENABLE_SEP is set but PMM_SEP_POSTGRES_PASSWORD is empty." >&2
+    echo "The entrypoint generates one when it is unset, so reaching here means /srv could not be written; check its ownership and try again." >&2
+    exit 1
+fi
+
+# Created only when absent: install -d on an existing directory also chmods it, which a
+# uid that does not own the mountpoint cannot do. -g 0 rather than the creator's egid,
+# because the group is what SEP reads these files through; the fallback covers a uid that
+# is not a member of group 0, which the inheritance check below then rejects.
+if [ ! -d "$SECRETS_DIR" ]; then
+    install -d -m 2770 -g 0 "$SECRETS_DIR" 2> /dev/null ||
+        install -d -m 2770 "$SECRETS_DIR" 2> /dev/null || true
+fi
+
+if [ ! -w "$SECRETS_DIR" ]; then
+    echo "FATAL: $SECRETS_DIR is not writable for uid $(id -u)." >&2
+    echo "Please make sure the volume mounted there is owned by uid $(id -u) and gid $(id -g) and try again." >&2
+    exit 1
+fi
+
+# -w answers whether this uid can create files here, which is not the question SEP cares
+# about: it reads the 0640 files through group 0, and nothing below sets a group, so the
+# files take whatever the directory hands them. Handing them the group is half of it -
+# group 0 needs r-x on the directory too, since a 0640 file behind a directory group 0
+# cannot search is as unreachable as one in the wrong group. A directory failing either
+# half publishes a complete, correct set of secrets SEP cannot open - and the failure
+# surfaces on the SEP side as a missing setting rather than here. Group write is not in
+# the mask: only this uid writes here, which -w already answered.
+declare SECRETS_DIR_GID SECRETS_DIR_MODE
+SECRETS_DIR_GID=$(stat -c '%g' "$SECRETS_DIR")
+SECRETS_DIR_MODE=$(stat -c '%a' "$SECRETS_DIR")
+if [ "$SECRETS_DIR_GID" -ne 0 ] || [ $((8#$SECRETS_DIR_MODE & 8#2050)) -ne $((8#2050)) ]; then
+    echo "FATAL: $SECRETS_DIR is gid $SECRETS_DIR_GID mode $SECRETS_DIR_MODE, not setgid group 0 with group r-x." >&2
+    echo "Please run 'chgrp 0 $SECRETS_DIR && chmod g+rwxs $SECRETS_DIR' on the volume mounted there and try again." >&2
+    exit 1
+fi
+
+ensure_secret_key
+
+# Read into a variable rather than inline at the call below: a command substitution that
+# fails in argument position does not trip errexit, so an unreadable key would publish an
+# empty SECRET_KEY over a good one and fail SEP's own gate instead of stopping here.
+# Regenerating instead would be worse - the file is non-empty, so something is using it.
+declare SECRET_KEY_VALUE=""
+if [ -r "$SECRET_KEY_FILE" ]; then
+    SECRET_KEY_VALUE=$(< "$SECRET_KEY_FILE")
+fi
+
+if [ -z "$SECRET_KEY_VALUE" ]; then
+    echo "FATAL: SEP's SECRET_KEY persisted at $SECRET_KEY_FILE is unreadable or blank." >&2
+    echo "Please make sure it is readable for uid $(id -u), or remove it to generate a new one - which signs every SEP session out." >&2
+    exit 1
+fi
+
+# Republished on every start, not only on generation, so a wiped or newly attached secrets
+# volume is refilled from the persisted key rather than left short a file.
+stage SECRET_KEY "$SECRET_KEY_VALUE"
+
+for name in "${DB_PASSWORD_FILES[@]}"; do
+    stage "$name" "$PMM_SEP_POSTGRES_PASSWORD"
+done
+
+commit_staged
+
+echo "Published SEP's SECRET_KEY and database credentials to $SECRETS_DIR."