PMM-15316 Stage SEP secret files before renaming

publish() interleaved the write and the rename per file, so a write that
died part-way through the set - a full volume being the realistic cause -
left SEP some files rotated and the rest stale. Verified against the
previous revision: a failure on the third password file rotated two of
them and left the third behind.

Split it into stage() and commit_staged(). Every fallible step now runs
before the first rename, so the same failure leaves the previous complete
set in place and the script still exits nonzero. The temporary is recorded
for cleanup immediately after mktemp rather than after the write, since
errexit abandons the rest of stage() on a failed write and an unrecorded
temporary is one the EXIT trap cannot find.

This narrows rather than closes the gap: the set is still four renames, so
a failure between them leaves the earlier files updated. Closing that needs
a directory swap the mountpoint contract rules out.

Signed-off-by: Yan Orestes <yan.orestes@percona.com>

Author: yyyyyyyan

build/ansible/roles/sep/files/sep-secrets

@@ -29,26 +29,55 @@ declare -a DB_PASSWORD_FILES=(
     TASKS__DATABASE__PASSWORD
 )
 declare -a MANAGED_FILES=(SECRET_KEY "${DB_PASSWORD_FILES[@]}")
+declare -a STAGED_TMP=() STAGED_DEST=()
 declare TMP_FILE=""
 
 cleanup() {
+    local tmp
+
     [ -n "$TMP_FILE" ] && rm -f "$TMP_FILE"
+    for tmp in "${STAGED_TMP[@]}"; do
+        rm -f "$tmp"
+    done
     return 0
 }
 trap cleanup EXIT
 
 is_enabled() { [ "$1" = "1" ] || [ "$1" = "true" ]; }
 
-# Written through a temporary file so SEP never reads a half-written secret, and with
+# Every fallible step - mktemp, the write, the chmod - runs here, before commit_staged
+# renames anything, so the failure that actually happens to this operation (a write dying
+# on a full volume) leaves SEP the previous complete set rather than a mix of new and
+# stale credentials. Interleaving write and rename per file is what makes that mix
+# reachable; the rename itself, within one filesystem and after a complete write, is not
+# the fallible part.
+#
 # printf '%s' so no trailing newline reaches a reader that strips rather than chomps.
-publish() {
-    local name="$1" value="$2"
-
-    TMP_FILE=$(mktemp "$SECRETS_DIR/.${name}.XXXXXX")
-    printf '%s' "$value" > "$TMP_FILE"
-    chmod 0640 "$TMP_FILE"
-    mv "$TMP_FILE" "$SECRETS_DIR/$name"
-    TMP_FILE=""
+stage() {
+    local name="$1" value="$2" tmp
+
+    tmp=$(mktemp "$SECRETS_DIR/.${name}.XXXXXX")
+    # Recorded before the write, not after: errexit abandons the rest of this function on
+    # a failed write, and a temporary created but not yet recorded is one cleanup cannot
+    # find.
+    STAGED_TMP+=("$tmp")
+    STAGED_DEST+=("$SECRETS_DIR/$name")
+    printf '%s' "$value" > "$tmp"
+    chmod 0640 "$tmp"
+}
+
+# Each rename replaces a whole file with one already written in full, so SEP never reads a
+# half-written secret. The set is still four renames rather than one operation: a failure
+# between them leaves the earlier files updated, which is the residue this cannot remove
+# without a directory swap the mountpoint contract rules out.
+commit_staged() {
+    local i
+
+    for i in "${!STAGED_TMP[@]}"; do
+        mv "${STAGED_TMP[$i]}" "${STAGED_DEST[$i]}"
+    done
+    STAGED_TMP=()
+    STAGED_DEST=()
 }
 
 # Never fatal, so the disabled path cannot fail a start over a directory this uid cannot
@@ -143,10 +172,12 @@ fi
 
 # Republished on every start, not only on generation, so a wiped or newly attached secrets
 # volume is refilled from the persisted key rather than left short a file.
-publish SECRET_KEY "$SECRET_KEY_VALUE"
+stage SECRET_KEY "$SECRET_KEY_VALUE"
 
 for name in "${DB_PASSWORD_FILES[@]}"; do
-    publish "$name" "$PMM_SEP_POSTGRES_PASSWORD"
+    stage "$name" "$PMM_SEP_POSTGRES_PASSWORD"
 done
 
+commit_staged
+
 echo "Published SEP's SECRET_KEY and database credentials to $SECRETS_DIR."