Add runtime validation packs

Author: marcofortina

README.md

@@ -190,7 +190,7 @@ Collect runtime evidence from a real SSH lab host:
 ```bash
 AUTOMAX_SSH_HOST=web01.example.com \
 AUTOMAX_SSH_USER=deploy \
-scripts/runtime-evidence.sh
+scripts/runtime-evidence.sh and scripts/runtime-validation-packs.sh
 ```
 
 Task-level and step-level restart points are also accepted:

docs/guides/runtime-validation-packs.md

@@ -0,0 +1,70 @@
+<!--
+Copyright (C) 2026 Marco Fortina
+SPDX-License-Identifier: AGPL-3.0-or-later
+-->
+
+# Runtime validation packs
+
+Runtime validation packs group release checks, package smoke, per-plugin check
+runbooks and SSH evidence collection into named validation profiles.
+
+List available packs:
+
+```bash
+scripts/runtime-validation-packs.sh --list
+```
+
+Run the package-only pack:
+
+```bash
+scripts/runtime-validation-packs.sh --pack package-only
+```
+
+A run writes logs, a manifest, a summary and checksums under
+`dist/runtime-packs/<pack>-<timestamp>/`.
+
+## Included packs
+
+`package-only` runs release metadata checks and package smoke. It is suitable for
+local release rehearsals where no disposable runtime host is available.
+
+`debian-like` and `redhat-like` are disposable VM profiles. They run release
+metadata, package smoke, per-plugin check runbooks and SSH evidence collection.
+Provide the normal SSH smoke variables before running them:
+
+```bash
+AUTOMAX_SSH_HOST=192.0.2.10
+AUTOMAX_SSH_USER=deploy
+AUTOMAX_SSH_KEY_FILE=~/.ssh/id_ed25519
+AUTOMAX_SSH_KNOWN_HOSTS=~/.ssh/known_hosts
+```
+
+`privileged-vm` is intended for disposable hosts where destructive or privileged
+checks are allowed by the operator. Keep it isolated from long-lived systems.
+
+## Plugin check requirements
+
+The `plugin-checks` step uses `examples/runbooks/scripts/run-all-checks.sh` and
+therefore requires a sudo password environment variable:
+
+```bash
+export AUTOMAX_SUDO_PASSWORD='...'
+```
+
+Use `AUTOMAX_SUDO_PASSWORD_ENV` when the password lives under a different
+environment variable name.
+
+## Evidence output
+
+Each pack creates:
+
+```text
+summary.txt
+manifest.json
+SHA256SUMS
+logs/<step>.log
+```
+
+Steps that require unavailable runtime inputs return `skipped` with `rc=77` in
+the manifest. Failed steps stop the pack by default. Use `--keep-going` to collect
+all step outcomes before returning a failing summary.

docs/reference/release.md

@@ -75,6 +75,7 @@ Keep the following with the release record:
 - `scripts/release-check.sh` output;
 - runtime evidence from disposable hosts when the release changes runtime
   behavior;
+- runtime validation pack output from `scripts/runtime-validation-packs.sh`;
 - `automax plugins coverage --strict` output for builtin and release-loaded
   plugins;
 - package smoke output for the built sdist and wheel.

examples/runtime-packs/debian-like.env

@@ -0,0 +1,6 @@
+# Copyright (C) 2026 Marco Fortina
+# SPDX-License-Identifier: AGPL-3.0-or-later
+
+AUTOMAX_RUNTIME_PACK_DESCRIPTION="Debian-like disposable VM validation pack"
+AUTOMAX_RUNTIME_PACK_STEPS="release-metadata package-smoke plugin-checks ssh-evidence"
+AUTOMAX_RUNTIME_OS_FAMILY="debian-like"

examples/runtime-packs/package-only.env

@@ -0,0 +1,5 @@
+# Copyright (C) 2026 Marco Fortina
+# SPDX-License-Identifier: AGPL-3.0-or-later
+
+AUTOMAX_RUNTIME_PACK_DESCRIPTION="Release metadata and package smoke validation"
+AUTOMAX_RUNTIME_PACK_STEPS="release-metadata package-smoke"

examples/runtime-packs/privileged-vm.env

@@ -0,0 +1,6 @@
+# Copyright (C) 2026 Marco Fortina
+# SPDX-License-Identifier: AGPL-3.0-or-later
+
+AUTOMAX_RUNTIME_PACK_DESCRIPTION="Privileged disposable VM validation pack"
+AUTOMAX_RUNTIME_PACK_STEPS="release-metadata package-smoke plugin-checks ssh-evidence"
+AUTOMAX_RUNTIME_PRIVILEGED="1"

examples/runtime-packs/redhat-like.env

@@ -0,0 +1,6 @@
+# Copyright (C) 2026 Marco Fortina
+# SPDX-License-Identifier: AGPL-3.0-or-later
+
+AUTOMAX_RUNTIME_PACK_DESCRIPTION="Red Hat-like disposable VM validation pack"
+AUTOMAX_RUNTIME_PACK_STEPS="release-metadata package-smoke plugin-checks ssh-evidence"
+AUTOMAX_RUNTIME_OS_FAMILY="redhat-like"

mkdocs.yml

@@ -69,6 +69,7 @@ nav:
     - Creating Plugins: guides/creating-plugins.md
     - Extended SSH Smoke: guides/ssh-smoke.md
     - Runtime Evidence Suite: guides/runtime-evidence.md
+    - Runtime Validation Packs: guides/runtime-validation-packs.md
     - Publishing Documentation: guides/publishing-docs.md
   - Reference:
     - CLI: reference/cli.md

scripts/runtime-validation-packs.sh

@@ -0,0 +1,189 @@
+#!/usr/bin/env bash
+# Copyright (C) 2026 Marco Fortina
+# SPDX-License-Identifier: AGPL-3.0-or-later
+
+set -euo pipefail
+
+ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
+PACK_DIR="${AUTOMAX_RUNTIME_PACK_DIR:-${ROOT}/examples/runtime-packs}"
+EVIDENCE_ROOT="${AUTOMAX_RUNTIME_PACK_EVIDENCE_DIR:-${ROOT}/dist/runtime-packs}"
+PACK_NAME="${AUTOMAX_RUNTIME_PACK:-package-only}"
+KEEP_GOING=0
+LIST_ONLY=0
+SUDO_PASSWORD_ENV="${AUTOMAX_SUDO_PASSWORD_ENV:-AUTOMAX_SUDO_PASSWORD}"
+RUN_ID="$(date -u +%Y%m%dT%H%M%SZ)"
+
+usage() {
+  cat <<'USAGE'
+Usage: scripts/runtime-validation-packs.sh [--pack NAME] [--evidence-dir DIR] [--keep-going] [--list]
+
+Run a named runtime validation pack and collect step logs, a manifest, a summary
+and checksums under dist/runtime-packs by default.
+
+Options:
+  --pack NAME        Runtime pack name, defaults to package-only.
+  --evidence-dir DIR Output directory root, defaults to dist/runtime-packs.
+  --keep-going       Continue after failed steps and return a failing summary.
+  --list             List available packs and exit.
+USAGE
+}
+
+while [[ $# -gt 0 ]]; do
+  case "$1" in
+    --pack)
+      PACK_NAME="${2:-}"
+      [[ -n "$PACK_NAME" ]] || { echo "--pack requires a name" >&2; exit 2; }
+      shift 2
+      ;;
+    --evidence-dir)
+      EVIDENCE_ROOT="${2:-}"
+      [[ -n "$EVIDENCE_ROOT" ]] || { echo "--evidence-dir requires a path" >&2; exit 2; }
+      shift 2
+      ;;
+    --keep-going)
+      KEEP_GOING=1
+      shift
+      ;;
+    --list)
+      LIST_ONLY=1
+      shift
+      ;;
+    -h|--help)
+      usage
+      exit 0
+      ;;
+    *)
+      echo "Unknown option: $1" >&2
+      usage >&2
+      exit 2
+      ;;
+  esac
+done
+
+list_packs() {
+  find "$PACK_DIR" -maxdepth 1 -type f -name '*.env' -printf '%f\n' \
+    | sed 's/[.]env$//' \
+    | sort
+}
+
+if [[ "$LIST_ONLY" -eq 1 ]]; then
+  list_packs
+  exit 0
+fi
+
+PACK_FILE="${PACK_DIR}/${PACK_NAME}.env"
+if [[ ! -f "$PACK_FILE" ]]; then
+  echo "Runtime pack not found: $PACK_FILE" >&2
+  echo "Available packs:" >&2
+  list_packs >&2
+  exit 2
+fi
+
+# shellcheck source=/dev/null
+source "$PACK_FILE"
+
+PACK_STEPS="${AUTOMAX_RUNTIME_PACK_STEPS:-}"
+if [[ -z "$PACK_STEPS" ]]; then
+  echo "Runtime pack $PACK_NAME does not define AUTOMAX_RUNTIME_PACK_STEPS" >&2
+  exit 2
+fi
+
+EVIDENCE_DIR="${EVIDENCE_ROOT}/${PACK_NAME}-${RUN_ID}"
+LOG_DIR="${EVIDENCE_DIR}/logs"
+mkdir -p "$LOG_DIR"
+
+SUMMARY="${EVIDENCE_DIR}/summary.txt"
+MANIFEST="${EVIDENCE_DIR}/manifest.json"
+CHECKSUMS="${EVIDENCE_DIR}/SHA256SUMS"
+
+status_names=()
+status_states=()
+status_codes=()
+
+run_step() {
+  local step="$1"
+  local log="${LOG_DIR}/${step}.log"
+  local rc=0
+  echo "== $step ==" | tee "$log"
+  case "$step" in
+    release-metadata)
+      (cd "$ROOT" && AUTOMAX_RELEASE_CHECK_SKIP_DOCS=1 AUTOMAX_RELEASE_CHECK_SKIP_PACKAGE=1 scripts/release-check.sh) >>"$log" 2>&1 || rc=$?
+      ;;
+    package-smoke)
+      (cd "$ROOT" && scripts/package-smoke.sh) >>"$log" 2>&1 || rc=$?
+      ;;
+    plugin-checks)
+      if [[ -z "${!SUDO_PASSWORD_ENV:-}" ]]; then
+        echo "SKIP: set ${SUDO_PASSWORD_ENV} for plugin-checks" >>"$log"
+        rc=77
+      else
+        (cd "$ROOT" && examples/runbooks/scripts/run-all-checks.sh --keep-going) >>"$log" 2>&1 || rc=$?
+      fi
+      ;;
+    ssh-evidence)
+      (cd "$ROOT" && scripts/runtime-evidence.sh) >>"$log" 2>&1 || rc=$?
+      ;;
+    *)
+      echo "Unknown runtime validation step: $step" >>"$log"
+      rc=2
+      ;;
+  esac
+  local state="passed"
+  if [[ "$rc" -eq 77 ]]; then
+    state="skipped"
+  elif [[ "$rc" -ne 0 ]]; then
+    state="failed"
+  fi
+  status_names+=("$step")
+  status_states+=("$state")
+  status_codes+=("$rc")
+  echo "$state rc=$rc log=$log"
+  if [[ "$state" == "failed" && "$KEEP_GOING" -ne 1 ]]; then
+    return "$rc"
+  fi
+  return 0
+}
+
+final_rc=0
+for step in $PACK_STEPS; do
+  if ! run_step "$step"; then
+    final_rc=$?
+    break
+  fi
+  last_index=$((${#status_states[@]} - 1))
+  if [[ "${status_states[$last_index]}" == "failed" ]]; then
+    final_rc="${status_codes[$last_index]}"
+  fi
+done
+
+python - "$PACK_NAME" "$RUN_ID" "$EVIDENCE_DIR" "${status_names[*]}" "${status_states[*]}" "${status_codes[*]}" >"$MANIFEST" <<'PY'
+import json
+import sys
+
+pack, run_id, evidence_dir, names, states, codes = sys.argv[1:7]
+steps = []
+for name, state, code in zip(names.split(), states.split(), codes.split()):
+    steps.append({"name": name, "status": state, "rc": int(code), "log": f"logs/{name}.log"})
+print(json.dumps({"pack": pack, "run_id": run_id, "evidence_dir": evidence_dir, "steps": steps}, indent=2, sort_keys=True))
+PY
+
+{
+  echo "Automax runtime validation pack"
+  echo "pack=${PACK_NAME}"
+  echo "run_id=${RUN_ID}"
+  echo "evidence_dir=${EVIDENCE_DIR}"
+  echo
+  for i in "${!status_names[@]}"; do
+    printf '%s: %s rc=%s\n' "${status_names[$i]}" "${status_states[$i]}" "${status_codes[$i]}"
+  done
+} >"$SUMMARY"
+
+(
+  cd "$EVIDENCE_DIR"
+  find . -type f ! -name SHA256SUMS -print0 | sort -z | xargs -0 sha256sum >"$CHECKSUMS"
+)
+
+cat "$SUMMARY"
+echo "manifest=${MANIFEST}"
+echo "checksums=${CHECKSUMS}"
+exit "$final_rc"

tests/test_documentation_and_regressions.py

@@ -1807,6 +1807,45 @@ def test_ci_runs_package_smoke_and_checks_all_shell_smokes():
     assert "matrix.python-version == '3.12'" in ci
 
 
+def test_runtime_validation_pack_script_and_docs_are_in_sync():
+    script = Path("scripts/runtime-validation-packs.sh")
+    docs = Path("docs/guides/runtime-validation-packs.md")
+    mkdocs = Path("mkdocs.yml").read_text(encoding="utf-8")
+    readme = Path("README.md").read_text(encoding="utf-8")
+    release = Path("docs/reference/release.md").read_text(encoding="utf-8")
+    packs = sorted(path.stem for path in Path("examples/runtime-packs").glob("*.env"))
+
+    assert script.is_file()
+    assert os.access(script, os.X_OK)
+    assert docs.is_file()
+    assert packs == ["debian-like", "package-only", "privileged-vm", "redhat-like"]
+    assert "Runtime Validation Packs: guides/runtime-validation-packs.md" in mkdocs
+    assert "scripts/runtime-validation-packs.sh" in readme
+    assert "scripts/runtime-validation-packs.sh" in release
+    text = script.read_text(encoding="utf-8")
+    assert "release-metadata" in text
+    assert "package-smoke" in text
+    assert "plugin-checks" in text
+    assert "ssh-evidence" in text
+    assert "manifest.json" in text
+    assert "SHA256SUMS" in text
+
+
+def test_runtime_validation_pack_list_command_returns_pack_names():
+    result = subprocess.run(
+        ["bash", "scripts/runtime-validation-packs.sh", "--list"],
+        check=True,
+        capture_output=True,
+        text=True,
+    )
+    assert result.stdout.splitlines() == [
+        "debian-like",
+        "package-only",
+        "privileged-vm",
+        "redhat-like",
+    ]
+
+
 def test_runtime_evidence_suite_is_documented_and_optional():
     script = Path("scripts/runtime-evidence.sh")
     guide = Path("docs/guides/runtime-evidence.md").read_text(encoding="utf-8")