Add a Vagrant rig: test any node's role on a real VM

vagrant/README.md

@@ -0,0 +1,43 @@
+# Testing roles on real VMs
+
+The rspec suite compiles catalogs; it cannot tell you whether the role
+actually converges, starts its services, or survives a second apply. This
+rig can: it generates one Vagrant machine per node in `data/nodes/`, gives
+each VM the node's real hostname (so hiera classifies it exactly like the
+real machine), and applies the current working tree masterless, the same
+`puppet apply` + pluginsync path the cloud-init bootstrap uses.
+
+## Use
+
+Needs [Vagrant](https://developer.hashicorp.com/vagrant/install) with any
+provider the `bento/ubuntu-24.04` box supports (VirtualBox, Parallels,
+VMware Desktop, libvirt; amd64 and arm64 both exist, so Apple Silicon
+works natively). Override the box with `CONTROLREPO_BOX` for nodes that
+run 22.04 in production.
+
+```console
+cd vagrant
+vagrant status                 # machines, generated from data/nodes/
+vagrant up voxpupu.li          # provision + first apply
+./scripts/converge voxpupu.li  # edit the checkout, re-apply (live share)
+vagrant destroy -f voxpupu.li  # throw it away
+```
+
+The edit loop needs no commits: the checkout is a live read-only share,
+`converge` re-syncs it into the VM's environment directory and re-applies.
+
+## Lab overrides
+
+`overrides/<fqdn>.yaml` is appended to the node's hiera data in the VM's
+copy only (dummy secrets, `manage_borg: false`, anything a lab cannot
+have). `overrides/<fqdn>.hosts` lists extra names to point at 127.0.0.1
+for roles that serve more than their own fqdn.
+
+## Expected lab-only failures (not bugs)
+
+- certbot/ACME fails against the fake domains, so nginx stays HTTP-only;
+  the profiles' `letsencrypt_directory` fact guards handle this by design
+- anything that needs a real secret runs with the dummies from overrides
+- `scripts/apply` tolerates puppet exit code 6 (changes + failures)
+  because of the ACME failures; read the output for anything that is not
+  ACME before calling a run clean

vagrant/Vagrantfile

@@ -0,0 +1,61 @@
+# Test this controlrepo on real VMs, one machine per node in data/nodes.
+#
+# Machines are generated from ../data/nodes/*.yaml and named by fqdn, so a
+# node added to hiera is a `vagrant up <fqdn>` away from being testable.
+# Each VM gets the node's real hostname, which makes site.pp resolve the
+# same role hiera would give the real machine, and applies the working
+# tree masterless: the same `puppet apply` + pluginsync path the cloud
+# init bootstrap uses.
+#
+# Usage, from this directory:
+#
+#   vagrant status                 # list the generated machines
+#   vagrant up voxpupu.li          # bring one up (they never autostart)
+#   ./scripts/converge voxpupu.li  # re-apply the current working tree
+#
+# Lab expectations (documented, not bugs): certbot fails against the fake
+# domains so nginx stays HTTP-only (the profiles' letsencrypt_directory
+# fact guards handle that by design), and anything needing real secrets
+# gets dummies from overrides/<fqdn>.yaml.
+
+require "yaml"
+
+nodes = Dir[File.expand_path("../data/nodes/*.yaml", __dir__)].to_h do |f|
+  data = YAML.safe_load(File.read(f)) || {}
+  [File.basename(f, ".yaml"), data["role"]]
+end
+
+# main + 7 synapse workers + postgres + redis are python-hungry
+SIZING = Hash.new({ memory: 4096, cpus: 2 }).merge(
+  "matrix01.voxpupu.li" => { memory: 6144, cpus: 4 },
+)
+
+Vagrant.configure("2") do |config|
+  config.vm.box = ENV.fetch("CONTROLREPO_BOX", "bento/ubuntu-24.04")
+
+  config.vm.synced_folder "..", "/vagrant-controlrepo", mount_options: ["ro"]
+  config.vm.synced_folder ".", "/vagrant-rig", mount_options: ["ro"]
+
+  nodes.each do |fqdn, role|
+    config.vm.define fqdn, autostart: false do |node|
+      node.vm.hostname = fqdn
+
+      %w[parallels virtualbox libvirt].each do |provider|
+        node.vm.provider provider do |p|
+          p.memory = SIZING[fqdn][:memory]
+          p.cpus = SIZING[fqdn][:cpus]
+        end
+      end
+      node.vm.provider "vmware_desktop" do |v|
+        v.vmx["memsize"] = SIZING[fqdn][:memory].to_s
+        v.vmx["numvcpus"] = SIZING[fqdn][:cpus].to_s
+      end
+
+      node.vm.provision "shell", inline: <<-SHELL
+        set -euo pipefail
+        /vagrant-rig/scripts/bootstrap
+        /vagrant-rig/scripts/apply
+      SHELL
+    end
+  end
+end

vagrant/overrides/matrix01.voxpupu.li.hosts

@@ -0,0 +1 @@
+voxpupuli.party

vagrant/overrides/matrix01.voxpupu.li.yaml

@@ -0,0 +1,13 @@
+# Appended to data/nodes/matrix01.voxpupu.li.yaml in the VM's copy only.
+# Dummy secrets for the Matrix role (#202); real values live in eyaml.
+profiles::matrix::sensitive_postgres_password: 'lab-postgres-password'
+profiles::matrix::sensitive_macaroon_secret_key: 'lab-macaroon-secret-key-000000000000'
+profiles::matrix::sensitive_form_secret: 'lab-form-secret-000000000000'
+profiles::matrix::sensitive_s3_access_key: 'LABDUMMYACCESSKEY'
+profiles::matrix::sensitive_s3_secret_key: 'lab-dummy-secret-key'
+profiles::matrix::s3_bucket: 'lab-media'
+profiles::matrix::s3_region: 'lab'
+profiles::matrix::s3_endpoint: 'https://s3.lab.invalid'
+
+# No borg server to reach from a lab VM.
+profiles::base::manage_borg: false

vagrant/overrides/voxpupu.li.hosts

@@ -0,0 +1,4 @@
+grafana.voxpupu.li
+sentry.voxpupu.li
+puppetmodule.info
+www.puppetmodule.info

vagrant/overrides/voxpupu.li.yaml

@@ -0,0 +1,3 @@
+# Appended to data/nodes/voxpupu.li.yaml in the VM's copy only.
+# No borg server to reach from a lab VM.
+profiles::base::manage_borg: false

vagrant/scripts/apply

@@ -0,0 +1,31 @@
+#!/bin/bash
+# Runs ON THE VM (root). Sync the controlrepo working tree, overlay the lab
+# hiera overrides, resolve Puppetfile modules, apply masterless the same
+# way site.pp expects (`puppet apply` + its pluginsync hack).
+set -euo pipefail
+
+fqdn="$(hostname -f)"
+env_dir=/etc/puppetlabs/code/environments/production
+node_yaml="$env_dir/data/nodes/${fqdn}.yaml"
+overrides="/vagrant-rig/overrides/${fqdn}.yaml"
+
+mkdir -p "$env_dir"
+rsync -a --delete --exclude='.git' --exclude='vagrant/.vagrant' \
+  /vagrant-controlrepo/ "$env_dir/"
+
+if [ -f "$overrides" ]; then
+  printf '\n# --- vagrant overrides (appended by vagrant/scripts/apply) ---\n' >> "$node_yaml"
+  grep -v '^#' "$overrides" >> "$node_yaml"
+fi
+
+cd "$env_dir"
+/opt/puppetlabs/puppet/bin/r10k puppetfile install --moduledir "$env_dir/modules" --force
+
+# Exit 2 = changes applied. Exit 6 = changes + failures, tolerated because
+# certbot always fails against the lab's fake domains (ACME cannot
+# validate them) — inspect the output for anything that is not ACME.
+/opt/puppetlabs/bin/puppet apply \
+  --hiera_config "$env_dir/hiera.yaml" \
+  --modulepath "$env_dir/site:$env_dir/modules" \
+  --detailed-exitcodes \
+  "$env_dir/manifests/site.pp" || { rc=$?; [ "$rc" -eq 2 ] || [ "$rc" -eq 6 ] || exit "$rc"; }

vagrant/scripts/bootstrap

@@ -0,0 +1,41 @@
+#!/bin/bash
+# Runs ON THE VM (root), once. Installs the OpenVox agent and the gems the
+# masterless apply path needs, mirroring the production cloud-init.
+set -euo pipefail
+
+fqdn="$(hostname -f)"
+
+# Fake DNS for this node and any extra names its role serves
+grep -q '# controlrepo-vagrant' /etc/hosts || {
+  printf '# controlrepo-vagrant\n127.0.0.1 %s %s\n' "$fqdn" "${fqdn%%.*}" >> /etc/hosts
+  if [ -f "/vagrant-rig/overrides/${fqdn}.hosts" ]; then
+    sed 's/^/127.0.0.1 /' "/vagrant-rig/overrides/${fqdn}.hosts" >> /etc/hosts
+  fi
+}
+
+export DEBIAN_FRONTEND=noninteractive
+apt-get update -qq
+# git: r10k needs a functional git provider for git-sourced modules
+apt-get install -y -qq curl gnupg rsync git >/dev/null
+
+if ! dpkg -s openvox8-release >/dev/null 2>&1; then
+  . /etc/os-release
+  curl -sfLo /tmp/openvox8-release.deb "https://apt.voxpupuli.org/openvox8-release-ubuntu${VERSION_ID}.deb"
+  dpkg -i /tmp/openvox8-release.deb
+  apt-get update -qq
+fi
+apt-get install -y -qq openvox-agent >/dev/null
+
+# hiera.yaml declares the eyaml backend; the gem must exist and the key
+# paths must resolve even though the lab data carries no encrypted values
+/opt/puppetlabs/puppet/bin/gem list -i r10k >/dev/null 2>&1 || \
+  /opt/puppetlabs/puppet/bin/gem install r10k --no-document
+/opt/puppetlabs/puppet/bin/gem list -i hiera-eyaml >/dev/null 2>&1 || \
+  /opt/puppetlabs/puppet/bin/gem install hiera-eyaml --no-document
+if [ ! -f /etc/puppetlabs/puppet/keys/private_key.pkcs7.pem ]; then
+  install -d -m 0700 /etc/puppetlabs/puppet/keys
+  cd /etc/puppetlabs/puppet/keys
+  /opt/puppetlabs/puppet/bin/eyaml createkeys \
+    --pkcs7-private-key=private_key.pkcs7.pem \
+    --pkcs7-public-key=public_key.pkcs7.pem >/dev/null
+fi

vagrant/scripts/converge

@@ -0,0 +1,12 @@
+#!/bin/bash
+# Runs ON THE HOST, from the vagrant/ directory. Re-apply the current
+# working tree on a running machine (shared folders are live, so edits in
+# the checkout are already visible in the VM).
+#
+#   ./scripts/converge <fqdn>
+set -euo pipefail
+here="$(cd "$(dirname "$0")" && pwd)"
+cd "$here/.."
+
+machine="${1:?usage: ./scripts/converge <fqdn>}"
+vagrant ssh "$machine" -c 'sudo /vagrant-rig/scripts/apply'