rudimentary integration test

fixtures/integrationtest.json

@@ -0,0 +1,46 @@
+[
+  {
+    "model": "resources.hypervisor",
+    "pk": 1,
+    "fields": {
+      "name": "local",
+      "interface": "eth0"
+    }
+  },
+  {
+    "model": "resources.profile",
+    "pk": 1,
+    "fields": {
+      "name": "default",
+      "enabled": true
+    }
+  },
+  {
+    "model": "resources.profile",
+    "pk": 2,
+    "fields": {
+      "name": "offline",
+      "enabled": false
+    }
+  },
+  {
+    "model": "resources.virtualmachine",
+    "pk": 1,
+    "fields": {
+      "create": "2023-07-15T18:10:44.750Z",
+      "name": "integrationtest",
+      "ram_in_mb": 4096,
+      "cpu": 8,
+      "host": 1,
+      "role": "shellserver",
+      "profile": 1,
+      "image": "debian-12-generic-amd64-20240717-1811.qcow2",
+      "static_ip": "192.168.253.123",
+      "saltmaster_ip": "127.0.0.1",
+      "enabled": true,
+      "extra_storage_in_gb": 1,
+      "network_interface_count": 1,
+      "extra_storage_pool": "default"
+    }
+  }
+]

resources/views.py

@@ -106,7 +106,7 @@
         pacman-key --populate archlinux
         pacman -Syy
         pacman -S gnupg archlinux-keyring --noconfirm
-        pacman -Su salt cloud-utils e2fsprogs python-tornado --noconfirm --overwrite /usr/bin/growpart
+        pacman -Su salt cloud-utils e2fsprogs python-tornado qemu-guest-agent git --noconfirm --overwrite /usr/bin/growpart
     else
         apt-get update --allow-releaseinfo-change
         apt-get install curl -y
@@ -136,11 +136,13 @@
         fi
 
         apt-get update
-        apt-get install cloud-guest-utils e2fsprogs salt-minion  -y
+        apt-get install cloud-guest-utils e2fsprogs salt-minion salt-master qemu-guest-agent git  -y
     fi
 
     growpart /dev/vda 1 || /bin/true
     resize2fs /dev/vda1 || /bin/true
+
+    # Configure salt-minion
     echo "master: {vm_saltmaster}" > /etc/salt/minion
     echo "tcp_keepalive: True" >> /etc/salt/minion
     echo "tcp_keepalive_idle: 30" >> /etc/salt/minion
@@ -156,6 +158,24 @@
     echo "hypervisor: {hypervisor}" >> /etc/salt/grains
     systemctl stop salt-minion || /bin/true
     systemctl enable salt-minion || /bin/true
+
+
+    # Configure salt-master (in case no central master is used)
+    rm -rf /etc/homelabmanager
+    (cd /etc/; git clone https://github.com/vdloo/homelabmanager)
+    if [ ! -L /srv/salt ]; then ln -s /etc/homelabmanager/saltstack/salt /srv/salt; fi
+    if [ ! -L /srv/pillar ]; then ln -s /etc/homelabmanager/saltstack/pillar /srv/pillar; fi
+    if [ ! -L /srv/reactor ]; then ln -s /etc/homelabmanager/saltstack/reactor /srv/reactor; fi
+    # Obviously never configure a Saltmaster like this in any real environment
+    # We only bind on localhost, no external can connect to this salt-master
+    echo "interface: 127.0.0.1" >> /etc/salt/master
+    echo "open_mode: True" > /etc/salt/master
+    echo "auto_accept: True" >> /etc/salt/master
+    echo "reactor:" >> /etc/salt/master
+    echo "  - 'salt/minion/*/start':" >> /etc/salt/master
+    echo "    - /srv/reactor/salt.sls" >> /etc/salt/master
+    systemctl stop salt-master || /bin/true
+    systemctl enable salt-master || /bin/true
     rm -f /etc/salt/minion_id
 
     # Delete the cloud-config config and reboot

scripts/integration_test.sh

@@ -0,0 +1,105 @@
+#!/usr/bin/env bash
+set -e
+
+SUCCESS=false
+HOMELABMANAGERPORT=$(shuf -i 4500-5500 -n 1)
+TESTVMIP="192.168.1.$(shuf -i 20-250 -n 1)"
+ROLETOTEST=${1-shellserver}
+echo "Testing role $ROLETOTEST on homelabmanager 127.0.0.1:$HOMELABMANAGERPORT with VM IP $TESTVMIP"
+
+function terraform_environment()
+{
+    # Ensure an environment
+    export LIBVIRT_DEFAULT_URI="qemu:///system" 
+    mkdir -p /tmp/integration-test-environment/terraform
+    cd /tmp/integration-test-environment/terraform
+    touch checksum
+    chronic wget "127.0.0.1:$HOMELABMANAGERPORT/terraform/?host=$1" -O main.tf
+    chronic sudo terraform init || /bin/true
+    chronic sudo terraform validate
+    if ! cat checksum | md5sum --quiet -c; then
+        md5sum main.tf > checksum
+        sudo terraform destroy --auto-approve || /bin/true
+        sudo virsh list --all | grep -v debian-10 | grep -v debian-12 | grep -v focal | grep -v jammy-server | grep -v ubuntu18 | grep -v arch-openstack | grep shut | awk '{print$2}' | xargs -I {} sh -c 'sudo virsh destroy {} || /bin/true; sudo virsh undefine {} || /bin/true' || /bin/true
+        cd /var/lib/libvirt/images
+        ls /var/lib/libvirt/images | grep -v focal | grep -v jammy-server | grep -v bionic-server | grep -v debian-10 | grep -v debian-12 | grep -v arch-openstack | xargs -I {} sudo rm -rf "{}"
+        cd -
+        sudo terraform apply --auto-approve || /bin/true
+        sleep 1
+        sudo terraform apply --auto-approve
+    fi
+}
+
+function kill_any_homelabmanager_api()
+{
+    pkill -f "127.0.0.1:$HOMELABMANAGERPORT" || /bin/true
+}
+
+function print_test_result()
+{
+    if $SUCCESS; then
+        echo "Tests passed!"
+        exit 0
+    else
+        echo "Test failed!"
+        /bin/false
+        exit 1
+    fi
+}
+
+# Clean up from any previous attempt
+kill_any_homelabmanager_api
+sudo rm -rf /tmp/integration-test-environment
+
+# Copy the repo to the test environment
+HOMELABDIR=$(dirname $(dirname $(realpath -s "$0")))
+mkdir -p /tmp/integration-test-environment
+cd /tmp/integration-test-environment
+cp -R $HOMELABDIR /tmp/integration-test-environment/
+
+# Start the homelabmanager API
+cd /tmp/integration-test-environment/homelabmanager
+python3 -m venv venv
+. venv/bin/activate
+pip3 install -r requirements/dev.txt
+ACTIVE_INTERFACE=$(ip addr | awk '/state UP/ {print $2}' | grep -v macvtap | cut -d ':' -f1 | tail -n 1)
+./manage.py migrate
+sed -i "s/eth0/$ACTIVE_INTERFACE/g" fixtures/integrationtest.json
+sed -i "s/192.168.253.123/$TESTVMIP/g" fixtures/integrationtest.json
+sed -i "s/shellserver/$ROLETOTEST/g" fixtures/integrationtest.json
+./manage.py loaddata fixtures/integrationtest.json
+export VM_SALTMASTER_IP=127.0.0.1
+./manage.py runserver 127.0.0.1:$HOMELABMANAGERPORT &
+sleep 5
+
+if test -d /tmp/integration-test-environment/terraform; then
+    terraform_environment cleanup
+fi
+terraform_environment local
+
+echo "Giving things some time to get started"
+sleep 120
+
+# Check if the unit tests pass on the server
+MACHINECHECKMAXATTEMPTS=10
+MACHINECHECKCOUNT=0
+while [ -z "$MACHINECHECKPID" ] && [ "$MACHINECHECKCOUNT" -lt "$MACHINECHECKMAXATTEMPTS" ] && ! sudo virsh -c qemu:///system qemu-agent-command integrationtest_on_local "{\"execute\": \"guest-exec-status\", \"arguments\": { \"pid\": $MACHINECHECKPID }}" | jq -r '.return.["out-data"]' | base64 --decode | grep 'All tests pass'; do
+    let MACHINECHECKCOUNT=MACHINECHECKCOUNT+1
+    echo "Checking if machine-check is passing yet"
+    MACHINECHECKPID=$(sudo virsh -c qemu:///system qemu-agent-command integrationtest_on_local '{"execute": "guest-exec", "arguments": { "path": "/usr/bin/machine-check", "arg": [ ], "capture-output": true }}' | jq .return.pid);
+    sleep 30;
+done
+
+if [ "$MACHINECHECKCOUNT" -lt "$MACHINECHECKMAXATTEMPTS" ]; then
+    echo "Tests passed!"
+    SUCCESS=true
+else
+    echo "Ran out of attempts! Test failed. The machine-check assertions did not pass, or failed to run."
+fi
+
+terraform_environment cleanup
+
+# Clean up after ourselves
+trap 'terraform_environment cleanup' EXIT
+trap kill_any_homelabmanager_api EXIT
+trap print_test_result EXIT