From 1e06e4b67763e61b9541df668f9f0ff0e3dc749a Mon Sep 17 00:00:00 2001
From: Adam Fidel <adam@fidel.id>
Date: Thu, 19 Jun 2025 11:46:20 -0500
Subject: [PATCH 01/14] feat: add integration testing for images

---
 .github/workflows/build-image-stable.yml |   8 ++
 .github/workflows/integration-test.yml   |  94 ++++++++++++++++
 tests/__fixtures__/qcow2-config.toml     |   4 +
 tests/qemu_test.py                       | 132 +++++++++++++++++++++++
 4 files changed, 238 insertions(+)
 create mode 100644 .github/workflows/integration-test.yml
 create mode 100644 tests/__fixtures__/qcow2-config.toml
 create mode 100644 tests/qemu_test.py

diff --git a/.github/workflows/build-image-stable.yml b/.github/workflows/build-image-stable.yml
index aa7f74f80..cd812ebb2 100644
--- a/.github/workflows/build-image-stable.yml
+++ b/.github/workflows/build-image-stable.yml
@@ -25,10 +25,18 @@ jobs:
       fail-fast: false
       matrix:
         brand_name: ["aurora"]
+    
     with:
       # kernel_pin: 6.13.8-200.fc41.x86_64 ## This is where kernels get pinned.
       brand_name: ${{ matrix.brand_name }}
       stream_name: stable
+  
+  integration-tests:
+    uses: ./.github/workflows/integration-test.yml
+    secrets: inherit
+    needs: build-image-stable
+    with:
+      image-ref: ghcr.io/${{ github.repository_owner }}/aurora:stable
 
   generate-release:
     name: Generate Release
diff --git a/.github/workflows/integration-test.yml b/.github/workflows/integration-test.yml
new file mode 100644
index 000000000..54bdf7d18
--- /dev/null
+++ b/.github/workflows/integration-test.yml
@@ -0,0 +1,94 @@
+name: Integration Tests
+
+on:
+  workflow_call:
+    inputs:
+      image-ref:
+        required: true
+        type: string
+
+jobs:
+  integration-test:
+    runs-on: ubuntu-24.04
+    env:
+      SSH_PORT: 2222
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Build QCOW Image
+        id: build
+        uses: bootc-warehouse/bootc-image-builder-action@main
+        with:
+          config-file: ./tests/__fixtures__/qcow2-config.toml
+          image: ${{ inputs.image-ref }}
+          types: |
+            qcow2
+
+      - name: Setup KVM
+        run: |
+          echo 'KERNEL=="kvm", GROUP="kvm", MODE="0666", OPTIONS+="static_node=kvm"' | sudo tee /etc/udev/rules.d/99-kvm.rules
+          sudo udevadm control --reload-rules
+          sudo udevadm trigger --name-match=kvm --settle
+          ls -l /dev/kvm
+
+          sudo apt-get update
+          sudo apt-get install -y qemu-system-x86
+
+      - name: Run QEMU VM
+        id: run-qemu
+        env:
+          QCOW2_PATH: ${{ steps.build.outputs.qcow2-output-path }}
+        run: |
+          qemu-system-x86_64 \
+            -enable-kvm \
+            -cpu host \
+            -smp 2 \
+            -display none \
+            -m 8192 \
+            -device virtio-rng-pci \
+            -serial file:/tmp/qemu-serial.log \
+            -drive file=$QCOW2_PATH,if=virtio \
+            -snapshot \
+            -net nic,model=virtio \
+            -net user,hostfwd=tcp::$SSH_PORT-:22 &
+          QEMU_PID=$!
+
+          # Wait for log file to appear (max 10s)
+          for i in {1..10}; do
+            [ -f "$LOG_FILE" ] && break
+            sleep 1
+          done
+
+          # Wait 20 seconds and show log
+          timeout 20 tail -f /tmp/qemu-serial.log || true
+
+          disown $QEMU_PID
+          echo "QEMU_PID=$QEMU_PID" >> $GITHUB_OUTPUT
+
+      - name: Wait for SSH
+        run: |
+          while ! nc -z localhost $SSH_PORT; do
+            sleep 10
+          done
+
+      - name: Setup Go
+        uses: actions/setup-go@v4
+
+      - name: Run Tests
+        working-directory: ./tests
+        env:
+          SSH_PRIVATE_KEY: ${{ secrets.CI_SSH_PRIVATE_KEY }}
+          SSH_PORT: ${{ env.SSH_PORT }}
+        run: |
+          go test -v ./...
+
+      - name: Stop QEMU
+        env:
+          QEMU_PID: ${{ steps.run-qemu.outputs.QEMU_PID }}
+        run: |
+          echo "Stopping QEMU with PID $QEMU_PID"
+          kill $QEMU_PID
+          wait $QEMU_PID || true
+          sleep 5
+          echo "QEMU stopped"
diff --git a/tests/__fixtures__/qcow2-config.toml b/tests/__fixtures__/qcow2-config.toml
new file mode 100644
index 000000000..58486c29b
--- /dev/null
+++ b/tests/__fixtures__/qcow2-config.toml
@@ -0,0 +1,4 @@
+[[customizations.user]]
+name = "aurora-ci-user"
+key = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIM353kmMBTMCgelDFUesQgRfgLdXOjIchlr0DyQCtTDG aurora-ci-user@github-actions.com"
+groups = ["wheel"]
diff --git a/tests/qemu_test.py b/tests/qemu_test.py
new file mode 100644
index 000000000..c93ba5bb0
--- /dev/null
+++ b/tests/qemu_test.py
@@ -0,0 +1,132 @@
+#!/usr/bin/env python3
+"""
+Integration tests for Aurora bootc image.
+Tests basic functionality by connecting to a VM via SSH.
+"""
+
+import os
+import sys
+import time
+import subprocess
+import tempfile
+from typing import List, Tuple
+
+
+class SSHClient:
+    def __init__(self, hostname: str, port: int, username: str, private_key: str):
+        self.hostname = hostname
+        self.port = port
+        self.username = username
+        
+        # Write private key to temporary file
+        self.key_file = tempfile.NamedTemporaryFile(mode='w', delete=False, suffix='.pem')
+        self.key_file.write(private_key)
+        self.key_file.close()
+        
+        # Set proper permissions on the key file
+        os.chmod(self.key_file.name, 0o600)
+    
+    def __del__(self):
+        # Clean up temporary key file
+        if hasattr(self, 'key_file'):
+            try:
+                os.unlink(self.key_file.name)
+            except FileNotFoundError:
+                pass
+    
+    def run_command(self, command: str, timeout: int = 30) -> Tuple[int, str, str]:
+        """Run a command via SSH and return (returncode, stdout, stderr)"""
+        ssh_cmd = [
+            'ssh',
+            '-i', self.key_file.name,
+            '-p', str(self.port),
+            '-o', 'StrictHostKeyChecking=no',
+            '-o', 'UserKnownHostsFile=/dev/null',
+            '-o', 'LogLevel=ERROR',
+            f'{self.username}@{self.hostname}',
+            command
+        ]
+        
+        try:
+            result = subprocess.run(
+                ssh_cmd,
+                capture_output=True,
+                text=True,
+                timeout=timeout
+            )
+            return result.returncode, result.stdout, result.stderr
+        except subprocess.TimeoutExpired:
+            return 124, "", f"Command timed out after {timeout} seconds"
+
+
+def run_test(ssh_client: SSHClient, test_name: str, command: str, expected_text: str, max_retries: int = 3) -> bool:
+    """Run a single test with retries"""
+    print(f"Running test: {test_name}")
+    
+    for attempt in range(max_retries):
+        if attempt > 0:
+            print(f"  Retry {attempt}/{max_retries - 1}")
+            time.sleep(5)
+        
+        returncode, stdout, stderr = ssh_client.run_command(command)
+        
+        if returncode != 0:
+            print(f"  ❌ Command failed with exit code {returncode}")
+            print(f"     stdout: {stdout.strip()}")
+            print(f"     stderr: {stderr.strip()}")
+            continue
+        
+        if expected_text in stdout:
+            print(f"  ✅ PASS")
+            return True
+        else:
+            print(f"  ❌ Expected '{expected_text}' not found in output")
+            print(f"     stdout: {stdout.strip()}")
+    
+    print(f"  ❌ FAIL after {max_retries} attempts")
+    return False
+
+
+def main():
+    # Get environment variables
+    ssh_private_key = os.getenv('SSH_PRIVATE_KEY')
+    ssh_port = int(os.getenv('SSH_PORT', '2222'))
+    
+    if not ssh_private_key:
+        print("❌ SSH_PRIVATE_KEY environment variable not set")
+        sys.exit(1)
+    
+    # Create SSH client
+    ssh_client = SSHClient('localhost', ssh_port, 'ci-user', ssh_private_key)
+    
+    # Define tests
+    tests = [
+        ("KernelInstalled", "rpm -q kernel", "kernel"),
+        ("CheckSELinuxStatus", "getenforce", "Enforcing"),
+        ("CheckSudoersValid", "sudo visudo -cf /etc/sudoers", "/etc/sudoers: parsed OK"),
+        ("CheckNftablesServiceEnabled", "systemctl is-enabled nftables", "enabled"),
+        ("CheckNftablesServiceActive", "systemctl is-active nftables", "active"),
+    ]
+    
+    print("Starting Aurora bootc integration tests...")
+    print(f"Connecting to localhost:{ssh_port} as ci-user")
+    
+    passed = 0
+    failed = 0
+    
+    for test_name, command, expected_text in tests:
+        if run_test(ssh_client, test_name, command, expected_text):
+            passed += 1
+        else:
+            failed += 1
+    
+    print(f"\nResults: {passed} passed, {failed} failed")
+    
+    if failed > 0:
+        sys.exit(1)
+    else:
+        print("✅ All tests passed!")
+
+
+if __name__ == "__main__":
+    main()

From fe0dfe9f05baa7de6c166d8e8dd3b50736238bf1 Mon Sep 17 00:00:00 2001
From: Adam Fidel <ledif@users.noreply.github.com>
Date: Thu, 19 Jun 2025 11:56:01 -0500
Subject: [PATCH 02/14] bootc-warehouse -> osbuild

Co-authored-by: Robert Sturla <robertsturla@outlook.com>
---
 .github/workflows/integration-test.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/integration-test.yml b/.github/workflows/integration-test.yml
index 54bdf7d18..0b2a793ae 100644
--- a/.github/workflows/integration-test.yml
+++ b/.github/workflows/integration-test.yml
@@ -18,7 +18,7 @@ jobs:
 
       - name: Build QCOW Image
         id: build
-        uses: bootc-warehouse/bootc-image-builder-action@main
+        uses: osbuild/bootc-image-builder-action@v0.0.2
         with:
           config-file: ./tests/__fixtures__/qcow2-config.toml
           image: ${{ inputs.image-ref }}

From bff742170b19af57b319f8d5909445538a4a18cb Mon Sep 17 00:00:00 2001
From: Adam Fidel <adam@fidel.id>
Date: Thu, 19 Jun 2025 12:09:13 -0500
Subject: [PATCH 03/14] update SSH key and call Python test

---
 .github/workflows/integration-test.yml | 15 +--------------
 tests/__fixtures__/qcow2-config.toml   |  2 +-
 2 files changed, 2 insertions(+), 15 deletions(-)

diff --git a/.github/workflows/integration-test.yml b/.github/workflows/integration-test.yml
index 0b2a793ae..f9127a940 100644
--- a/.github/workflows/integration-test.yml
+++ b/.github/workflows/integration-test.yml
@@ -72,23 +72,10 @@ jobs:
             sleep 10
           done
 
-      - name: Setup Go
-        uses: actions/setup-go@v4
-
       - name: Run Tests
         working-directory: ./tests
         env:
           SSH_PRIVATE_KEY: ${{ secrets.CI_SSH_PRIVATE_KEY }}
           SSH_PORT: ${{ env.SSH_PORT }}
         run: |
-          go test -v ./...
-
-      - name: Stop QEMU
-        env:
-          QEMU_PID: ${{ steps.run-qemu.outputs.QEMU_PID }}
-        run: |
-          echo "Stopping QEMU with PID $QEMU_PID"
-          kill $QEMU_PID
-          wait $QEMU_PID || true
-          sleep 5
-          echo "QEMU stopped"
+          python3 ./qemu_test.py
diff --git a/tests/__fixtures__/qcow2-config.toml b/tests/__fixtures__/qcow2-config.toml
index 58486c29b..fd9690357 100644
--- a/tests/__fixtures__/qcow2-config.toml
+++ b/tests/__fixtures__/qcow2-config.toml
@@ -1,4 +1,4 @@
 [[customizations.user]]
 name = "aurora-ci-user"
-key = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIM353kmMBTMCgelDFUesQgRfgLdXOjIchlr0DyQCtTDG aurora-ci-user@github-actions.com"
+key = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIMnkmox0kV4ediDt7MsvHlZ3mncW6GIZzl8KgrJn+zbr aurora-ci-user@github-actions.com"
 groups = ["wheel"]

From 9705d897e2192dbbe6f3a82cc62bb28a2bd0a668 Mon Sep 17 00:00:00 2001
From: Adam Fidel <adam@fidel.id>
Date: Thu, 19 Jun 2025 13:20:07 -0500
Subject: [PATCH 04/14] cleanup disk

---
 .github/workflows/integration-test.yml | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/.github/workflows/integration-test.yml b/.github/workflows/integration-test.yml
index f9127a940..636f2494f 100644
--- a/.github/workflows/integration-test.yml
+++ b/.github/workflows/integration-test.yml
@@ -16,6 +16,11 @@ jobs:
       - name: Checkout
         uses: actions/checkout@v4
 
+      - name: Maximize build space
+        uses: ublue-os/remove-unwanted-software@cc0becac701cf642c8f0a6613bbdaf5dc36b259e # v9
+        with:
+          remove-codeql: true
+
       - name: Build QCOW Image
         id: build
         uses: osbuild/bootc-image-builder-action@v0.0.2

From 61820b37b3a3125cacf18f8b5c92bec5c3730be6 Mon Sep 17 00:00:00 2001
From: Adam Fidel <adam@fidel.id>
Date: Thu, 19 Jun 2025 13:35:16 -0500
Subject: [PATCH 05/14] cleanup Python

---
 tests/qemu_test.py | 59 +++++++++++++++++++++-------------------------
 1 file changed, 27 insertions(+), 32 deletions(-)

diff --git a/tests/qemu_test.py b/tests/qemu_test.py
index c93ba5bb0..3f7d8d3de 100644
--- a/tests/qemu_test.py
+++ b/tests/qemu_test.py
@@ -9,7 +9,7 @@
 import time
 import subprocess
 import tempfile
-from typing import List, Tuple
+from typing import Tuple
 
 
 class SSHClient:
@@ -17,23 +17,20 @@ def __init__(self, hostname: str, port: int, username: str, private_key: str):
         self.hostname = hostname
         self.port = port
         self.username = username
-        
+
         # Write private key to temporary file
         self.key_file = tempfile.NamedTemporaryFile(mode='w', delete=False, suffix='.pem')
         self.key_file.write(private_key)
         self.key_file.close()
-        
-        # Set proper permissions on the key file
         os.chmod(self.key_file.name, 0o600)
-    
+
     def __del__(self):
-        # Clean up temporary key file
         if hasattr(self, 'key_file'):
             try:
                 os.unlink(self.key_file.name)
             except FileNotFoundError:
                 pass
-    
+
     def run_command(self, command: str, timeout: int = 30) -> Tuple[int, str, str]:
         """Run a command via SSH and return (returncode, stdout, stderr)"""
         ssh_cmd = [
@@ -46,7 +43,7 @@ def run_command(self, command: str, timeout: int = 30) -> Tuple[int, str, str]:
             f'{self.username}@{self.hostname}',
             command
         ]
-        
+
         try:
             result = subprocess.run(
                 ssh_cmd,
@@ -62,28 +59,28 @@ def run_command(self, command: str, timeout: int = 30) -> Tuple[int, str, str]:
 def run_test(ssh_client: SSHClient, test_name: str, command: str, expected_text: str, max_retries: int = 3) -> bool:
     """Run a single test with retries"""
     print(f"Running test: {test_name}")
-    
+
     for attempt in range(max_retries):
         if attempt > 0:
             print(f"  Retry {attempt}/{max_retries - 1}")
             time.sleep(5)
-        
+
         returncode, stdout, stderr = ssh_client.run_command(command)
-        
+
         if returncode != 0:
-            print(f"  ❌ Command failed with exit code {returncode}")
+            print(f" Command failed with exit code {returncode}")
             print(f"     stdout: {stdout.strip()}")
             print(f"     stderr: {stderr.strip()}")
             continue
-        
+
         if expected_text in stdout:
-            print(f"  ✅ PASS")
+            print(f"PASS")
             return True
         else:
-            print(f"  ❌ Expected '{expected_text}' not found in output")
-            print(f"     stdout: {stdout.strip()}")
-    
-    print(f"  ❌ FAIL after {max_retries} attempts")
+            print(f"Expected '{expected_text}' not found in output")
+            print(f"stdout: {stdout.strip()}")
+
+    print(f"FAIL after {max_retries} attempts")
     return False
 
 
@@ -91,14 +88,14 @@ def main():
     # Get environment variables
     ssh_private_key = os.getenv('SSH_PRIVATE_KEY')
     ssh_port = int(os.getenv('SSH_PORT', '2222'))
-    
+
     if not ssh_private_key:
-        print("❌ SSH_PRIVATE_KEY environment variable not set")
+        print("ERROR: SSH_PRIVATE_KEY environment variable not set")
         sys.exit(1)
-    
+
     # Create SSH client
-    ssh_client = SSHClient('localhost', ssh_port, 'ci-user', ssh_private_key)
-    
+    ssh_client = SSHClient('localhost', ssh_port, 'aurora-ci-user', ssh_private_key)
+
     # Define tests
     tests = [
         ("KernelInstalled", "rpm -q kernel", "kernel"),
@@ -107,26 +104,24 @@ def main():
         ("CheckNftablesServiceEnabled", "systemctl is-enabled nftables", "enabled"),
         ("CheckNftablesServiceActive", "systemctl is-active nftables", "active"),
     ]
-    
-    print("Starting Aurora bootc integration tests...")
-    print(f"Connecting to localhost:{ssh_port} as ci-user")
-    
+
+    print("Starting Aurora integration tests...")
+
     passed = 0
     failed = 0
-    
+
     for test_name, command, expected_text in tests:
         if run_test(ssh_client, test_name, command, expected_text):
             passed += 1
         else:
             failed += 1
-    
+
     print(f"\nResults: {passed} passed, {failed} failed")
-    
+
     if failed > 0:
         sys.exit(1)
     else:
-        print("✅ All tests passed!")
-
+        print("All tests passed!")
 
 if __name__ == "__main__":
     main()

From ccec1b622a20963284e22616144015994be96f7d Mon Sep 17 00:00:00 2001
From: Adam Fidel <adam@fidel.id>
Date: Thu, 19 Jun 2025 15:03:22 -0500
Subject: [PATCH 06/14] rename CI_SSH_PRIVATE_KEY -> SSH_PRIVATE_KEY

---
 .github/workflows/build-image-stable.yml | 1 +
 .github/workflows/integration-test.yml   | 2 +-
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/build-image-stable.yml b/.github/workflows/build-image-stable.yml
index cd812ebb2..f43a36359 100644
--- a/.github/workflows/build-image-stable.yml
+++ b/.github/workflows/build-image-stable.yml
@@ -32,6 +32,7 @@ jobs:
       stream_name: stable
   
   integration-tests:
+    name: Integration Test
     uses: ./.github/workflows/integration-test.yml
     secrets: inherit
     needs: build-image-stable
diff --git a/.github/workflows/integration-test.yml b/.github/workflows/integration-test.yml
index 636f2494f..b55e614d9 100644
--- a/.github/workflows/integration-test.yml
+++ b/.github/workflows/integration-test.yml
@@ -80,7 +80,7 @@ jobs:
       - name: Run Tests
         working-directory: ./tests
         env:
-          SSH_PRIVATE_KEY: ${{ secrets.CI_SSH_PRIVATE_KEY }}
+          SSH_PRIVATE_KEY: ${{ secrets.SSH_PRIVATE_KEY }}
           SSH_PORT: ${{ env.SSH_PORT }}
         run: |
           python3 ./qemu_test.py

From bd3167e992c392227a183c3f126a943c080068e9 Mon Sep 17 00:00:00 2001
From: Adam Fidel <adam@fidel.id>
Date: Tue, 22 Jul 2025 21:07:49 -0500
Subject: [PATCH 07/14] fix: generate SSH secrets in the job

---
 .github/workflows/integration-test.yml | 23 ++++++++++++++++++++---
 1 file changed, 20 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/integration-test.yml b/.github/workflows/integration-test.yml
index b55e614d9..ccc2aee64 100644
--- a/.github/workflows/integration-test.yml
+++ b/.github/workflows/integration-test.yml
@@ -16,6 +16,21 @@ jobs:
       - name: Checkout
         uses: actions/checkout@v4
 
+      - name: Generate SSH Key Pair
+        run: |
+          ssh-keygen -t ed25519 -f ~/.ssh/test_key -N "" -C "aurora-ci-user@github-actions.com"
+          echo "SSH_PRIVATE_KEY<<EOF" >> $GITHUB_ENV
+          cat ~/.ssh/test_key >> $GITHUB_ENV
+          echo "EOF" >> $GITHUB_ENV
+
+          # Update config file with new public key
+          PUBLIC_KEY=$(cat ~/.ssh/test_key.pub)
+          sed -i "s|key = \".*\"|key = \"$PUBLIC_KEY\"|" ./tests/__fixtures__/qcow2-config.toml
+
+          # Verify the update
+          echo "Updated config file:"
+          cat ./tests/__fixtures__/qcow2-config.toml
+
       - name: Maximize build space
         uses: ublue-os/remove-unwanted-software@cc0becac701cf642c8f0a6613bbdaf5dc36b259e # v9
         with:
@@ -45,6 +60,8 @@ jobs:
         env:
           QCOW2_PATH: ${{ steps.build.outputs.qcow2-output-path }}
         run: |
+          LOG_FILE="/tmp/qemu-serial.log"
+
           qemu-system-x86_64 \
             -enable-kvm \
             -cpu host \
@@ -52,7 +69,7 @@ jobs:
             -display none \
             -m 8192 \
             -device virtio-rng-pci \
-            -serial file:/tmp/qemu-serial.log \
+            -serial file:$LOG_FILE \
             -drive file=$QCOW2_PATH,if=virtio \
             -snapshot \
             -net nic,model=virtio \
@@ -66,7 +83,7 @@ jobs:
           done
 
           # Wait 20 seconds and show log
-          timeout 20 tail -f /tmp/qemu-serial.log || true
+          timeout 20 tail -f $LOG_FILE || true
 
           disown $QEMU_PID
           echo "QEMU_PID=$QEMU_PID" >> $GITHUB_OUTPUT
@@ -80,7 +97,7 @@ jobs:
       - name: Run Tests
         working-directory: ./tests
         env:
-          SSH_PRIVATE_KEY: ${{ secrets.SSH_PRIVATE_KEY }}
+          SSH_PRIVATE_KEY: ${{ env.SSH_PRIVATE_KEY }}
           SSH_PORT: ${{ env.SSH_PORT }}
         run: |
           python3 ./qemu_test.py

From c0244a25f987e808e3ec2b5251775eceff68fe99 Mon Sep 17 00:00:00 2001
From: Adam Fidel <adam@fidel.id>
Date: Tue, 22 Jul 2025 21:34:37 -0500
Subject: [PATCH 08/14] rearchitect to run nightly

---
 .github/workflows/build-image-stable.yml      |  8 -----
 .../workflows/nightly-integration-tests.yml   | 36 +++++++++++++++++++
 tests/qemu_test.py                            |  3 +-
 3 files changed, 38 insertions(+), 9 deletions(-)
 create mode 100644 .github/workflows/nightly-integration-tests.yml

diff --git a/.github/workflows/build-image-stable.yml b/.github/workflows/build-image-stable.yml
index f43a36359..0396b39b4 100644
--- a/.github/workflows/build-image-stable.yml
+++ b/.github/workflows/build-image-stable.yml
@@ -31,14 +31,6 @@ jobs:
       brand_name: ${{ matrix.brand_name }}
       stream_name: stable
   
-  integration-tests:
-    name: Integration Test
-    uses: ./.github/workflows/integration-test.yml
-    secrets: inherit
-    needs: build-image-stable
-    with:
-      image-ref: ghcr.io/${{ github.repository_owner }}/aurora:stable
-
   generate-release:
     name: Generate Release
     needs: [build-image-stable]
diff --git a/.github/workflows/nightly-integration-tests.yml b/.github/workflows/nightly-integration-tests.yml
new file mode 100644
index 000000000..f84bbb17e
--- /dev/null
+++ b/.github/workflows/nightly-integration-tests.yml
@@ -0,0 +1,36 @@
+name: Nightly Integration Tests
+
+on:
+  schedule:
+    - cron: "30 6 * * *" # 6:30 UTC daily
+  workflow_dispatch:
+  pull_request:
+    branches:
+      - main
+    paths:
+      - ".github/workflows/integration-test.yml"
+      - ".github/workflows/nightly-integration-tests.yml"
+      - "tests/**"
+
+jobs:
+  integration-test:
+    name: Integration Test
+    uses: ./.github/workflows/integration-test.yml
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - image_flavor: "main"
+            image_ref: "ghcr.io/${{ github.repository_owner }}/aurora:latest"
+          - image_flavor: "nvidia"
+            image_ref: "ghcr.io/${{ github.repository_owner }}/aurora-nvidia:latest"
+          - image_flavor: "nvidia-open"
+            image_ref: "ghcr.io/${{ github.repository_owner }}/aurora-nvidia-open:latest"
+          - image_flavor: "dx"
+            image_ref: "ghcr.io/${{ github.repository_owner }}/aurora-dx:latest"
+          - image_flavor: "dx-nvidia"
+            image_ref: "ghcr.io/${{ github.repository_owner }}/aurora-dx-nvidia:latest"
+          - image_flavor: "dx-nvidia-open"
+            image_ref: "ghcr.io/${{ github.repository_owner }}/aurora-dx-nvidia-open:latest"
+    with:
+      image-ref: ${{ matrix.image_ref }}
diff --git a/tests/qemu_test.py b/tests/qemu_test.py
index 3f7d8d3de..10a77e238 100644
--- a/tests/qemu_test.py
+++ b/tests/qemu_test.py
@@ -74,7 +74,7 @@ def run_test(ssh_client: SSHClient, test_name: str, command: str, expected_text:
             continue
 
         if expected_text in stdout:
-            print(f"PASS")
+            print("PASS")
             return True
         else:
             print(f"Expected '{expected_text}' not found in output")
@@ -123,5 +123,6 @@ def main():
     else:
         print("All tests passed!")
 
+
 if __name__ == "__main__":
     main()

From 2b49930ba26591aed39e90809ffa600fd4c0a57b Mon Sep 17 00:00:00 2001
From: Adam Fidel <adam@fidel.id>
Date: Tue, 22 Jul 2025 22:17:46 -0500
Subject: [PATCH 09/14] fix qcow config

---
 .github/workflows/integration-test.yml        | 55 ++++++++++++++++---
 .../workflows/nightly-integration-tests.yml   | 20 +++----
 tests/__fixtures__/qcow2-config.toml          |  4 --
 3 files changed, 53 insertions(+), 26 deletions(-)
 delete mode 100644 tests/__fixtures__/qcow2-config.toml

diff --git a/.github/workflows/integration-test.yml b/.github/workflows/integration-test.yml
index ccc2aee64..5e8f386d0 100644
--- a/.github/workflows/integration-test.yml
+++ b/.github/workflows/integration-test.yml
@@ -23,13 +23,17 @@ jobs:
           cat ~/.ssh/test_key >> $GITHUB_ENV
           echo "EOF" >> $GITHUB_ENV
 
-          # Update config file with new public key
           PUBLIC_KEY=$(cat ~/.ssh/test_key.pub)
-          sed -i "s|key = \".*\"|key = \"$PUBLIC_KEY\"|" ./tests/__fixtures__/qcow2-config.toml
+          cat > ./tests/qcow2-config.toml << EOF
+          [[customizations.user]]
+          name = "aurora-ci-user"
+          key = "$PUBLIC_KEY"
+          groups = ["wheel"]
+          EOF
 
           # Verify the update
           echo "Updated config file:"
-          cat ./tests/__fixtures__/qcow2-config.toml
+          cat ./tests/qcow2-config.toml
 
       - name: Maximize build space
         uses: ublue-os/remove-unwanted-software@cc0becac701cf642c8f0a6613bbdaf5dc36b259e # v9
@@ -40,7 +44,7 @@ jobs:
         id: build
         uses: osbuild/bootc-image-builder-action@v0.0.2
         with:
-          config-file: ./tests/__fixtures__/qcow2-config.toml
+          config-file: ./tests/qcow2-config.toml
           image: ${{ inputs.image-ref }}
           types: |
             qcow2
@@ -76,24 +80,57 @@ jobs:
             -net user,hostfwd=tcp::$SSH_PORT-:22 &
           QEMU_PID=$!
 
-          # Wait for log file to appear (max 10s)
-          for i in {1..10}; do
+          echo "QEMU started with PID: $QEMU_PID"
+
+          # Wait for log file to appear (max 30s)
+          for i in {1..30}; do
             [ -f "$LOG_FILE" ] && break
             sleep 1
           done
 
-          # Wait 20 seconds and show log
-          timeout 20 tail -f $LOG_FILE || true
+          if [ ! -f "$LOG_FILE" ]; then
+            echo "Warning: Log file did not appear"
+          fi
+
+          # Wait 60 seconds and show log
+          echo "Waiting for VM to boot (60 seconds)..."
+          timeout 60 tail -f $LOG_FILE || true
+
+          echo "Current log contents:"
+          cat $LOG_FILE || echo "No log file available"
 
           disown $QEMU_PID
           echo "QEMU_PID=$QEMU_PID" >> $GITHUB_OUTPUT
 
       - name: Wait for SSH
         run: |
-          while ! nc -z localhost $SSH_PORT; do
+          echo "Waiting for SSH service to be ready..."
+          max_attempts=60  # Wait up to 10 minutes
+          attempt=0
+
+          while [ $attempt -lt $max_attempts ]; do
+            if nc -z localhost $SSH_PORT; then
+              echo "Port $SSH_PORT is open, waiting a bit more for SSH to be fully ready..."
+              sleep 10
+              # Test if we can actually connect with SSH
+              if ssh -i ~/.ssh/test_key -p $SSH_PORT -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o ConnectTimeout=5 -o LogLevel=ERROR aurora-ci-user@localhost "echo 'SSH connection test successful'" 2>/dev/null; then
+                echo "SSH connection successful!"
+                break
+              else
+                echo "Port open but SSH not ready yet, waiting..."
+              fi
+            else
+              echo "Waiting for SSH port to open... (attempt $((attempt + 1))/$max_attempts)"
+            fi
             sleep 10
+            attempt=$((attempt + 1))
           done
 
+          if [ $attempt -eq $max_attempts ]; then
+            echo "SSH service did not become ready within the timeout period"
+            exit 1
+          fi
+
       - name: Run Tests
         working-directory: ./tests
         env:
diff --git a/.github/workflows/nightly-integration-tests.yml b/.github/workflows/nightly-integration-tests.yml
index f84bbb17e..15d5937b1 100644
--- a/.github/workflows/nightly-integration-tests.yml
+++ b/.github/workflows/nightly-integration-tests.yml
@@ -20,17 +20,11 @@ jobs:
       fail-fast: false
       matrix:
         include:
-          - image_flavor: "main"
-            image_ref: "ghcr.io/${{ github.repository_owner }}/aurora:latest"
-          - image_flavor: "nvidia"
-            image_ref: "ghcr.io/${{ github.repository_owner }}/aurora-nvidia:latest"
-          - image_flavor: "nvidia-open"
-            image_ref: "ghcr.io/${{ github.repository_owner }}/aurora-nvidia-open:latest"
-          - image_flavor: "dx"
-            image_ref: "ghcr.io/${{ github.repository_owner }}/aurora-dx:latest"
-          - image_flavor: "dx-nvidia"
-            image_ref: "ghcr.io/${{ github.repository_owner }}/aurora-dx-nvidia:latest"
-          - image_flavor: "dx-nvidia-open"
-            image_ref: "ghcr.io/${{ github.repository_owner }}/aurora-dx-nvidia-open:latest"
+          - image_flavor: "aurora"
+          - image_flavor: "aurora-nvidia"
+          - image_flavor: "aurora-nvidia-open"
+          - image_flavor: "aurora-dx"
+          - image_flavor: "aurora-dx-nvidia"
+          - image_flavor: "aurora-dx-nvidia-open"
     with:
-      image-ref: ${{ matrix.image_ref }}
+      image-ref: ghcr.io/${{ github.repository_owner }}/${{ matrix.image_flavor }}:latest
diff --git a/tests/__fixtures__/qcow2-config.toml b/tests/__fixtures__/qcow2-config.toml
deleted file mode 100644
index fd9690357..000000000
--- a/tests/__fixtures__/qcow2-config.toml
+++ /dev/null
@@ -1,4 +0,0 @@
-[[customizations.user]]
-name = "aurora-ci-user"
-key = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIMnkmox0kV4ediDt7MsvHlZ3mncW6GIZzl8KgrJn+zbr aurora-ci-user@github-actions.com"
-groups = ["wheel"]

From 6852c3cd82f82177199c46248c3bd7f60ee99837 Mon Sep 17 00:00:00 2001
From: Adam Fidel <adam@fidel.id>
Date: Wed, 23 Jul 2025 08:27:55 -0500
Subject: [PATCH 10/14] add ssh debug

---
 .github/workflows/integration-test.yml | 56 ++++++++++++++++++++------
 tests/__fixtures__/qcow2-config.toml   |  3 ++
 2 files changed, 46 insertions(+), 13 deletions(-)
 create mode 100644 tests/__fixtures__/qcow2-config.toml

diff --git a/.github/workflows/integration-test.yml b/.github/workflows/integration-test.yml
index 5e8f386d0..a6f6550bd 100644
--- a/.github/workflows/integration-test.yml
+++ b/.github/workflows/integration-test.yml
@@ -23,7 +23,15 @@ jobs:
           cat ~/.ssh/test_key >> $GITHUB_ENV
           echo "EOF" >> $GITHUB_ENV
 
+          # Update config file with new public key
           PUBLIC_KEY=$(cat ~/.ssh/test_key.pub)
+          
+          echo "Generated SSH key:"
+          echo "Public key: $PUBLIC_KEY"
+          echo "Private key fingerprint:"
+          ssh-keygen -lf ~/.ssh/test_key
+          
+          # Create a new config file with the generated key
           cat > ./tests/qcow2-config.toml << EOF
           [[customizations.user]]
           name = "aurora-ci-user"
@@ -96,8 +104,11 @@ jobs:
           echo "Waiting for VM to boot (60 seconds)..."
           timeout 60 tail -f $LOG_FILE || true
 
-          echo "Current log contents:"
-          cat $LOG_FILE || echo "No log file available"
+          echo "Current log contents (last 50 lines):"
+          tail -50 $LOG_FILE || echo "No log file available"
+          
+          echo "Checking if we can see SSH-related messages in the log:"
+          grep -i ssh $LOG_FILE || echo "No SSH messages found in log"
 
           disown $QEMU_PID
           echo "QEMU_PID=$QEMU_PID" >> $GITHUB_OUTPUT
@@ -105,19 +116,35 @@ jobs:
       - name: Wait for SSH
         run: |
           echo "Waiting for SSH service to be ready..."
-          max_attempts=60  # Wait up to 10 minutes
+          max_attempts=120  # Wait up to 20 minutes
           attempt=0
-
+          
           while [ $attempt -lt $max_attempts ]; do
             if nc -z localhost $SSH_PORT; then
-              echo "Port $SSH_PORT is open, waiting a bit more for SSH to be fully ready..."
-              sleep 10
-              # Test if we can actually connect with SSH
-              if ssh -i ~/.ssh/test_key -p $SSH_PORT -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o ConnectTimeout=5 -o LogLevel=ERROR aurora-ci-user@localhost "echo 'SSH connection test successful'" 2>/dev/null; then
+              echo "Port $SSH_PORT is open, testing SSH connection..."
+              
+              # Try to get SSH version first
+              echo "Checking SSH service version:"
+              timeout 10 nc localhost $SSH_PORT || echo "Could not get SSH banner"
+              
+              # Try SSH connection with more debugging
+              if timeout 15 ssh -i ~/.ssh/test_key -p $SSH_PORT \
+                  -o StrictHostKeyChecking=no \
+                  -o UserKnownHostsFile=/dev/null \
+                  -o ConnectTimeout=10 \
+                  -o ServerAliveInterval=5 \
+                  -o ServerAliveCountMax=3 \
+                  -o BatchMode=yes \
+                  -v \
+                  aurora-ci-user@localhost "echo 'SSH connection test successful'" 2>&1; then
                 echo "SSH connection successful!"
                 break
               else
-                echo "Port open but SSH not ready yet, waiting..."
+                echo "SSH connection failed, checking what's listening on port $SSH_PORT..."
+                netstat -tlnp | grep ":$SSH_PORT " || echo "Nothing found listening on port $SSH_PORT"
+                echo "Attempting telnet test:"
+                timeout 5 telnet localhost $SSH_PORT || echo "Telnet failed"
+                echo "Port open but SSH not ready yet, waiting... (attempt $((attempt + 1))/$max_attempts)"
               fi
             else
               echo "Waiting for SSH port to open... (attempt $((attempt + 1))/$max_attempts)"
@@ -125,13 +152,16 @@ jobs:
             sleep 10
             attempt=$((attempt + 1))
           done
-
+          
           if [ $attempt -eq $max_attempts ]; then
             echo "SSH service did not become ready within the timeout period"
+            echo "Final diagnostic information:"
+            netstat -tlnp | grep ":$SSH_PORT " || echo "Nothing listening on port $SSH_PORT"
+            ps aux | grep ssh || echo "No SSH processes found"
+            echo "VM serial log (last 50 lines):"
+            tail -50 /tmp/qemu-serial.log || echo "No serial log available"
             exit 1
-          fi
-
-      - name: Run Tests
+          fi      - name: Run Tests
         working-directory: ./tests
         env:
           SSH_PRIVATE_KEY: ${{ env.SSH_PRIVATE_KEY }}
diff --git a/tests/__fixtures__/qcow2-config.toml b/tests/__fixtures__/qcow2-config.toml
new file mode 100644
index 000000000..6e2eaaeb4
--- /dev/null
+++ b/tests/__fixtures__/qcow2-config.toml
@@ -0,0 +1,3 @@
+[[customizations.user]]
+name = "aurora-ci-user"
+groups = ["wheel"]

From 8bab6d3a086c7058d592c384d98732eac59db3e4 Mon Sep 17 00:00:00 2001
From: Adam Fidel <adam@fidel.id>
Date: Wed, 23 Jul 2025 08:29:38 -0500
Subject: [PATCH 11/14] fix bad merge

---
 .github/workflows/integration-test.yml | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/integration-test.yml b/.github/workflows/integration-test.yml
index a6f6550bd..a6d6cdc87 100644
--- a/.github/workflows/integration-test.yml
+++ b/.github/workflows/integration-test.yml
@@ -161,7 +161,8 @@ jobs:
             echo "VM serial log (last 50 lines):"
             tail -50 /tmp/qemu-serial.log || echo "No serial log available"
             exit 1
-          fi      - name: Run Tests
+          fi
+      - name: Run Tests
         working-directory: ./tests
         env:
           SSH_PRIVATE_KEY: ${{ env.SSH_PRIVATE_KEY }}

From 5e38c64672b1527fd6d57d09689a18ca06dd724f Mon Sep 17 00:00:00 2001
From: Adam Fidel <adam@fidel.id>
Date: Wed, 23 Jul 2025 10:03:42 -0500
Subject: [PATCH 12/14] more ssh debug

---
 .github/workflows/integration-test.yml | 41 +++++++++++++++++++++-----
 1 file changed, 33 insertions(+), 8 deletions(-)

diff --git a/.github/workflows/integration-test.yml b/.github/workflows/integration-test.yml
index a6d6cdc87..8f64a81be 100644
--- a/.github/workflows/integration-test.yml
+++ b/.github/workflows/integration-test.yml
@@ -32,12 +32,15 @@ jobs:
           ssh-keygen -lf ~/.ssh/test_key
           
           # Create a new config file with the generated key
-          cat > ./tests/qcow2-config.toml << EOF
-          [[customizations.user]]
-          name = "aurora-ci-user"
-          key = "$PUBLIC_KEY"
-          groups = ["wheel"]
-          EOF
+          cat > ./tests/qcow2-config.toml << 'EOF'
+[[customizations.user]]
+name = "aurora-ci-user"
+password = "$6$rounds=4096$saltysalt$UiZikbV3VeeBPsg8./Q5DAfq9aj2.9.MTZ.gzcjF4jhfjsdljfdjsfdlksjfdf.LDL4IzYYqKxMf3eiexVyNs3."
+groups = ["wheel", "users"]
+home = "/home/aurora-ci-user"
+shell = "/bin/bash"
+EOF
+          echo "key = \"$PUBLIC_KEY\"" >> ./tests/qcow2-config.toml
 
           # Verify the update
           echo "Updated config file:"
@@ -65,7 +68,7 @@ jobs:
           ls -l /dev/kvm
 
           sudo apt-get update
-          sudo apt-get install -y qemu-system-x86
+          sudo apt-get install -y qemu-system-x86 sshpass
 
       - name: Run QEMU VM
         id: run-qemu
@@ -128,6 +131,7 @@ jobs:
               timeout 10 nc localhost $SSH_PORT || echo "Could not get SSH banner"
               
               # Try SSH connection with more debugging
+              echo "Attempting SSH connection..."
               if timeout 15 ssh -i ~/.ssh/test_key -p $SSH_PORT \
                   -o StrictHostKeyChecking=no \
                   -o UserKnownHostsFile=/dev/null \
@@ -135,13 +139,34 @@ jobs:
                   -o ServerAliveInterval=5 \
                   -o ServerAliveCountMax=3 \
                   -o BatchMode=yes \
-                  -v \
+                  -o LogLevel=DEBUG3 \
                   aurora-ci-user@localhost "echo 'SSH connection test successful'" 2>&1; then
                 echo "SSH connection successful!"
                 break
               else
                 echo "SSH connection failed, checking what's listening on port $SSH_PORT..."
                 netstat -tlnp | grep ":$SSH_PORT " || echo "Nothing found listening on port $SSH_PORT"
+                
+                echo "Let's check if we can see what's inside the VM via the serial console..."
+                echo "Looking for recent SSH logs in VM serial output:"
+                grep -i "ssh\|sshd\|authentication\|login\|user" /tmp/qemu-serial.log | tail -10 || echo "No SSH-related messages in serial log"
+                
+                echo "Let's try connecting as root to see if that works:"
+                timeout 10 ssh -i ~/.ssh/test_key -p $SSH_PORT \
+                  -o StrictHostKeyChecking=no \
+                  -o UserKnownHostsFile=/dev/null \
+                  -o ConnectTimeout=5 \
+                  -o BatchMode=yes \
+                  root@localhost "whoami" 2>&1 || echo "Root connection also failed"
+                
+                echo "Let's try with password authentication (password is 'testpass'):"
+                timeout 10 sshpass -p 'testpass' ssh -p $SSH_PORT \
+                  -o StrictHostKeyChecking=no \
+                  -o UserKnownHostsFile=/dev/null \
+                  -o ConnectTimeout=5 \
+                  -o PreferredAuthentications=password \
+                  aurora-ci-user@localhost "whoami" 2>&1 || echo "Password auth also failed"
+                
                 echo "Attempting telnet test:"
                 timeout 5 telnet localhost $SSH_PORT || echo "Telnet failed"
                 echo "Port open but SSH not ready yet, waiting... (attempt $((attempt + 1))/$max_attempts)"

From 90b8dfe9503a77d92585d8b4a9c96ef4f8fc2b12 Mon Sep 17 00:00:00 2001
From: Adam Fidel <adam@fidel.id>
Date: Wed, 23 Jul 2025 11:29:12 -0500
Subject: [PATCH 13/14] move config file out

---
 .github/workflows/integration-test.yml | 12 ++----------
 tests/qcow2-config.toml                |  6 ++++++
 2 files changed, 8 insertions(+), 10 deletions(-)
 create mode 100644 tests/qcow2-config.toml

diff --git a/.github/workflows/integration-test.yml b/.github/workflows/integration-test.yml
index 8f64a81be..1dc9dbaec 100644
--- a/.github/workflows/integration-test.yml
+++ b/.github/workflows/integration-test.yml
@@ -31,18 +31,9 @@ jobs:
           echo "Private key fingerprint:"
           ssh-keygen -lf ~/.ssh/test_key
           
-          # Create a new config file with the generated key
-          cat > ./tests/qcow2-config.toml << 'EOF'
-[[customizations.user]]
-name = "aurora-ci-user"
-password = "$6$rounds=4096$saltysalt$UiZikbV3VeeBPsg8./Q5DAfq9aj2.9.MTZ.gzcjF4jhfjsdljfdjsfdlksjfdf.LDL4IzYYqKxMf3eiexVyNs3."
-groups = ["wheel", "users"]
-home = "/home/aurora-ci-user"
-shell = "/bin/bash"
-EOF
+          # Append the SSH key to the existing config file
           echo "key = \"$PUBLIC_KEY\"" >> ./tests/qcow2-config.toml
 
-          # Verify the update
           echo "Updated config file:"
           cat ./tests/qcow2-config.toml
 
@@ -187,6 +178,7 @@ EOF
             tail -50 /tmp/qemu-serial.log || echo "No serial log available"
             exit 1
           fi
+
       - name: Run Tests
         working-directory: ./tests
         env:
diff --git a/tests/qcow2-config.toml b/tests/qcow2-config.toml
new file mode 100644
index 000000000..72d81b82c
--- /dev/null
+++ b/tests/qcow2-config.toml
@@ -0,0 +1,6 @@
+[[customizations.user]]
+name = "aurora-ci-user"
+password = "$6$rounds=4096$saltysalt$UiZikbV3VeeBPsg8./Q5DAfq9aj2.9.MTZ.gzcjF4jhfjsdljfdjsfdlksjfdf.LDL4IzYYqKxMf3eiexVyNs3."
+groups = ["wheel", "users"]
+home = "/home/aurora-ci-user"
+shell = "/bin/bash"

From d71cb9531962952af6ed0dc7be7c9ffffb3f795b Mon Sep 17 00:00:00 2001
From: Adam Fidel <adam@fidel.id>
Date: Wed, 23 Jul 2025 19:18:01 -0500
Subject: [PATCH 14/14] remove users group

---
 tests/qcow2-config.toml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/qcow2-config.toml b/tests/qcow2-config.toml
index 72d81b82c..c54d35b62 100644
--- a/tests/qcow2-config.toml
+++ b/tests/qcow2-config.toml
@@ -1,6 +1,6 @@
 [[customizations.user]]
 name = "aurora-ci-user"
 password = "$6$rounds=4096$saltysalt$UiZikbV3VeeBPsg8./Q5DAfq9aj2.9.MTZ.gzcjF4jhfjsdljfdjsfdlksjfdf.LDL4IzYYqKxMf3eiexVyNs3."
-groups = ["wheel", "users"]
+groups = ["wheel"]
 home = "/home/aurora-ci-user"
 shell = "/bin/bash"