diff --git a/.github/workflows/build-image-stable.yml b/.github/workflows/build-image-stable.yml index aa7f74f80..0396b39b4 100644 --- a/.github/workflows/build-image-stable.yml +++ b/.github/workflows/build-image-stable.yml @@ -25,11 +25,12 @@ jobs: fail-fast: false matrix: brand_name: ["aurora"] + with: # kernel_pin: 6.13.8-200.fc41.x86_64 ## This is where kernels get pinned. brand_name: ${{ matrix.brand_name }} stream_name: stable - + generate-release: name: Generate Release needs: [build-image-stable] diff --git a/.github/workflows/integration-test.yml b/.github/workflows/integration-test.yml new file mode 100644 index 000000000..1dc9dbaec --- /dev/null +++ b/.github/workflows/integration-test.yml @@ -0,0 +1,188 @@ +name: Integration Tests + +on: + workflow_call: + inputs: + image-ref: + required: true + type: string + +jobs: + integration-test: + runs-on: ubuntu-24.04 + env: + SSH_PORT: 2222 + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Generate SSH Key Pair + run: | + ssh-keygen -t ed25519 -f ~/.ssh/test_key -N "" -C "aurora-ci-user@github-actions.com" + echo "SSH_PRIVATE_KEY<> $GITHUB_ENV + cat ~/.ssh/test_key >> $GITHUB_ENV + echo "EOF" >> $GITHUB_ENV + + # Update config file with new public key + PUBLIC_KEY=$(cat ~/.ssh/test_key.pub) + + echo "Generated SSH key:" + echo "Public key: $PUBLIC_KEY" + echo "Private key fingerprint:" + ssh-keygen -lf ~/.ssh/test_key + + # Append the SSH key to the existing config file + echo "key = \"$PUBLIC_KEY\"" >> ./tests/qcow2-config.toml + + echo "Updated config file:" + cat ./tests/qcow2-config.toml + + - name: Maximize build space + uses: ublue-os/remove-unwanted-software@cc0becac701cf642c8f0a6613bbdaf5dc36b259e # v9 + with: + remove-codeql: true + + - name: Build QCOW Image + id: build + uses: osbuild/bootc-image-builder-action@v0.0.2 + with: + config-file: ./tests/qcow2-config.toml + image: ${{ inputs.image-ref }} + types: | + qcow2 + + - name: Setup KVM + run: | + echo 'KERNEL=="kvm", GROUP="kvm", MODE="0666", OPTIONS+="static_node=kvm"' | sudo tee /etc/udev/rules.d/99-kvm.rules + sudo udevadm control --reload-rules + sudo udevadm trigger --name-match=kvm --settle + ls -l /dev/kvm + + sudo apt-get update + sudo apt-get install -y qemu-system-x86 sshpass + + - name: Run QEMU VM + id: run-qemu + env: + QCOW2_PATH: ${{ steps.build.outputs.qcow2-output-path }} + run: | + LOG_FILE="/tmp/qemu-serial.log" + + qemu-system-x86_64 \ + -enable-kvm \ + -cpu host \ + -smp 2 \ + -display none \ + -m 8192 \ + -device virtio-rng-pci \ + -serial file:$LOG_FILE \ + -drive file=$QCOW2_PATH,if=virtio \ + -snapshot \ + -net nic,model=virtio \ + -net user,hostfwd=tcp::$SSH_PORT-:22 & + QEMU_PID=$! + + echo "QEMU started with PID: $QEMU_PID" + + # Wait for log file to appear (max 30s) + for i in {1..30}; do + [ -f "$LOG_FILE" ] && break + sleep 1 + done + + if [ ! -f "$LOG_FILE" ]; then + echo "Warning: Log file did not appear" + fi + + # Wait 60 seconds and show log + echo "Waiting for VM to boot (60 seconds)..." + timeout 60 tail -f $LOG_FILE || true + + echo "Current log contents (last 50 lines):" + tail -50 $LOG_FILE || echo "No log file available" + + echo "Checking if we can see SSH-related messages in the log:" + grep -i ssh $LOG_FILE || echo "No SSH messages found in log" + + disown $QEMU_PID + echo "QEMU_PID=$QEMU_PID" >> $GITHUB_OUTPUT + + - name: Wait for SSH + run: | + echo "Waiting for SSH service to be ready..." + max_attempts=120 # Wait up to 20 minutes + attempt=0 + + while [ $attempt -lt $max_attempts ]; do + if nc -z localhost $SSH_PORT; then + echo "Port $SSH_PORT is open, testing SSH connection..." + + # Try to get SSH version first + echo "Checking SSH service version:" + timeout 10 nc localhost $SSH_PORT || echo "Could not get SSH banner" + + # Try SSH connection with more debugging + echo "Attempting SSH connection..." + if timeout 15 ssh -i ~/.ssh/test_key -p $SSH_PORT \ + -o StrictHostKeyChecking=no \ + -o UserKnownHostsFile=/dev/null \ + -o ConnectTimeout=10 \ + -o ServerAliveInterval=5 \ + -o ServerAliveCountMax=3 \ + -o BatchMode=yes \ + -o LogLevel=DEBUG3 \ + aurora-ci-user@localhost "echo 'SSH connection test successful'" 2>&1; then + echo "SSH connection successful!" + break + else + echo "SSH connection failed, checking what's listening on port $SSH_PORT..." + netstat -tlnp | grep ":$SSH_PORT " || echo "Nothing found listening on port $SSH_PORT" + + echo "Let's check if we can see what's inside the VM via the serial console..." + echo "Looking for recent SSH logs in VM serial output:" + grep -i "ssh\|sshd\|authentication\|login\|user" /tmp/qemu-serial.log | tail -10 || echo "No SSH-related messages in serial log" + + echo "Let's try connecting as root to see if that works:" + timeout 10 ssh -i ~/.ssh/test_key -p $SSH_PORT \ + -o StrictHostKeyChecking=no \ + -o UserKnownHostsFile=/dev/null \ + -o ConnectTimeout=5 \ + -o BatchMode=yes \ + root@localhost "whoami" 2>&1 || echo "Root connection also failed" + + echo "Let's try with password authentication (password is 'testpass'):" + timeout 10 sshpass -p 'testpass' ssh -p $SSH_PORT \ + -o StrictHostKeyChecking=no \ + -o UserKnownHostsFile=/dev/null \ + -o ConnectTimeout=5 \ + -o PreferredAuthentications=password \ + aurora-ci-user@localhost "whoami" 2>&1 || echo "Password auth also failed" + + echo "Attempting telnet test:" + timeout 5 telnet localhost $SSH_PORT || echo "Telnet failed" + echo "Port open but SSH not ready yet, waiting... (attempt $((attempt + 1))/$max_attempts)" + fi + else + echo "Waiting for SSH port to open... (attempt $((attempt + 1))/$max_attempts)" + fi + sleep 10 + attempt=$((attempt + 1)) + done + + if [ $attempt -eq $max_attempts ]; then + echo "SSH service did not become ready within the timeout period" + echo "Final diagnostic information:" + netstat -tlnp | grep ":$SSH_PORT " || echo "Nothing listening on port $SSH_PORT" + ps aux | grep ssh || echo "No SSH processes found" + echo "VM serial log (last 50 lines):" + tail -50 /tmp/qemu-serial.log || echo "No serial log available" + exit 1 + fi + + - name: Run Tests + working-directory: ./tests + env: + SSH_PRIVATE_KEY: ${{ env.SSH_PRIVATE_KEY }} + SSH_PORT: ${{ env.SSH_PORT }} + run: | + python3 ./qemu_test.py diff --git a/.github/workflows/nightly-integration-tests.yml b/.github/workflows/nightly-integration-tests.yml new file mode 100644 index 000000000..15d5937b1 --- /dev/null +++ b/.github/workflows/nightly-integration-tests.yml @@ -0,0 +1,30 @@ +name: Nightly Integration Tests + +on: + schedule: + - cron: "30 6 * * *" # 6:30 UTC daily + workflow_dispatch: + pull_request: + branches: + - main + paths: + - ".github/workflows/integration-test.yml" + - ".github/workflows/nightly-integration-tests.yml" + - "tests/**" + +jobs: + integration-test: + name: Integration Test + uses: ./.github/workflows/integration-test.yml + strategy: + fail-fast: false + matrix: + include: + - image_flavor: "aurora" + - image_flavor: "aurora-nvidia" + - image_flavor: "aurora-nvidia-open" + - image_flavor: "aurora-dx" + - image_flavor: "aurora-dx-nvidia" + - image_flavor: "aurora-dx-nvidia-open" + with: + image-ref: ghcr.io/${{ github.repository_owner }}/${{ matrix.image_flavor }}:latest diff --git a/tests/__fixtures__/qcow2-config.toml b/tests/__fixtures__/qcow2-config.toml new file mode 100644 index 000000000..6e2eaaeb4 --- /dev/null +++ b/tests/__fixtures__/qcow2-config.toml @@ -0,0 +1,3 @@ +[[customizations.user]] +name = "aurora-ci-user" +groups = ["wheel"] diff --git a/tests/qcow2-config.toml b/tests/qcow2-config.toml new file mode 100644 index 000000000..c54d35b62 --- /dev/null +++ b/tests/qcow2-config.toml @@ -0,0 +1,6 @@ +[[customizations.user]] +name = "aurora-ci-user" +password = "$6$rounds=4096$saltysalt$UiZikbV3VeeBPsg8./Q5DAfq9aj2.9.MTZ.gzcjF4jhfjsdljfdjsfdlksjfdf.LDL4IzYYqKxMf3eiexVyNs3." +groups = ["wheel"] +home = "/home/aurora-ci-user" +shell = "/bin/bash" diff --git a/tests/qemu_test.py b/tests/qemu_test.py new file mode 100644 index 000000000..10a77e238 --- /dev/null +++ b/tests/qemu_test.py @@ -0,0 +1,128 @@ +#!/usr/bin/env python3 +""" +Integration tests for Aurora bootc image. +Tests basic functionality by connecting to a VM via SSH. +""" + +import os +import sys +import time +import subprocess +import tempfile +from typing import Tuple + + +class SSHClient: + def __init__(self, hostname: str, port: int, username: str, private_key: str): + self.hostname = hostname + self.port = port + self.username = username + + # Write private key to temporary file + self.key_file = tempfile.NamedTemporaryFile(mode='w', delete=False, suffix='.pem') + self.key_file.write(private_key) + self.key_file.close() + os.chmod(self.key_file.name, 0o600) + + def __del__(self): + if hasattr(self, 'key_file'): + try: + os.unlink(self.key_file.name) + except FileNotFoundError: + pass + + def run_command(self, command: str, timeout: int = 30) -> Tuple[int, str, str]: + """Run a command via SSH and return (returncode, stdout, stderr)""" + ssh_cmd = [ + 'ssh', + '-i', self.key_file.name, + '-p', str(self.port), + '-o', 'StrictHostKeyChecking=no', + '-o', 'UserKnownHostsFile=/dev/null', + '-o', 'LogLevel=ERROR', + f'{self.username}@{self.hostname}', + command + ] + + try: + result = subprocess.run( + ssh_cmd, + capture_output=True, + text=True, + timeout=timeout + ) + return result.returncode, result.stdout, result.stderr + except subprocess.TimeoutExpired: + return 124, "", f"Command timed out after {timeout} seconds" + + +def run_test(ssh_client: SSHClient, test_name: str, command: str, expected_text: str, max_retries: int = 3) -> bool: + """Run a single test with retries""" + print(f"Running test: {test_name}") + + for attempt in range(max_retries): + if attempt > 0: + print(f" Retry {attempt}/{max_retries - 1}") + time.sleep(5) + + returncode, stdout, stderr = ssh_client.run_command(command) + + if returncode != 0: + print(f" Command failed with exit code {returncode}") + print(f" stdout: {stdout.strip()}") + print(f" stderr: {stderr.strip()}") + continue + + if expected_text in stdout: + print("PASS") + return True + else: + print(f"Expected '{expected_text}' not found in output") + print(f"stdout: {stdout.strip()}") + + print(f"FAIL after {max_retries} attempts") + return False + + +def main(): + # Get environment variables + ssh_private_key = os.getenv('SSH_PRIVATE_KEY') + ssh_port = int(os.getenv('SSH_PORT', '2222')) + + if not ssh_private_key: + print("ERROR: SSH_PRIVATE_KEY environment variable not set") + sys.exit(1) + + # Create SSH client + ssh_client = SSHClient('localhost', ssh_port, 'aurora-ci-user', ssh_private_key) + + # Define tests + tests = [ + ("KernelInstalled", "rpm -q kernel", "kernel"), + ("CheckSELinuxStatus", "getenforce", "Enforcing"), + ("CheckSudoersValid", "sudo visudo -cf /etc/sudoers", "/etc/sudoers: parsed OK"), + ("CheckNftablesServiceEnabled", "systemctl is-enabled nftables", "enabled"), + ("CheckNftablesServiceActive", "systemctl is-active nftables", "active"), + ] + + print("Starting Aurora integration tests...") + + passed = 0 + failed = 0 + + for test_name, command, expected_text in tests: + if run_test(ssh_client, test_name, command, expected_text): + passed += 1 + else: + failed += 1 + + print(f"\nResults: {passed} passed, {failed} failed") + + if failed > 0: + sys.exit(1) + else: + print("All tests passed!") + + +if __name__ == "__main__": + main()