|
| 1 | +#!/bin/bash |
| 2 | + |
| 3 | +# Copyright (C) 2026 Brad House <bhouse@nexthop.ai> |
| 4 | +# |
| 5 | +# SPDX-License-Identifier: GPL-2.0 |
| 6 | + |
| 7 | +# |
| 8 | +# OCE (ONIE Compliance Environment) CI harness. |
| 9 | +# |
| 10 | +# Drives ONIE's own contrib/oce/test-onie.py compliance suite against the |
| 11 | +# kvm_x86_64 image, headless under QEMU. Where install-test hands ONIE the |
| 12 | +# installer URL directly (install_url=), this exercises the REAL OCP discovery |
| 13 | +# paths: for each selected OCE test, OCE generates the matching server set |
| 14 | +# (isc-dhcp-server with the ONIE VIVSO option, nginx, tftpd-hpa, dnsmasq) and |
| 15 | +# ONIE must discover + fetch the installer itself. USB tests instead attach a |
| 16 | +# USB mass-storage image carrying the installer. |
| 17 | +# |
| 18 | +# Each test runs a FULL install: ONIE discovers via the method, fetches the |
| 19 | +# installer, and installs it -- the harness asserts "ONIE: NOS install |
| 20 | +# successful". To avoid re-embedding per test, ONIE is embedded onto a base |
| 21 | +# disk ONCE up front; each test gets a fresh copy-on-write qcow2 overlay of that |
| 22 | +# base (instant reset) as its install target. |
| 23 | +# |
| 24 | +# Tests run CHAINED (one VM at a time), ordered fast->slow, and the run STOPS on |
| 25 | +# the first failure (fail-fast). Each test is capped by a timeout so a method |
| 26 | +# that never installs fails in bounded time instead of looping ONIE's |
| 27 | +# retry/sleep waterfall forever. |
| 28 | +# |
| 29 | +# Usage: ci-oce-test.sh <vmlinuz> <initrd> <demo-installer> <onie-updater> [scope] |
| 30 | +# scope: "default" (broad per-method installer+updater set, the per-PR sweep), |
| 31 | +# "full" (every feasible test 3-121), or an explicit space/comma list |
| 32 | +# of test numbers. Default: "default". |
| 33 | +# Env: OCE_DIR=<contrib/oce> OCE_PYTHON=<python3 w/ oce deps> |
| 34 | +# SERIAL_PREFIX=<path-prefix> WORKDIR=<dir> TAP=<ifname> |
| 35 | +# PER_TEST_TIMEOUT=<secs, default 180> |
| 36 | +# |
| 37 | +# Requires (install in the workflow): qemu-system-x86_64, qemu-utils, dosfstools, |
| 38 | +# isc-dhcp-server, nginx, tftpd-hpa, dnsmasq, and python3 with |
| 39 | +# jinja2/netifaces/psutil. Needs sudo. ONIE is serial end-to-end -> no OCR. |
| 40 | + |
| 41 | +set -uo pipefail |
| 42 | + |
| 43 | +VMLINUZ="${1:?usage: ci-oce-test.sh <vmlinuz> <initrd> <demo-installer> <onie-updater> [scope]}" |
| 44 | +INITRD="${2:?need initrd}" |
| 45 | +INSTALLER="${3:?need demo-installer.bin}" |
| 46 | +UPDATER="${4:?need onie-updater}" |
| 47 | +SCOPE="${5:-default}" |
| 48 | +for f in "$VMLINUZ" "$INITRD" "$INSTALLER" "$UPDATER"; do |
| 49 | + [ -r "$f" ] || { echo "ERROR: not readable: $f" >&2; exit 2; } |
| 50 | +done |
| 51 | +VMLINUZ="$(realpath "$VMLINUZ")"; INITRD="$(realpath "$INITRD")" |
| 52 | +INSTALLER="$(realpath "$INSTALLER")"; UPDATER="$(realpath "$UPDATER")" |
| 53 | + |
| 54 | +OCE_DIR="${OCE_DIR:-$(cd "$(dirname "$0")/../contrib/oce" 2>/dev/null && pwd)}" |
| 55 | +OCE_PYTHON="${OCE_PYTHON:-python3}" |
| 56 | +[ -r "$OCE_DIR/test-onie.py" ] || { echo "ERROR: OCE not found at OCE_DIR=$OCE_DIR" >&2; exit 2; } |
| 57 | + |
| 58 | +WORKDIR="${WORKDIR:-$(mktemp -d)}"; mkdir -p "$WORKDIR" |
| 59 | +SERIAL_PREFIX="${SERIAL_PREFIX:-$WORKDIR/oce}"; mkdir -p "$(dirname "$SERIAL_PREFIX")" |
| 60 | +TAP="${TAP:-onie-oce0}" |
| 61 | +HOST_V4="192.168.1.1"; DUT_V4="192.168.1.100"; CIDR="24"; DUT_MAC="52:54:00:0c:e0:01" |
| 62 | +PER_TEST_TIMEOUT="${PER_TEST_TIMEOUT:-180}" |
| 63 | +EMBED_PORT=8920 |
| 64 | +ACCEL="accel=kvm:tcg"; CONSOLE="console=tty0 console=ttyS0,115200n8" |
| 65 | +BASE_DISK="$WORKDIR/onie-base.qcow2" |
| 66 | + |
| 67 | +# ---- test sets (ordered fast->slow: USB/DHCP-direct, then DHCP+server, then no-DHCP/fallback) ---- |
| 68 | +DEFAULT_TESTS="3 9 10 69 70 11 12 13 14 20 26 32 71 72 73 74 80 86 92 44 50 56 104 110 116" |
| 69 | +FULL_TESTS="$(seq 3 121)" |
| 70 | +case "$SCOPE" in |
| 71 | + default) TESTS="$DEFAULT_TESTS" ;; |
| 72 | + full) TESTS="$FULL_TESTS" ;; |
| 73 | + *) TESTS="$(echo "$SCOPE" | tr ',' ' ')" ;; |
| 74 | +esac |
| 75 | + |
| 76 | +echo "== ONIE OCE compliance sweep (full install) ==" |
| 77 | +echo " scope : $SCOPE ($(echo $TESTS | wc -w) tests)" |
| 78 | +echo " tap iface : $TAP ($HOST_V4/$CIDR, DUT $DUT_V4 mac $DUT_MAC)" |
| 79 | +echo " accel : kvm:tcg ($([ -w /dev/kvm ] && echo KVM || echo TCG))" |
| 80 | +echo " per-test : ${PER_TEST_TIMEOUT}s cap" |
| 81 | + |
| 82 | +stop_services() { |
| 83 | + sudo pkill -f "dhcpd.leases.oce" 2>/dev/null |
| 84 | + sudo pkill -f "nginx.*$WORKDIR" 2>/dev/null |
| 85 | + sudo pkill -x in.tftpd 2>/dev/null |
| 86 | + sudo pkill -f "dnsmasq.*$WORKDIR" 2>/dev/null |
| 87 | + true |
| 88 | +} |
| 89 | +cleanup() { |
| 90 | + stop_services |
| 91 | + sudo ip link del "$TAP" 2>/dev/null |
| 92 | + [ -n "${KEEP:-}" ] || rm -rf "$WORKDIR" |
| 93 | +} |
| 94 | +trap cleanup EXIT |
| 95 | + |
| 96 | +# ---- poll a serial log until a marker (regex) or panic or timeout; kill QEMU; echo elapsed ---- |
| 97 | +wait_marker() { # <log> <success-regex> <timeout> |
| 98 | + local log="$1" ok_re="$2" tmo="$3" qpid=$! t=0 |
| 99 | + while kill -0 "$qpid" 2>/dev/null; do |
| 100 | + grep -Eaq "$ok_re" "$log" && break |
| 101 | + grep -aq 'Kernel panic' "$log" && break |
| 102 | + [ "$t" -ge "$tmo" ] && break |
| 103 | + sleep 1; t=$((t + 1)) |
| 104 | + done |
| 105 | + kill "$qpid" 2>/dev/null; sleep 1; kill -9 "$qpid" 2>/dev/null; wait "$qpid" 2>/dev/null |
| 106 | + echo "$t" |
| 107 | +} |
| 108 | + |
| 109 | +# ---- one-time setup: stop conflicting services, set up the tap ---- |
| 110 | +echo "== setup ==" |
| 111 | +sudo systemctl stop nginx isc-dhcp-server dnsmasq tftpd-hpa 2>/dev/null |
| 112 | +sudo pkill -x nginx 2>/dev/null; sudo pkill -x dnsmasq 2>/dev/null; sudo pkill -x dhcpd 2>/dev/null |
| 113 | +sudo ip link del "$TAP" 2>/dev/null |
| 114 | +sudo ip tuntap add dev "$TAP" mode tap user "$(id -un)" |
| 115 | +sudo ip addr add "$HOST_V4/$CIDR" dev "$TAP" |
| 116 | +sudo ip -6 addr add fd00:ce::1/64 dev "$TAP" 2>/dev/null |
| 117 | +sudo ip link set "$TAP" up |
| 118 | + |
| 119 | +# ---- embed ONIE onto the base disk ONCE (reused via overlays); via SLIRP + a temp HTTP updater ---- |
| 120 | +echo "== embed ONIE onto base disk (one-time) ==" |
| 121 | +qemu-img create -f qcow2 "$BASE_DISK" 4G >/dev/null 2>&1 |
| 122 | +python3 -m http.server "$EMBED_PORT" --directory "$(dirname "$UPDATER")" >"$WORKDIR/embed-http.log" 2>&1 & |
| 123 | +EHP=$!; sleep 1 |
| 124 | +: > "$WORKDIR/embed.log" |
| 125 | +qemu-system-x86_64 \ |
| 126 | + -machine pc,${ACCEL} -m 2048 -smp 2 \ |
| 127 | + -kernel "$VMLINUZ" -initrd "$INITRD" \ |
| 128 | + -append "quiet ${CONSOLE} boot_env=recovery boot_reason=embed install_url=http://10.0.2.2:${EMBED_PORT}/$(basename "$UPDATER")" \ |
| 129 | + -drive file="$BASE_DISK",if=virtio,format=qcow2 \ |
| 130 | + -netdev user,id=n0 -device virtio-net,netdev=n0 \ |
| 131 | + -display none -serial "file:$WORKDIR/embed.log" -no-reboot >/dev/null 2>&1 & |
| 132 | +esecs="$(wait_marker "$WORKDIR/embed.log" 'ONIE: NOS install successful|ONIE: Rebooting' 240)" |
| 133 | +kill "$EHP" 2>/dev/null |
| 134 | +if ! grep -aqE 'ONIE: NOS install successful|ONIE: Rebooting|Installing ONIE on' "$WORKDIR/embed.log"; then |
| 135 | + echo "ERROR: embed failed (${esecs}s) -- cannot run install tests" |
| 136 | + sed 's/\x1b\[[0-9;?]*[A-Za-z]//g; s/\r/\n/g' "$WORKDIR/embed.log" | grep -avE '^\s*$' | tail -20 |
| 137 | + exit 1 |
| 138 | +fi |
| 139 | +echo " embedded in ${esecs}s -> $BASE_DISK (cached; tests overlay this)" |
| 140 | + |
| 141 | +# ---- build a vfat USB image carrying the installer under ONIE's default names ---- |
| 142 | +build_usb_image() { # <out.img> |
| 143 | + local img="$1" |
| 144 | + # The vfat must hold the installer under all 4 names below; size it to fit |
| 145 | + # (installer size x4) plus slack for FAT overhead, so a >16MB installer does |
| 146 | + # not overflow a fixed 64MB image partway through the copies. |
| 147 | + local mb=$(( ($(stat -c%s "$INSTALLER") / 1048576 + 1) * 4 + 32 )) |
| 148 | + qemu-img create -f raw "$img" "${mb}M" >/dev/null 2>&1 |
| 149 | + mkfs.vfat -n ONIE-USB "$img" >/dev/null 2>&1 |
| 150 | + local mnt; mnt="$(mktemp -d)" |
| 151 | + sudo mount -o loop "$img" "$mnt" |
| 152 | + for n in onie-installer-x86_64-kvm_x86_64-r0 onie-installer-kvm_x86_64 \ |
| 153 | + onie-installer-x86_64 onie-installer; do |
| 154 | + sudo cp "$INSTALLER" "$mnt/$n" |
| 155 | + done |
| 156 | + sudo sync; sudo umount "$mnt"; rmdir "$mnt" |
| 157 | +} |
| 158 | + |
| 159 | +# ---- run one OCE test (full install); return 0 on "NOS install successful" ---- |
| 160 | +run_one() { # <test_num> |
| 161 | + local t="$1" |
| 162 | + local tdir="$WORKDIR/oce-out/test-$t" |
| 163 | + local log="${SERIAL_PREFIX}-test${t}.log" |
| 164 | + local disk="$WORKDIR/target-$t.qcow2" |
| 165 | + rm -rf "$tdir"; : > "$log" |
| 166 | + stop_services; sleep 1 |
| 167 | + |
| 168 | + # installer tests = 3-61, updater tests = 63-121 (onie-tests.json). An |
| 169 | + # updater is discovered/run only under boot_reason=update and reports a |
| 170 | + # different success line than an installer. |
| 171 | + local reason ok_re |
| 172 | + if [ "$t" -ge 62 ]; then reason=update; ok_re='ONIE: Success: Firmware update|Firmware update install successful' |
| 173 | + else reason=install; ok_re='ONIE: NOS install successful'; fi |
| 174 | + |
| 175 | + # fresh CoW overlay of the embedded base = this test's install target (instant reset) |
| 176 | + qemu-img create -f qcow2 -b "$BASE_DISK" -F qcow2 "$disk" >/dev/null 2>&1 |
| 177 | + |
| 178 | + ( cd "$OCE_DIR" && "$OCE_PYTHON" test-onie.py -D \ |
| 179 | + -I "$TAP" -m "$DUT_MAC" -i "$DUT_V4/$CIDR" -t "$t" \ |
| 180 | + -o onie_installer "$INSTALLER" onie_updater "$UPDATER" \ |
| 181 | + onie_arch x86_64 onie_vendor kvm onie_machine x86_64 onie_machine_rev 0 \ |
| 182 | + onie_switch_asic qemu \ |
| 183 | + -O "$WORKDIR/oce-out" ) >"$WORKDIR/oce-gen-$t.log" 2>&1 |
| 184 | + |
| 185 | + chmod -R o+rX "$WORKDIR" 2>/dev/null # nginx/tftpd/dnsmasq workers run as nobody/tftp |
| 186 | + |
| 187 | + local is_usb=no; local -a qemu_src |
| 188 | + if [ -f "$tdir/dhcpd.conf" ] || [ -f "$tdir/nginx.conf" ] || [ -f "$tdir/tftp.sh" ] || [ -f "$tdir/dnsmasq.conf" ]; then |
| 189 | + if [ -f "$tdir/dhcpd.conf" ]; then # dhcpd is AppArmor-confined -> use allowed paths |
| 190 | + sudo cp "$tdir/dhcpd.conf" /etc/dhcp/oce-ci.conf |
| 191 | + sudo sh -c ': > /var/lib/dhcp/dhcpd.leases.oce' |
| 192 | + sudo dhcpd -f --no-pid -cf /etc/dhcp/oce-ci.conf -lf /var/lib/dhcp/dhcpd.leases.oce "$TAP" \ |
| 193 | + >"$WORKDIR/dhcpd-$t.out" 2>&1 & |
| 194 | + fi |
| 195 | + [ -f "$tdir/nginx.conf" ] && sudo nginx -c "$tdir/nginx.conf" >"$WORKDIR/nginx-$t.out" 2>&1 & |
| 196 | + [ -f "$tdir/tftp.sh" ] && ( cd "$tdir" && sudo bash tftp.sh >"$WORKDIR/tftp-$t.out" 2>&1 & ) |
| 197 | + [ -f "$tdir/dnsmasq.conf" ] && sudo dnsmasq -k -C "$tdir/dnsmasq.conf" >"$WORKDIR/dnsmasq-$t.out" 2>&1 & |
| 198 | + sleep 3 |
| 199 | + qemu_src=(-netdev tap,id=n0,ifname="$TAP",script=no,downscript=no |
| 200 | + -device virtio-net,netdev=n0,mac="$DUT_MAC") |
| 201 | + else |
| 202 | + is_usb=yes |
| 203 | + build_usb_image "$WORKDIR/usb-$t.img" |
| 204 | + qemu_src=(-device qemu-xhci -device usb-storage,drive=ud |
| 205 | + -drive id=ud,if=none,format=raw,file="$WORKDIR/usb-$t.img" |
| 206 | + -netdev user,id=n0 -device virtio-net,netdev=n0) |
| 207 | + fi |
| 208 | + |
| 209 | + qemu-system-x86_64 \ |
| 210 | + -machine pc,${ACCEL} -m 2048 -smp 2 \ |
| 211 | + -kernel "$VMLINUZ" -initrd "$INITRD" \ |
| 212 | + -append "quiet ${CONSOLE} boot_env=recovery boot_reason=${reason}" \ |
| 213 | + -drive file="$disk",if=virtio,format=qcow2 \ |
| 214 | + "${qemu_src[@]}" \ |
| 215 | + -display none -serial "file:$log" -no-reboot >/dev/null 2>&1 & |
| 216 | + local secs; secs="$(wait_marker "$log" "$ok_re" "$PER_TEST_TIMEOUT")" |
| 217 | + stop_services |
| 218 | + rm -f "$disk" |
| 219 | + |
| 220 | + local kind; kind="$([ "$is_usb" = yes ] && echo USB || echo net)" |
| 221 | + if grep -Eaq "$ok_re" "$log"; then |
| 222 | + printf ' PASS test %-3s [%s/%s] %ss ok\n' "$t" "$kind" "$reason" "$secs" |
| 223 | + return 0 |
| 224 | + fi |
| 225 | + printf ' FAIL test %-3s [%s/%s] %ss did NOT reach "%s"\n' "$t" "$kind" "$reason" "$secs" "$ok_re" |
| 226 | + echo "---- serial tail (test $t) ----" |
| 227 | + sed 's/\x1b\[[0-9;?]*[A-Za-z]//g; s/\x1b[=>]//g; s/\r/\n/g' "$log" 2>/dev/null | grep -avE '^\s*$' | tail -25 |
| 228 | + return 1 |
| 229 | +} |
| 230 | + |
| 231 | +# ---- chained, fail-fast ---- |
| 232 | +echo "== running '$SCOPE' sweep ($(echo $TESTS | wc -w) tests), full install, fail-fast ==" |
| 233 | +pass=0 |
| 234 | +for t in $TESTS; do |
| 235 | + if run_one "$t"; then pass=$((pass + 1)); else |
| 236 | + echo "RESULT: FAIL (OCE sweep stopped at test $t; $pass passed before it)"; exit 1 |
| 237 | + fi |
| 238 | +done |
| 239 | +echo "RESULT: PASS (OCE sweep: all $pass tests installed via discovery)" |
| 240 | +exit 0 |
0 commit comments