Skip to content

Many vm setup siyuan #32

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
siyuanchai1999 wants to merge 4 commits into
base: many-vm-setup
Choose a base branch
from
Open

Many vm setup siyuan #32

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
04dd1ed
Bunch of fixes
siyuanchai1999 Mar 27, 2026
5b4b61d
Pass in home dir to guest script
siyuanchai1999 Mar 27, 2026
f71d246
- Expose guestVIOMMU option to external script
siyuanchai1999 Mar 27, 2026
e0893ea
Add client logging file
siyuanchai1999 Mar 28, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
55 changes: 55 additions & 0 deletions multi_vms/run_all.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,55 @@
#!/bin/bash
set -euo pipefail

SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
cd "$SCRIPT_DIR"

VIOMMU_MODES=("off" "nested" "nested-vFree")

# --- Experiment configurations ---
# (num_vms, num_total_cores, num_active_cores, num_flows)
# num_flows = num_active_cores per VM
CONFIGS=(
"2 16 15 15"
"2 16 16 16"
"4 8 7 7"
"4 8 8 8"
"8 4 3 3"
"8 4 4 4"
)

run_idx=0
total=$((${#CONFIGS[@]} * ${#VIOMMU_MODES[@]}))

for config in "${CONFIGS[@]}"; do
read -r num_vms num_total_cores num_active_cores num_flows <<< "$config"

for viommu in "${VIOMMU_MODES[@]}"; do
run_idx=$((run_idx + 1))
echo ""
echo "================================================================"
echo " Run ${run_idx}/${total}: --viommu ${viommu} --num-vms ${num_vms}" \
"--num-total-cores ${num_total_cores} --num-active-cores ${num_active_cores}" \
"--num-flows ${num_flows}"
echo "================================================================"
echo ""

./run_vms.sh \
--viommu "$viommu" \
--num-vms "$num_vms" \
--num-total-cores "$num_total_cores" \
--num-active-cores "$num_active_cores" \
--num-flows "$num_flows" \
--skip-reset-host

rc=$?
if [[ $rc -ne 0 ]]; then
echo "WARNING: Run ${run_idx} failed (exit $rc), continuing..."
fi
done
done

echo ""
echo "================================================================"
echo " All ${total} experiments complete"
echo "================================================================"
173 changes: 117 additions & 56 deletions multi_vms/run_vms.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,32 +5,29 @@ set -euo pipefail
# Configuration
# ============================================================

# GUEST_CMD_LINE_NESTED="root=/dev/vda2 ro console=ttyS0,115200 earlyprintk=serial,ttyS0,115200 intel_iommu=on,sm_on iommu.strict=1 intel_iommu_pinned=on intel_iommu_dfp=on"
GUEST_CMD_LINE_NESTED="root=/dev/vda2 ro console=ttyS0,115200 earlyprintk=serial,ttyS0,115200 intel_iommu=on,sm_on iommu.strict=1"
GUEST_CMD_LINE_OFF="root=/dev/vda2 ro console=ttyS0,115200 earlyprintk=serial,ttyS0,115200 intel_iommu=off"

# GUEST_KERNEL="6.12.9-iommufd-nested-iova-contig-cb-opt"
# GUEST_KERNEL_PATH="/boot-VM/vmlinuz-$GUEST_KERNEL"
# GUEST_INITRD_PATH="/boot-VM/initrd.img-$GUEST_KERNEL"
GUEST_KERNEL="6.12.9-iommufd"
GUEST_KERNEL_PATH="/boot/vmlinuz-$GUEST_KERNEL"
GUEST_INITRD_PATH="/boot/initrd.img-$GUEST_KERNEL"
GUEST_VIOMMU="off" # nested/off
NUM_VMS=12
NUM_CORES="2"
NUM_IPRF="1"
NUM_FLOWS="1"
GUEST_VIOMMU="nested-vFree" # off/nested/nested-vFree
NUM_VMS=1
NUM_TOTAL_CORES="16"
NUM_ACTIVE_CORES="12"
NUM_FLOWS="12"
REUSE=0
SKIP_RESET_HOST=0

# --- Hardcoded experiment config ---
GIT_REPO="/home/schai/viommu"
GIT_BRANCH="many-vm-setup"
VM_SCRIPT="cd /home/schai/viommu/scripts/sosp24-experiments; ./many_vm_flows_exp.sh"

# --- Host paths (this script runs ON the host) ---
HOST_HOME="/home/lbalara"
HOST_FandS_REL="viommu/ManyVM-FandS"
HOST_SETUP_DIR="${HOST_HOME}/${HOST_FandS_REL}/utils"
HOST_FandS_ABS=$(dirname "$(dirname "$(readlink -f "${BASH_SOURCE[0]}")")")
HOST_SETUP_DIR="${HOST_FandS_ABS}/utils"
echo "Host F&S dir: ${HOST_FandS_ABS}"

# --- Host config (auto-inferred, passed to VMs so they can SCP results back) ---
HOST_IP="192.17.101.97"
HOST_SSH_UNAME="$(whoami)"
HOST_HOME="$(eval echo ~$(whoami))"
HOST_RESULTS_DIR="${HOST_FandS_ABS}/utils/reports/"

# --- Client machine config ---
CLIENT_HOME="/home/siyuanc3"
Expand Down Expand Up @@ -77,9 +74,8 @@ NIC_WAIT=120 # seconds to wait for guest NIC
# --- Guest NIC interface name (SR-IOV VF) ---
GUEST_NIC="enp0s1"

# --- Host-side in-tree modules to push into guests ---
HOST_MLX5_CORE="/lib/modules/$GUEST_KERNEL/kernel/drivers/net/ethernet/mellanox/mlx5/core/mlx5_core.ko"
HOST_MLXFW="/lib/modules/$GUEST_KERNEL/kernel/drivers/net/ethernet/mellanox/mlxfw/mlxfw.ko"
# --- Host-side in-tree modules (set after GUEST_KERNEL is resolved) ---
# HOST_MLX5_CORE and HOST_MLXFW are set after --viommu parsing below

# Save original directory
ORIG_DIR="$(pwd)"
Expand All @@ -91,29 +87,35 @@ ORIG_DIR="$(pwd)"
usage() {
cat <<-USAGE
Usage: $0 [OPTIONS]
--num-cores N Number of cores per VM experiment
--viommu MODE Guest vIOMMU mode: off, nested, nested-vFree (default: $GUEST_VIOMMU)
--num-total-cores N Total cores per VM
--num-active-cores N Active cores (iperf instances) per VM
--num-flows N Number of flows per VM experiment
--num-vms N Number of VMs (default: $NUM_VMS)
--reuse Reuse already-defined VMs (skip undefine/define, just start)
--skip-reset-host Skip host reset (host setup will still run)
--boot-timeout N Seconds to wait for SSH per VM (default: $BOOT_TIMEOUT)
-h, --help Show this help
USAGE
}

while [[ $# -gt 0 ]]; do
case "$1" in
--viommu) GUEST_VIOMMU="$2"; shift 2 ;;
--num-vms) NUM_VMS="$2"; shift 2 ;;
--num-cores) NUM_CORES="$2"; shift 2 ;;
--num-total-cores) NUM_TOTAL_CORES="$2"; shift 2 ;;
--num-active-cores) NUM_ACTIVE_CORES="$2"; shift 2 ;;
--num-flows) NUM_FLOWS="$2"; shift 2 ;;
--reuse) REUSE=1; shift ;;
--reuse) REUSE=1; shift ;;
--skip-reset-host) SKIP_RESET_HOST=1; shift ;;
--boot-timeout) BOOT_TIMEOUT="$2"; shift 2 ;;
-h|--help) usage; exit 0 ;;
*) echo "Error: unknown option '$1'" >&2; usage >&2; exit 1 ;;
esac
done

if [[ -z "$NUM_CORES" || -z "$NUM_FLOWS" || -z "$NUM_VMS" || -z "$NUM_IPRF" ]]; then
echo "Error: --num-cores, --num-flows, --num-iperf, and --num-vms are required" >&2
if [[ -z "$NUM_TOTAL_CORES" || -z "$NUM_FLOWS" || -z "$NUM_VMS" || -z "$NUM_ACTIVE_CORES" ]]; then
echo "Error: --num-total-cores, --num-active-cores, --num-flows, and --num-vms are required" >&2
usage >&2
exit 1
fi
Expand All @@ -127,12 +129,37 @@ else
SSH_CLIENT_CMD="ssh -i $CLIENT_SSH_IDENTITY_FILE ${CLIENT_SSH_UNAME}@${CLIENT_SSH_HOST}"
fi

GUEST_CMD_LINE=""
if [[ "$GUEST_VIOMMU" == "nested" ]]; then
GUEST_CMD_LINE="$GUEST_CMD_LINE_NESTED"
else
GUEST_CMD_LINE="$GUEST_CMD_LINE_OFF"
fi
# --- Derive guest kernel, initrd, and cmdline from vIOMMU mode ---
GUEST_CMD_LINE_BASE="root=/dev/vda2 ro console=ttyS0,115200 earlyprintk=serial,ttyS0,115200"

case "$GUEST_VIOMMU" in
off)
GUEST_KERNEL="6.12.9-iommufd"
GUEST_KERNEL_PATH="/boot/vmlinuz-$GUEST_KERNEL"
GUEST_INITRD_PATH="/boot/initrd.img-$GUEST_KERNEL"
GUEST_CMD_LINE="$GUEST_CMD_LINE_BASE intel_iommu=off"
;;
nested)
GUEST_KERNEL="6.12.9-iommufd"
GUEST_KERNEL_PATH="/boot/vmlinuz-$GUEST_KERNEL"
GUEST_INITRD_PATH="/boot/initrd.img-$GUEST_KERNEL"
GUEST_CMD_LINE="$GUEST_CMD_LINE_BASE intel_iommu=on,sm_on iommu.strict=1"
;;
nested-vFree)
GUEST_KERNEL="6.12.9-iommufd-nested-iova-contig-cb-opt"
GUEST_KERNEL_PATH="/boot-VM/vmlinuz-$GUEST_KERNEL"
GUEST_INITRD_PATH="/boot-VM/initrd.img-$GUEST_KERNEL"
GUEST_CMD_LINE="$GUEST_CMD_LINE_BASE intel_iommu=on,sm_on iommu.strict=1 intel_iommu_pinned=on intel_iommu_dfp=on"
;;
*)
echo "Error: unknown --viommu mode '$GUEST_VIOMMU' (expected: off, nested, nested-vFree)" >&2
exit 1
;;
esac

# --- Host-side in-tree modules to push into guests ---
HOST_MLX5_CORE="/lib/modules/$GUEST_KERNEL/kernel/drivers/net/ethernet/mellanox/mlx5/core/mlx5_core.ko"
HOST_MLXFW="/lib/modules/$GUEST_KERNEL/kernel/drivers/net/ethernet/mellanox/mlxfw/mlxfw.ko"

# ============================================================
# Helper functions
Expand Down Expand Up @@ -196,9 +223,24 @@ vm_ip() {
echo "${prefix}.$((base_last_octet + idx))"
}

VM_DESTROY_WHITELIST=(
"server-mini-off"
)

destroy_all_running_vms() {
log_info "Shutting down any running VMs..."
for vm in $(virsh list --name 2>/dev/null | grep -v '^$'); do
local skip=false
for wl in "${VM_DESTROY_WHITELIST[@]}"; do
if [[ "$vm" == "$wl" ]]; then
skip=true
break
fi
done
if $skip; then
echo " Skipping (whitelisted): $vm"
continue
fi
echo " Destroying: $vm"
virsh destroy "$vm" 2>/dev/null || true
done
Expand Down Expand Up @@ -382,10 +424,14 @@ cleanup() {
sudo ./sriov_undo.sh

# Host reset
log_info "Running host reset..."
cd "$HOST_SETUP_DIR" || { log_error "Failed to cd to $HOST_SETUP_DIR"; return; }
sudo ./reset-host.sh
cd "$ORIG_DIR"
if [[ "$SKIP_RESET_HOST" -eq 1 ]]; then
log_info "Skipping host reset (--skip-reset-host)"
else
log_info "Running host reset..."
cd "$HOST_SETUP_DIR" || { log_error "Failed to cd to $HOST_SETUP_DIR"; return; }
sudo ./reset-host.sh
cd "$ORIG_DIR"
fi

log_info "=== Cleanup complete ==="
}
Expand All @@ -400,18 +446,23 @@ host_iommu_config=$(parse_iommu_mode "$host_cmdline")
iommu_config="host-${host_iommu_config}-guest-${GUEST_VIOMMU}"
echo "iommu_config: $iommu_config"

# --- Step 1: Destroy any running VMs ---
echo ""
log_info "Step 1: Destroying any running VMs..."
destroy_all_running_vms

log_info "Doing sriov undo"
sudo ./sriov_undo.sh

# --- Step 1: Generate XML files ---
# --- Step 2: Generate XML files ---
if [[ $REUSE -eq 0 ]]; then
rm -rf ./generated
./xml_generator.sh --kernel "$GUEST_KERNEL_PATH" --initrd "$GUEST_INITRD_PATH" --cmdline "$GUEST_CMD_LINE" \
--vcpus $NUM_CORES --num-vms $NUM_VMS --viommu $GUEST_VIOMMU
--vcpus $NUM_TOTAL_CORES --num-vms $NUM_VMS --viommu $GUEST_VIOMMU
fi

timestamp=$(date '+%Y-%m-%d-%H-%M-%S')
EXP_NAME="${timestamp}-$GUEST_KERNEL-MANY-flow${NUM_FLOWS}-${iommu_config}-${NUM_CORES}cores-${NUM_IPRF}iprf"
EXP_NAME="${timestamp}-$GUEST_KERNEL-MANY-flow${NUM_FLOWS}-${iommu_config}-${NUM_TOTAL_CORES}cores-${NUM_ACTIVE_CORES}active-cores"

echo "============================================================"
echo " VM Benchmark Runner"
Expand All @@ -420,8 +471,8 @@ echo " VMs: ${NUM_VMS}"
echo " XML dir: ${XML_DIR}"
echo " Reuse: ${REUSE}"
echo " Branch: ${GIT_BRANCH}"
echo " Cores: ${NUM_CORES}"
echo " Iperf: ${NUM_IPRF}"
echo " Total Cores: ${NUM_TOTAL_CORES}"
echo " Active Cores: ${NUM_ACTIVE_CORES}"
echo " Flows: ${NUM_FLOWS}"
echo " Exp name: ${EXP_NAME}"
echo " MTU: ${MTU}"
Expand All @@ -430,8 +481,8 @@ echo " Socket buf: ${TCP_SOCKET_BUF_MB} MB"
echo "============================================================"
echo ""

# --- Step 2: Discover XML files and VM names ---
log_info "Step 1: Discovering VM XML files..."
# --- Step 3: Discover XML files and VM names ---
log_info "Step 3: Discovering VM XML files..."
xml_files=()
for f in "${XML_DIR}"/*.xml; do
[[ -f "$f" ]] || continue
Expand All @@ -453,11 +504,6 @@ for ((i = 0; i < NUM_VMS; i++)); do
echo " VM${i}: ${vm_name}"
done

# --- Step 3: Destroy any running VMs ---
echo ""
log_info "Step 3: Destroying any running VMs..."
destroy_all_running_vms

# --- Step 4: Define or reuse VMs ---
echo ""
if [[ $REUSE -eq 0 ]]; then
Expand Down Expand Up @@ -544,6 +590,8 @@ fi
echo ""
log_info "Step 10: Waiting for guest NIC ($GUEST_NIC)..."
failed=0
# Add sudo so that over users can overwrite
sudo rm -f /tmp/modules.tar.gz
tar czf /tmp/modules.tar.gz -C /lib/modules/$GUEST_KERNEL .
for ((i = 0; i < NUM_VMS; i++)); do
ip=$(vm_ip "$i")
Expand Down Expand Up @@ -589,8 +637,8 @@ fi
echo ""
log_info "Step 13: Launching experiments on all VMs..."
echo " Script: ${VM_SCRIPT}"
echo " Cores: ${NUM_CORES}"
echo " Iperf: ${NUM_IPRF}"
echo " Total Cores: ${NUM_TOTAL_CORES}"
echo " Active Cores: ${NUM_ACTIVE_CORES}"
echo " Flows: ${NUM_FLOWS}"
echo " Exp name: ${EXP_NAME}"
echo ""
Expand All @@ -600,7 +648,10 @@ for ((i = 0; i < NUM_VMS; i++)); do
ip=$(vm_ip "$i")
name="${vm_names[$i]}"

vm_cmd="${VM_SCRIPT} --vm-name ${name} --num-cores ${NUM_IPRF} --num-flows ${NUM_FLOWS} --exp-name ${EXP_NAME}-${name}"
vm_cmd="${VM_SCRIPT} --vm-name ${name} --num-cores ${NUM_ACTIVE_CORES} --num-flows ${NUM_FLOWS} \
--exp-name ${EXP_NAME}-${name} \
--host-ip ${HOST_IP} --host-ssh-uname ${HOST_SSH_UNAME} \
--host-home ${HOST_HOME} --host-results-dir ${HOST_RESULTS_DIR}"
ssh $SSH_OPTS "$SSH_USER@$ip" "$vm_cmd" &>"${name}_experiment.log" &
ssh_pids+=($!)
echo " Launched on ${name} (${ip}), log -> ${name}_experiment.log"
Expand Down Expand Up @@ -631,8 +682,18 @@ if [[ "$any_failed" == true ]]; then
echo ""
log_info "=== Experiment finished with ERRORS (check *_experiment.log files) ==="
exit 1
else
echo ""
log_info "=== Experiment finished successfully ==="
exit 0
fi
fi

# --- Step 16: Aggregate results from all VMs ---
echo ""
log_info "Step 16: Aggregating results from all VMs..."
REPORT_SCRIPT="${HOST_FandS_ABS}/scripts/many-vm-report-tput-metrics.py"
SUMMARY_DIR="${HOST_RESULTS_DIR}/${EXP_NAME}"

mkdir -p "$SUMMARY_DIR"
cd "${HOST_FandS_ABS}/scripts" || true
python3 "$REPORT_SCRIPT" "$EXP_NAME" tput,cpu "$NUM_VMS" | tee "$SUMMARY_DIR/summary.txt"
cd "$ORIG_DIR"

log_info "=== Experiment finished successfully ==="
exit 0
8 changes: 4 additions & 4 deletions multi_vms/xml_generator.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -87,10 +87,10 @@ for var in KERNEL INITRD CMDLINE; do
done
[[ $err -ne 0 ]] && exit 1

if [[ "$VIOMMU" != "nested" && "$VIOMMU" != "off" ]]; then
echo "Error: --viommu must be 'nested' or 'off'" >&2
exit 1
fi
# if [[ "$VIOMMU" != "nested" && "$VIOMMU" != "off" ]]; then
# echo "Error: --viommu must be 'nested' or 'off'" >&2
# exit 1
# fi

if [[ "$NUM_VMS" -lt 1 || "$NUM_VMS" -gt "$MAX_VMS" ]]; then
echo "Error: --num-vms must be between 1 and $MAX_VMS" >&2
Expand Down
Loading