Skip to content

change script to use cross numa node #36

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
leshnabalara wants to merge 2 commits into
base: many-investigation
Choose a base branch
from
Open

change script to use cross numa node #36

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
acd2cff
change script to use cross numa node
leshnabalara Mar 31, 2026
d8626f1
remove waits
leshnabalara Mar 31, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 3 additions & 4 deletions multi_vms/run_all.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,16 +4,15 @@ set -euo pipefail
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
cd "$SCRIPT_DIR"

VIOMMU_MODES=("off" "nested" "nested-vFree")
VIOMMU_MODES=("off" "nested-vFree" "nested")
# VIOMMU_MODES=("nested-vFree")

# --- Experiment configurations ---
# (num_vms, num_total_cores, num_active_cores, num_flows)
# num_flows = num_active_cores per VM
CONFIGS=(
"2 16 16 16"
"4 8 8 8"
"8 4 4 4"
# "2 16 16 16"
"4 16 16 16"
)

run_idx=0
Expand Down
2 changes: 1 addition & 1 deletion multi_vms/run_vms.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ SKIP_RESET_HOST=0

# --- Hardcoded experiment config ---
GIT_REPO="/home/schai/viommu"
GIT_BRANCH="many-investigation"
GIT_BRANCH="multi-cross-numa"
VM_SCRIPT="cd /home/schai/viommu/scripts/sosp24-experiments; ./many_vm_flows_exp.sh"

# --- Host paths (this script runs ON the host) ---
Expand Down
4 changes: 3 additions & 1 deletion multi_vms/xml_generator.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -155,6 +155,8 @@ for ((vm = 0; vm < NUM_VMS; vm++)); do
disk="/data/server_small${vf_num}.qcow2"
fi
cpu_base=$((CPU_START + vm * VCPUS))
cpu_base=$(( cpu_base % 128 ))
NUMA_NODE=$(( cpu_base / 32 ))
mac_last=$(printf '%02x' $(( (0xe2 + vm) & 0xff )))
mac="52:54:00:14:26:${mac_last}"
outfile="${OUTPUT_DIR}/${TAG}-iommufd-${VIOMMU}-vcpu${VCPUS}-vm${vm}.xml"
Expand Down Expand Up @@ -293,7 +295,7 @@ $(gen_pcie_root_ports)
</domain>
EOF

echo "Generated: ${outfile} (${vm_name}, VF=${vf_pci}, CPUs=${cpu_base}-$((cpu_base + VCPUS - 1)), disk=${disk})"
echo "Generated: ${outfile} (${vm_name}, VF=${vf_pci}, CPUs=${cpu_base}-$((cpu_base + VCPUS - 1)), NUMA=${NUMA_NODE}, disk=${disk})"
done

echo "Done: ${NUM_VMS} VM config(s) in ${OUTPUT_DIR}/"
12 changes: 6 additions & 6 deletions scripts/many-vm-run-dctcp-tput-experiment.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,7 @@ VM_ID="0" # Unique per-VM identifier for session names, ports, file prefixes
NUM_RUNS=1 # Always 1 for multi-VM; coordination handled by host
CORE_DURATION_S=20 # Duration for the main workload
MLC_CORES="none"
EBPF_TRACING_ENABLED=1
EBPF_TRACING_ENABLED=0
EBPF_TRACING_HOST_ENABLED=0
COLLECT_MEM_STATS=0

Expand Down Expand Up @@ -335,7 +335,7 @@ cleanup() {
$SSH_HOST_CMD \
"sudo pkill -SIGINT -f '$host_loader_basename'; sleep 5; sudo pkill -9 -f '$host_loader_basename'; screen -S ebpf_host_tracer -X quit || true"
fi
sleep 5
# sleep 5


# Only kill THIS VM's screen sessions on the client (not other VMs')
Expand Down Expand Up @@ -436,7 +436,7 @@ client_app_log_file_remote="${client_reports_dir_remote}/client_app.log"
ebpf_host_stats="${host_reports_dir_remote}/ebpf_host_stats.csv"

sudo mkdir -p "$current_guest_reports_dir"
$SSH_HOST_CMD "mkdir -p '$host_reports_dir_remote'"
# $SSH_HOST_CMD "mkdir -p '$host_reports_dir_remote'"

# --- Pre-run cleanup ---
cleanup
Expand Down Expand Up @@ -490,14 +490,14 @@ $SSH_CLIENT_CMD "screen -dmS $SCREEN_CLIENT_SESSION sudo bash -c \"$client_cmd\"

# --- Warmup Phase ---
log_info "Warming up experiment (120 seconds)..."
progress_bar 116 2
progress_bar 120 2

if [ "$EBPF_TRACING_HOST_ENABLED" -eq 1 ]; then
log_info "Starting HOST eBPF tracer on $HOST_IP..."
host_loader_cmd="sudo taskset -c 33 $EBPF_HOST_LOADER -o $ebpf_host_stats"
$SSH_HOST_CMD "screen -dmS ebpf_host_tracer sudo bash -c \"$host_loader_cmd\""
fi
sleep 4 # Allow eBPF loaders to initialize
# sleep 4 # Allow eBPF loaders to initialize

# --- Start Guest eBPF Tracers (if enabled) ---
if [ "$EBPF_TRACING_ENABLED" -eq 1 ]; then
Expand All @@ -507,7 +507,7 @@ if [ "$EBPF_TRACING_ENABLED" -eq 1 ]; then
fi

# Sleep outside so all VMs nearly in sync
sleep 2
# sleep 2

# --- Guest Ftrace Setup ---
log_info "Configuring GUEST ftrace (Buffer: ${FTRACE_BUFFER_SIZE_KB}KB, Overwrite: ${FTRACE_OVERWRITE_ON_FULL})..."
Expand Down
8 changes: 4 additions & 4 deletions scripts/sosp24-experiments/many_vm_flows_exp.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -101,7 +101,7 @@ fi
# Each VM uses a different slice of client cores based on VM_INDEX.
# VM0 uses cores 0..NUM_CORES-1, VM1 uses NUM_CORES..2*NUM_CORES-1, etc.
# Server cores always start at 0 within each VM (guest-local).
all_cores="0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31"
all_cores=$(echo {0..63} | tr ' ' ',')

client_core_offset=$((VM_INDEX * NUM_CORES))
client_cores_mask=$(echo "$all_cores" | tr ',' '\n' | tail -n +$((client_core_offset + 1)) | head -n "$NUM_CORES" | tr '\n' ',' | sed 's/,$//')
Expand Down Expand Up @@ -175,9 +175,9 @@ for z in $run_list; do
"$local_report_dir"/* \
"${HOST_SSH_UNAME}@${HOST_IP}:${HOST_RESULTS_DIR}/${EXP_NAME}/"

$SCP_HOST_CMD -r \
"$local_run_dir"/* \
"${HOST_SSH_UNAME}@${HOST_IP}:${HOST_RESULTS_DIR}/${EXP_NAME}-RUN-0/"
# $SCP_HOST_CMD -r \
# "$local_run_dir"/* \
# "${HOST_SSH_UNAME}@${HOST_IP}:${HOST_RESULTS_DIR}/${EXP_NAME}-RUN-0/"

echo "Results copied to ${HOST_SSH_UNAME}@${HOST_IP}:${HOST_RESULTS_DIR}/${EXP_NAME}/"
done
Expand Down