Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions hack/e2e/README.md
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
# AKS Flex Node E2E Tests

The E2E suite provisions an AKS cluster and three Ubuntu VMs in Azure, joins the VMs as Flex Nodes, validates workloads, exercises unjoin/rejoin behavior, validates repave, collects logs, and tears down the resources.
The E2E suite provisions an AKS cluster and three Ubuntu VMs in Azure, joins the VMs as Flex Nodes, validates workloads, exercises unjoin/rejoin behavior, verifies reset cleanup, validates repave, collects logs, and tears down the resources.

## Prerequisites

Expand Down Expand Up @@ -30,7 +30,7 @@ The default `all` command runs:
2. Deploy AKS and three VMs with Bicep.
3. Join all three VMs.
4. Validate node readiness, node-problem-detector status, and run smoke workloads.
5. Unjoin all Flex Nodes and verify they are absent.
5. Unjoin all Flex Nodes and verify they are absent, including reset cleanup of host network artifacts.
6. Rejoin all Flex Nodes and validate again.
7. Run local-machine-driven repave validation.
8. Collect logs and clean up Azure resources.
Expand Down
64 changes: 64 additions & 0 deletions hack/e2e/lib/node-join.sh
Original file line number Diff line number Diff line change
Expand Up @@ -206,6 +206,65 @@ _validate_rp_delete_cleanup() {
remote_exec "${vm_ip}" "E2E_NODE_JOIN_TIMEOUT=${E2E_NODE_JOIN_TIMEOUT} bash -s" <<'REMOTE'
set -euo pipefail

validate_no_wireguard_interfaces() {
local interfaces
local -a matches

shopt -s nullglob
matches=(/sys/class/net/wg*)
shopt -u nullglob

Comment thread
bcho marked this conversation as resolved.
if ((${#matches[@]} == 0)); then
return 0
fi

interfaces="$(printf '%s\n' "${matches[@]##*/}" | sort)"
echo "WireGuard interfaces still exist after reset cleanup:"
echo "${interfaces}"
while IFS= read -r iface; do
ip link show "${iface}" || true
done <<<"${interfaces}"
exit 1
}

validate_no_overlay_interfaces() {
local iface

for iface in geneve0 vxlan0 ipip0 unbounded0 cbr0; do
if [[ -e "/sys/class/net/${iface}" ]]; then
echo "reset cleanup interface ${iface} still exists"
ip link show "${iface}" || true
exit 1
fi
done
}

validate_no_wireguard_keys() {
local path

for path in /etc/wireguard/server.priv /etc/wireguard/server.pub; do
if [[ -e "${path}" ]]; then
echo "reset cleanup WireGuard key ${path} still exists"
sudo ls -l "${path}" || true
exit 1
fi
done
}

validate_no_policy_routing_state() {
if ip rule show | grep -Eq 'lookup 51898|table 51898'; then
echo "reset cleanup policy routing rule for table 51898 still exists"
ip rule show
exit 1
fi

if ip route show table 51898 | grep -q .; then
echo "reset cleanup route table 51898 is not empty"
ip route show table 51898
exit 1
fi
}

deadline=$((SECONDS + E2E_NODE_JOIN_TIMEOUT))
while systemctl list-unit-files aks-flex-node-agent.service --no-legend | grep -q '^aks-flex-node-agent.service'; do
if (( SECONDS >= deadline )); then
Expand All @@ -230,6 +289,11 @@ for machine in kube1 kube2; do
fi
done

validate_no_wireguard_interfaces
validate_no_overlay_interfaces
validate_no_wireguard_keys
validate_no_policy_routing_state

for path in /etc/aks-flex-node /var/log/aks-flex-node; do
if [[ -e "${path}" ]]; then
echo "runtime path ${path} still exists after delete"
Expand Down
6 changes: 5 additions & 1 deletion pkg/daemon/reset.go
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,11 @@ func ResetNode(log *slog.Logger) phases.Task {
reset.CleanupMachine(log, goalstates.NSpawnMachineKube1),
reset.CleanupMachine(log, goalstates.NSpawnMachineKube2),
),
reset.CleanupRoutes(log),
phases.Parallel(log,
Comment thread
bcho marked this conversation as resolved.
reset.RemoveNetworkInterfaces(log),
reset.RemoveWireGuardKeys(log),
reset.CleanupRoutes(log),
),
reset.ReloadSystemd(log),
config.RemoveRuntimeDirs(log),
arc.UninstallArc(log),
Expand Down
6 changes: 4 additions & 2 deletions scripts/uninstall.sh
Original file line number Diff line number Diff line change
Expand Up @@ -47,9 +47,10 @@ confirm_uninstall() {
echo "• Configuration directory ($CONFIG_DIR)"
echo "• Data directory ($DATA_DIR)"
echo "• Log directory ($LOG_DIR)"
echo "• Host network state created by unbounded-net"
echo "• Azure CLI"
echo ""
echo -e "${YELLOW}NOTE: This will first run 'aks-flex-node reset' to clean up cluster and Arc resources.${NC}"
echo -e "${YELLOW}NOTE: This will first run 'aks-flex-node reset' to clean up cluster, Arc, and host network resources.${NC}"
Comment thread
bcho marked this conversation as resolved.
echo ""

# Always prompt for confirmation, even when piped
Expand All @@ -66,7 +67,7 @@ confirm_uninstall() {
}

run_reset() {
log_info "Running reset to clean up cluster and Arc resources..."
log_info "Running reset to clean up cluster, Arc, and host network resources..."

# Check if aks-flex-node binary exists
if [[ ! -f "$INSTALL_DIR/aks-flex-node" ]]; then
Expand Down Expand Up @@ -183,6 +184,7 @@ show_completion_message() {
echo "✅ Service user and permissions"
echo "✅ Configuration and data directories"
echo "✅ Log files"
echo "✅ Host network state"
Comment thread
bcho marked this conversation as resolved.
echo "✅ Azure CLI"
echo ""
echo -e "${GREEN}Complete uninstallation finished!${NC}"
Expand Down
Loading