Skip to content

Commit f729b5d

Browse files
mhenriksmhenriks
committed
run e2e in kind
Signed-off-by: Michael Henriksen <mhenriks@redhat.com>
1 parent 2cf8cc3 commit f729b5d

File tree

7 files changed

+218
-34
lines changed

7 files changed

+218
-34
lines changed

‎.github/workflows/test-e2e.yml‎

Lines changed: 25 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -20,12 +20,35 @@ jobs:
2020
- name: Install system dependencies
2121
run: |
2222
sudo apt-get update
23-
sudo apt-get install -y qemu-kvm libvirt-daemon-system libvirt-clients bridge-utils
23+
sudo apt-get install -y qemu-kvm libvirt-daemon-system libvirt-clients bridge-utils podman
24+
25+
- name: Prepare environment for kubevirtci
26+
run: |
27+
# Create audit log directory required by kind-1.34 provider
28+
sudo mkdir -p /var/log/audit
29+
30+
# Ensure Docker is preferred over podman for kind
31+
# Podman runs rootless by default and can't access /dev/kvm properly
32+
# Docker runs as root and can access /dev/kvm with default permissions
33+
sudo systemctl restart docker
34+
35+
# Temporarily make podman unavailable for kind by renaming it
36+
# (cluster-sync will restore it for building/pushing images)
37+
sudo mv /usr/bin/podman /usr/bin/podman.bak || true
38+
39+
# Verify Docker is working
40+
docker version
2441
2542
- name: Set up kubevirtci
2643
run: |
2744
make cluster-up
2845
46+
- name: Restore podman for image building
47+
run: |
48+
# Restore podman for cluster-sync to use for building/pushing images
49+
sudo mv /usr/bin/podman.bak /usr/bin/podman 2>/dev/null || \
50+
echo "::warning::Podman not restored (may not have been backed up)"
51+
2952
- name: Build and deploy webhook
3053
run: |
3154
make cluster-sync
@@ -37,7 +60,7 @@ jobs:
3760
- name: Collect logs on failure
3861
if: failure()
3962
run: |
40-
export KUBECONFIG=$(pwd)/_kubevirtci/_ci-configs/k8s-1.30/.kubeconfig
63+
export KUBECONFIG=$(pwd)/_kubevirtci/_ci-configs/kind-1.34/.kubeconfig
4164
kubectl get pods -n kubevirt-rbac-webhook-system
4265
kubectl logs -n kubevirt-rbac-webhook-system -l control-plane=controller-manager --tail=100 || true
4366
kubectl get events -n kubevirt-rbac-webhook-system --sort-by='.lastTimestamp' || true

‎KUBEVIRTCI.md‎

Lines changed: 13 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,16 @@ This project uses [kubevirtci](https://github.com/kubevirt/kubevirtci) for devel
1111

1212
## Quick Start
1313

14+
### Prerequisites
15+
16+
- **Docker** (recommended) or Podman for running containers
17+
- Docker is preferred for running the cluster (runs as root, better /dev/kvm access)
18+
- Podman can be used for building/pushing images
19+
- Git (for cloning kubevirtci)
20+
- Make
21+
22+
**Note:** If using both Docker and Podman, the scripts automatically use Docker for cluster operations and Podman for image building. This is optimal for hardware virtualization support.
23+
1424
### 1. Start kubevirtci Cluster
1525

1626
```bash
@@ -19,7 +29,7 @@ make cluster-up
1929

2030
This will:
2131
- Clone kubevirtci (if not already present in `_kubevirtci/`)
22-
- Start a Kubernetes cluster with KubeVirt pre-installed
32+
- Start a Kubernetes cluster (kind-1.34) with KubeVirt pre-installed
2333
- Export KUBECONFIG for the cluster
2434

2535
### 2. Build and Deploy Webhook
@@ -92,7 +102,7 @@ Configuration is managed in `hack/config.sh`:
92102
- `IMAGE_REGISTRY`: Container registry (default: `localhost:5000`)
93103
- `IMAGE_NAME`: Image name (default: `kubevirt-rbac-webhook`)
94104
- `IMAGE_TAG`: Image tag (default: `devel`)
95-
- `KUBEVIRT_PROVIDER`: Kubernetes version (default: `k8s-1.34`)
105+
- `KUBEVIRT_PROVIDER`: Kubernetes version (default: `kind-1.34`)
96106
- `KUBEVIRTCI_VERSION`: kubevirtci version/tag (default: `2510141807-f21813f1`)
97107

98108
## Manual kubectl Access
@@ -114,7 +124,7 @@ _kubevirtci/cluster-up/kubectl.sh get pods -n kubevirt
114124
### Alternative: Use kubectl directly with KUBECONFIG
115125

116126
```bash
117-
export KUBECONFIG=$(pwd)/_kubevirtci/_ci-configs/k8s-1.34/.kubeconfig
127+
export KUBECONFIG=$(pwd)/_kubevirtci/_ci-configs/kind-1.34/.kubeconfig
118128
kubectl get nodes
119129
kubectl get vms -A
120130
```
@@ -228,4 +238,3 @@ The test suite supports these environment variables:
228238

229239
- [kubevirtci GitHub](https://github.com/kubevirt/kubevirtci)
230240
- [KubeVirt Documentation](https://kubevirt.io/user-guide/)
231-
- [virt-template (reference implementation)](https://github.com/kubevirt/virt-template)

‎hack/cluster-down.sh‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,10 +4,14 @@ set -e
44

55
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
66
source "${SCRIPT_DIR}/common.sh"
7+
source "${SCRIPT_DIR}/config.sh"
78

89
main() {
910
log_info "Stopping kubevirtci cluster..."
11+
12+
# Run kubevirtci's cluster-down
1013
kubevirtci::down
14+
1115
log_info "Cluster stopped successfully"
1216
}
1317

‎hack/cluster-sync.sh‎

Lines changed: 50 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -18,18 +18,53 @@ main() {
1818

1919
export KUBECONFIG=$(kubevirtci::kubeconfig)
2020

21-
# Get the registry port from kubevirtci
22-
REGISTRY_PORT=$("${KUBEVIRTCI_ROOT}/cluster-up/cli.sh" ports registry)
23-
REGISTRY_ADDR="localhost:${REGISTRY_PORT}"
24-
log_info "Cluster registry: ${REGISTRY_ADDR}"
25-
26-
# Build and push the webhook image to kubevirtci registry
27-
# Use podman with --tls-verify=false for HTTP registry (following virt-template pattern)
28-
log_info "Building and pushing webhook image..."
29-
(cd "${PROJECT_ROOT}" && \
30-
make docker-build CONTAINER_TOOL=podman IMG="${REGISTRY_ADDR}/kubevirt-rbac-webhook:devel" && \
31-
podman push --tls-verify=false "${REGISTRY_ADDR}/kubevirt-rbac-webhook:devel"
32-
)
21+
# Build and push image to cluster registry
22+
# For kind providers, registry is on localhost:5000; for k8s-* providers, use cli.sh
23+
if [[ "${KUBEVIRT_PROVIDER}" == kind-* ]]; then
24+
log_info "Kind provider detected, using kubevirtci registry"
25+
26+
# For kind providers, kubevirtci creates a registry on localhost:5000
27+
REGISTRY_ADDR="localhost:5000"
28+
log_info "Cluster registry: ${REGISTRY_ADDR}"
29+
30+
# Build and push the webhook image to kubevirtci registry
31+
# Prefer podman (has --tls-verify flag), fallback to docker
32+
if command -v podman &> /dev/null; then
33+
log_info "Building and pushing webhook image with podman..."
34+
(cd "${PROJECT_ROOT}" && \
35+
make docker-build CONTAINER_TOOL=podman IMG="${REGISTRY_ADDR}/kubevirt-rbac-webhook:devel" && \
36+
podman push --tls-verify=false "${REGISTRY_ADDR}/kubevirt-rbac-webhook:devel"
37+
)
38+
else
39+
log_info "Building and pushing webhook image with docker..."
40+
# Docker requires insecure registry to be configured, but localhost:5000 is usually allowed by default
41+
(cd "${PROJECT_ROOT}" && \
42+
make docker-build CONTAINER_TOOL=docker IMG="${REGISTRY_ADDR}/kubevirt-rbac-webhook:devel" && \
43+
docker push "${REGISTRY_ADDR}/kubevirt-rbac-webhook:devel"
44+
)
45+
fi
46+
47+
# Deploy using in-cluster registry name
48+
DEPLOY_IMAGE="registry:5000/kubevirt-rbac-webhook:devel"
49+
else
50+
log_info "Using kubevirtci registry for k8s-* provider"
51+
52+
# Get the registry port from kubevirtci
53+
REGISTRY_PORT=$("${KUBEVIRTCI_ROOT}/cluster-up/cli.sh" ports registry)
54+
REGISTRY_ADDR="localhost:${REGISTRY_PORT}"
55+
log_info "Cluster registry: ${REGISTRY_ADDR}"
56+
57+
# Build and push the webhook image to kubevirtci registry
58+
# Use podman with --tls-verify=false for HTTP registry
59+
log_info "Building and pushing webhook image..."
60+
(cd "${PROJECT_ROOT}" && \
61+
make docker-build CONTAINER_TOOL=podman IMG="${REGISTRY_ADDR}/kubevirt-rbac-webhook:devel" && \
62+
podman push --tls-verify=false "${REGISTRY_ADDR}/kubevirt-rbac-webhook:devel"
63+
)
64+
65+
# Deploy using in-cluster registry name
66+
DEPLOY_IMAGE="registry:5000/kubevirt-rbac-webhook:devel"
67+
fi
3368

3469
# Check and install cert-manager if needed
3570
if ! kubevirtci::kubectl get crd certificates.cert-manager.io &>/dev/null; then
@@ -39,13 +74,13 @@ main() {
3974
log_info "cert-manager installed successfully"
4075
fi
4176

42-
# Undeploy old version (following virt-template pattern for clean reinstall)
77+
# Undeploy old version for clean reinstall
4378
log_info "Cleaning up previous deployment..."
4479
(cd "${PROJECT_ROOT}" && make undeploy || true)
4580

46-
# Deploy the webhook (using in-cluster registry name)
81+
# Deploy the webhook
4782
log_info "Deploying webhook to cluster..."
48-
(cd "${PROJECT_ROOT}" && make deploy IMG="registry:5000/kubevirt-rbac-webhook:devel")
83+
(cd "${PROJECT_ROOT}" && make deploy IMG="${DEPLOY_IMAGE}")
4984

5085
# Wait for webhook to be ready
5186
log_info "Waiting for webhook deployment to be ready..."

‎hack/cluster-up.sh‎

Lines changed: 112 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,7 @@ main() {
1414

1515
# Change to kubevirtci directory and start cluster
1616
log_info "Starting kubevirtci cluster..."
17-
export KUBEVIRT_PROVIDER=${KUBEVIRT_PROVIDER:-k8s-1.34}
17+
export KUBEVIRT_PROVIDER=${KUBEVIRT_PROVIDER:-kind-1.34}
1818
export KUBEVIRTCI_PATH="${KUBEVIRTCI_ROOT}/cluster-up/"
1919
export KUBEVIRTCI_CLUSTER_PATH="${KUBEVIRTCI_ROOT}/cluster-up/cluster"
2020
log_info "Using provider: ${KUBEVIRT_PROVIDER}"
@@ -28,10 +28,41 @@ main() {
2828

2929
# Wait for cluster to be ready
3030
log_info "Waiting for cluster to be ready (this may take a few minutes)..."
31-
kubevirtci::kubectl wait --for=condition=Ready nodes --all --timeout=10m || {
31+
if ! kubevirtci::kubectl wait --for=condition=Ready nodes --all --timeout=10m; then
3232
log_error "Cluster nodes did not become ready"
33+
log_error "Collecting cluster diagnostic information..."
34+
35+
echo "====== Cluster Info ======"
36+
kubevirtci::kubectl cluster-info || true
37+
38+
echo "====== Node Status ======"
39+
kubevirtci::kubectl get nodes -o wide || true
40+
kubevirtci::kubectl describe nodes || true
41+
42+
echo "====== System Pods ======"
43+
kubevirtci::kubectl get pods -A || true
44+
3345
exit 1
34-
}
46+
fi
47+
48+
# Check if nested virtualization is available
49+
log_info "Checking virtualization support..."
50+
if [ -e /dev/kvm ]; then
51+
if [ -r /dev/kvm ] && [ -w /dev/kvm ]; then
52+
log_info "✓ /dev/kvm is accessible"
53+
log_info " KubeVirt will use hardware acceleration (fast)"
54+
else
55+
log_warn "⚠ /dev/kvm exists but not accessible"
56+
log_warn " Software emulation will be used (slower)"
57+
log_warn " For Docker: Should work as-is (runs as root)"
58+
log_warn " For rootless podman: Consider using Docker instead"
59+
fi
60+
else
61+
log_warn "⚠ /dev/kvm NOT available"
62+
log_warn " Software emulation will be automatically enabled (VERY SLOW)"
63+
log_warn " For CI: This is expected and acceptable for basic testing"
64+
log_warn " For local dev: Enable virtualization in BIOS and kernel modules"
65+
fi
3566

3667
# Install KubeVirt if not present
3768
log_info "Checking for KubeVirt installation..."
@@ -45,15 +76,82 @@ main() {
4576
# Install KubeVirt operator
4677
kubevirtci::kubectl create -f "https://github.com/kubevirt/kubevirt/releases/download/${KUBEVIRT_VERSION}/kubevirt-operator.yaml"
4778

79+
# Wait for operator to be ready
80+
log_info "Waiting for KubeVirt operator to be ready..."
81+
kubevirtci::kubectl wait --for=condition=Available -n kubevirt deployment/virt-operator --timeout=5m
82+
4883
# Install KubeVirt CR
4984
kubevirtci::kubectl create -f "https://github.com/kubevirt/kubevirt/releases/download/${KUBEVIRT_VERSION}/kubevirt-cr.yaml"
5085

86+
# Enable software emulation if /dev/kvm is not available or not accessible
87+
NEED_EMULATION=false
88+
if [ ! -e /dev/kvm ]; then
89+
log_warn "/dev/kvm does not exist - emulation required"
90+
NEED_EMULATION=true
91+
elif [ ! -r /dev/kvm ] || [ ! -w /dev/kvm ]; then
92+
log_warn "/dev/kvm exists but is not accessible - emulation may be required"
93+
NEED_EMULATION=true
94+
fi
95+
96+
if [ "$NEED_EMULATION" = true ]; then
97+
log_warn "Enabling software emulation in KubeVirt"
98+
log_warn "This will make VMs run VERY slowly - only suitable for basic testing"
99+
100+
# Patch the KubeVirt CR to enable software emulation
101+
kubevirtci::kubectl patch kubevirt kubevirt -n kubevirt --type=merge -p '{"spec":{"configuration":{"developerConfiguration":{"useEmulation":true}}}}'
102+
else
103+
log_info "Hardware acceleration available - KubeVirt will use /dev/kvm"
104+
fi
105+
51106
# Wait for KubeVirt to be ready
52107
log_info "Waiting for KubeVirt to be ready (this may take several minutes)..."
53-
kubevirtci::kubectl wait --for=condition=Available -n kubevirt kv/kubevirt --timeout=10m || {
54-
log_error "KubeVirt deployment failed"
108+
if ! kubevirtci::kubectl wait --for=condition=Available -n kubevirt kv/kubevirt --timeout=10m; then
109+
log_error "KubeVirt deployment timed out or failed"
110+
log_error "Collecting diagnostic information..."
111+
112+
echo "====== KubeVirt Resource Status ======"
113+
kubevirtci::kubectl get kv -n kubevirt -o yaml || true
114+
115+
echo "====== KubeVirt Pods Status ======"
116+
kubevirtci::kubectl get pods -n kubevirt -o wide || true
117+
118+
echo "====== KubeVirt Pods with Restart Counts ======"
119+
kubevirtci::kubectl get pods -n kubevirt -o custom-columns=NAME:.metadata.name,STATUS:.status.phase,RESTARTS:.status.containerStatuses[*].restartCount,READY:.status.containerStatuses[*].ready || true
120+
121+
echo "====== KubeVirt DaemonSets (virt-handler) ======"
122+
kubevirtci::kubectl get daemonsets -n kubevirt -o wide || true
123+
kubevirtci::kubectl describe daemonsets -n kubevirt virt-handler || true
124+
125+
echo "====== KubeVirt Events ======"
126+
kubevirtci::kubectl get events -n kubevirt --sort-by='.lastTimestamp' || true
127+
128+
echo "====== KubeVirt Operator Logs ======"
129+
kubevirtci::kubectl logs -n kubevirt -l kubevirt.io=virt-operator --tail=100 || true
130+
echo "====== KubeVirt Operator Previous Logs (if crash-looping) ======"
131+
kubevirtci::kubectl logs -n kubevirt -l kubevirt.io=virt-operator --previous --tail=50 2>/dev/null || echo "No previous operator logs"
132+
133+
echo "====== KubeVirt API Logs ======"
134+
kubevirtci::kubectl logs -n kubevirt -l kubevirt.io=virt-api --tail=50 || true
135+
echo "====== KubeVirt API Previous Logs (if crash-looping) ======"
136+
kubevirtci::kubectl logs -n kubevirt -l kubevirt.io=virt-api --previous --tail=50 2>/dev/null || echo "No previous API logs"
137+
138+
echo "====== KubeVirt Controller Logs ======"
139+
kubevirtci::kubectl logs -n kubevirt -l kubevirt.io=virt-controller --tail=50 || true
140+
echo "====== KubeVirt Controller Previous Logs (if crash-looping) ======"
141+
kubevirtci::kubectl logs -n kubevirt -l kubevirt.io=virt-controller --previous --tail=50 2>/dev/null || echo "No previous controller logs"
142+
143+
echo "====== KubeVirt Handler Logs (DaemonSet - runs on nodes) ======"
144+
kubevirtci::kubectl logs -n kubevirt -l kubevirt.io=virt-handler --tail=100 --all-containers=true || true
145+
146+
echo "====== KubeVirt Handler Previous Logs (if crash-looping) ======"
147+
kubevirtci::kubectl logs -n kubevirt -l kubevirt.io=virt-handler --previous --tail=100 --all-containers=true 2>/dev/null || echo "No previous logs available (pods may not have restarted yet)"
148+
149+
echo "====== Node Status ======"
150+
kubevirtci::kubectl get nodes -o wide || true
151+
kubevirtci::kubectl describe nodes || true
152+
55153
exit 1
56-
}
154+
fi
57155

58156
log_info "KubeVirt installed successfully"
59157
else
@@ -72,6 +170,14 @@ main() {
72170

73171
if ! kubevirtci::kubectl get crd virtualmachines.kubevirt.io &>/dev/null; then
74172
log_error "KubeVirt CRDs not found after installation!"
173+
log_error "Collecting CRD diagnostic information..."
174+
175+
echo "====== All CRDs ======"
176+
kubevirtci::kubectl get crds | grep -i kubevirt || true
177+
178+
echo "====== KubeVirt Namespace Status ======"
179+
kubevirtci::kubectl get all -n kubevirt || true
180+
75181
exit 1
76182
fi
77183

‎hack/common.sh‎

Lines changed: 13 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,6 @@
11
#!/usr/bin/env bash
22

33
# kubevirtci integration utilities
4-
# Based on patterns from kubevirt/virt-template
54

65
set -e
76

@@ -48,7 +47,7 @@ kubevirtci::up() {
4847
kubevirtci::install
4948

5049
# Set the provider
51-
export KUBEVIRT_PROVIDER=${KUBEVIRT_PROVIDER:-k8s-1.30}
50+
export KUBEVIRT_PROVIDER=${KUBEVIRT_PROVIDER:-kind-1.34}
5251

5352
# Run cluster-up from kubevirtci root directory
5453
(cd "${KUBEVIRTCI_ROOT}" && make cluster-up)
@@ -69,7 +68,7 @@ kubevirtci::down() {
6968
}
7069

7170
kubevirtci::kubeconfig() {
72-
local provider=${KUBEVIRT_PROVIDER:-k8s-1.34}
71+
local provider=${KUBEVIRT_PROVIDER:-kind-1.34}
7372
echo "${KUBEVIRTCI_ROOT}/_ci-configs/${provider}/.kubeconfig"
7473
}
7574

@@ -82,10 +81,18 @@ kubevirtci::ssh() {
8281
"${KUBEVIRTCI_ROOT}/cluster-up/ssh.sh" "$@"
8382
}
8483

85-
# Get the dynamic cluster registry port
84+
# Get the cluster registry address
8685
kubevirtci::registry() {
87-
local port=$("${KUBEVIRTCI_ROOT}/cluster-up/cli.sh" ports registry)
88-
echo "localhost:${port}"
86+
local provider=${KUBEVIRT_PROVIDER:-kind-1.34}
87+
88+
# For kind providers, kubevirtci creates a registry on localhost:5000
89+
if [[ "${provider}" == kind-* ]]; then
90+
echo "localhost:5000"
91+
else
92+
# For k8s-* providers, use cli.sh to get dynamic port
93+
local port=$("${KUBEVIRTCI_ROOT}/cluster-up/cli.sh" ports registry)
94+
echo "localhost:${port}"
95+
fi
8996
}
9097

9198
# Check if cluster is running

0 commit comments

Comments
 (0)