ci: add tests for underlay installation on Talos (#5147)

Signed-off-by: zhangzujian <zhangzujian.7@gmail.com>

Author: zhangzujian

.github/workflows/build-x86-image.yaml

@@ -2051,6 +2051,9 @@ jobs:
           - ipv4
           - ipv6
           - dual
+        mode:
+          - overlay
+          - underlay
     steps:
       - uses: jlumbroso/free-disk-space@v1.3.1
         with:
@@ -2089,11 +2092,16 @@ jobs:
       - name: Install Kube-OVN
         id: install
         run: |
-          make talos-install-${{ matrix.ip-family }}
+          make talos-install-${{ matrix.mode }}-${{ matrix.ip-family }}
           kubectl -n kube-system rollout status ds/kube-ovn-pinger
           kubectl -n kube-system rollout status deploy/coredns
           kubectl get pod -o wide -A
           kubectl get endpointslice -A
+          kubectl get subnet -o wide
+          if [ "${{ matrix.mode }}" = "underlay" ]; then
+            kubectl get provider-network -o wide
+            kubectl get vlan -o wide
+          fi
 
       - name: Check kube ovn pod restarts
         if: ${{ success() || (failure() && steps.install.conclusion == 'failure') }}

Makefile

@@ -182,9 +182,19 @@ install-chart: untaint-control-plane
 		--set OPENVSWITCH_DIR=$(shell echo $${OPENVSWITCH_DIR:-/etc/origin/openvswitch}) \
 		--set DISABLE_MODULES_MANAGEMENT=$(shell echo $${DISABLE_MODULES_MANAGEMENT:-false}) \
 		--set cni_conf.MOUNT_LOCAL_BIN_DIR=$(shell echo $${MOUNT_LOCAL_BIN_DIR:-true}) \
-		--set networking.NET_STACK=$(shell echo $${NET_STACK:-ipv4} | sed 's/^dual$$/dual_stack/') \
 		--set networking.ENABLE_SSL=$(shell echo $${ENABLE_SSL:-false}) \
+		--set networking.NETWORK_TYPE=$(shell echo $${NETWORK_TYPE:-geneve}) \
 		--set networking.TUNNEL_TYPE=$(shell echo $${TUNNEL_TYPE:-geneve}) \
+		--set networking.vlan.VLAN_INTERFACE_NAME=$(shell echo $${VLAN_INTERFACE_NAME:-}) \
+		--set networking.vlan.VLAN_ID=$(shell echo $${VLAN_ID:-100}) \
+		--set networking.NET_STACK=$(shell echo $${NET_STACK:-ipv4} | sed 's/^dual$$/dual_stack/') \
+		--set-json networking.EXCLUDE_IPS='"$(shell echo $${EXCLUDE_IPS:-})"' \
+		--set-json ipv4.POD_CIDR='"$(shell echo $${POD_CIDR:-10.16.0.0/16})"' \
+		--set-json ipv4.POD_GATEWAY='"$(shell echo $${POD_GATEWAY:-10.16.0.1})"' \
+		--set-json ipv6.POD_CIDR='"$(shell echo $${POD_CIDR:-fd00:10:16::/112})"' \
+		--set-json ipv6.POD_GATEWAY='"$(shell echo $${POD_GATEWAY:-fd00:10:16::1})"' \
+		--set-json dual_stack.POD_CIDR='"$(shell echo $${POD_CIDR:-10.16.0.0/16,fd00:10:16::/112})"' \
+		--set-json dual_stack.POD_GATEWAY='"$(shell echo $${POD_GATEWAY:-10.16.0.1,fd00:10:16::1})"' \
 		--set func.SECURE_SERVING=$(shell echo $${SECURE_SERVING:-false}) \
 		--set func.ENABLE_BIND_LOCAL_IP=$(shell echo $${ENABLE_BIND_LOCAL_IP:-true}) \
 		--set func.ENABLE_OVN_IPSEC=$(shell echo $${ENABLE_OVN_IPSEC:-false}) \
@@ -199,9 +209,19 @@ upgrade-chart:
 		--set OPENVSWITCH_DIR=$(shell echo $${OPENVSWITCH_DIR:-/etc/origin/openvswitch}) \
 		--set DISABLE_MODULES_MANAGEMENT=$(shell echo $${DISABLE_MODULES_MANAGEMENT:-false}) \
 		--set cni_conf.MOUNT_LOCAL_BIN_DIR=$(shell echo $${MOUNT_LOCAL_BIN_DIR:-true}) \
-		--set networking.NET_STACK=$(shell echo $${NET_STACK:-ipv4} | sed 's/^dual$$/dual_stack/') \
 		--set networking.ENABLE_SSL=$(shell echo $${ENABLE_SSL:-false}) \
+		--set networking.NETWORK_TYPE=$(shell echo $${NETWORK_TYPE:-geneve}) \
 		--set networking.TUNNEL_TYPE=$(shell echo $${TUNNEL_TYPE:-geneve}) \
+		--set networking.vlan.VLAN_INTERFACE_NAME=$(shell echo $${VLAN_INTERFACE_NAME:-}) \
+		--set networking.vlan.VLAN_ID=$(shell echo $${VLAN_ID:-100}) \
+		--set networking.NET_STACK=$(shell echo $${NET_STACK:-ipv4} | sed 's/^dual$$/dual_stack/') \
+		--set-json networking.EXCLUDE_IPS='"$(shell echo $${EXCLUDE_IPS:-})"' \
+		--set-json ipv4.POD_CIDR='"$(shell echo $${POD_CIDR:-10.16.0.0/16})"' \
+		--set-json ipv4.POD_GATEWAY='"$(shell echo $${POD_GATEWAY:-10.16.0.1})"' \
+		--set-json ipv6.POD_CIDR='"$(shell echo $${POD_CIDR:-fd00:10:16::/112})"' \
+		--set-json ipv6.POD_GATEWAY='"$(shell echo $${POD_GATEWAY:-fd00:10:16::1})"' \
+		--set-json dual_stack.POD_CIDR='"$(shell echo $${POD_CIDR:-10.16.0.0/16,fd00:10:16::/112})"' \
+		--set-json dual_stack.POD_GATEWAY='"$(shell echo $${POD_GATEWAY:-10.16.0.1,fd00:10:16::1})"' \
 		--set func.SECURE_SERVING=$(shell echo $${SECURE_SERVING:-false}) \
 		--set func.ENABLE_BIND_LOCAL_IP=$(shell echo $${ENABLE_BIND_LOCAL_IP:-true}) \
 		--set func.ENABLE_OVN_IPSEC=$(shell echo $${ENABLE_OVN_IPSEC:-false}) \

talos.mk

@@ -31,6 +31,16 @@ TALOS_WORKER_COUNT ?= 1
 
 TALOS_API_PORT ?= 50000
 
+TALOS_UNDERLAY_CIDR_IPV4 = 172.99.99.0/24
+TALOS_UNDERLAY_CIDR_IPV6 = 2001:db8:99:99::1/120
+TALOS_UNDERLAY_CIDR_DUAL = $(TALOS_UNDERLAY_CIDR_IPV4),$(TALOS_UNDERLAY_CIDR_IPV6)
+TALOS_UNDERLAY_GATEWAY_IPV4 = 172.99.99.1
+TALOS_UNDERLAY_GATEWAY_IPV6 = 2001:db8:99:99::1
+TALOS_UNDERLAY_GATEWAY_DUAL = $(TALOS_UNDERLAY_GATEWAY_IPV4),$(TALOS_UNDERLAY_GATEWAY_IPV6)
+TALOS_UNDERLAY_EXCLUDE_IPS_IPV4 = 172.99.99.11..172.99.99.99
+TALOS_UNDERLAY_EXCLUDE_IPS_IPV6 = 2001:db8:99:99::11..2001:db8:99:99::99
+TALOS_UNDERLAY_EXCLUDE_IPS_DUAL = $(TALOS_UNDERLAY_EXCLUDE_IPS_IPV4),$(TALOS_UNDERLAY_EXCLUDE_IPS_IPV6)
+
 # geneve causes kernel panic on my local libvirt virtual machines
 # use vxlan instead
 TALOS_TUNNEL_TYPE = vxlan
@@ -71,7 +81,7 @@ talos-libvirt-init: talos-libvirt-clean
 		sudo mkdir -p "$(TALOS_IMAGE_DIR)" && \
 		sudo chmod 777 "$(TALOS_IMAGE_DIR)" && \
 		echo ">>> Downloading Talos image $(TALOS_IMAGE_ISO) into $(TALOS_IMAGE_DIR)..." && \
-		wget "$(TALOS_IMAGE_URL)" -O "$(TALOS_IMAGE_PATH)" && \
+		wget "$(TALOS_IMAGE_URL)" --quiet -O "$(TALOS_IMAGE_PATH)" && \
 		echo ">>> Talos image downloaded."; \
 	fi
 	@echo ">>> Creating libvirt network $(TALOS_LIBVIRT_NETWORK_NAME)..."
@@ -199,19 +209,20 @@ talos-init-%: talos-libvirt-init talos-prepare-images
 	@echo ">>> Waiting for k8s endpoint to be ready..."
 	@while true; do \
 		if kubectl get nodes &>/dev/null; then \
-			echo ">>>>>> K8s endpoint is ready."; \
+			echo ">>> K8s endpoint is ready."; \
 			break; \
 		fi; \
 		echo ">>>>>> Waiting for k8s endpoint..."; \
 		sleep 2; \
 	done
-	@echo ">>> Waiting for all k8s nodes to be present..."
+	@echo ">>> Waiting for all k8s nodes to be registered..."
 	@while true; do \
-		if [ $$(kubectl get nodes -o name | wc -l) -eq $$(($(TALOS_CONTROL_PLANE_COUNT)+$(TALOS_WORKER_COUNT))) ]; then \
-			echo ">>>>>> K8s nodes are present."; \
+		count=$$(kubectl get nodes -o name | wc -l); \
+		echo ">>>>>> $${count} node(s) are registered..."; \
+		if [ $${count} -eq $$(($(TALOS_CONTROL_PLANE_COUNT)+$(TALOS_WORKER_COUNT))) ]; then \
+			echo ">>> All k8s nodes are registered."; \
 			break; \
 		fi; \
-		echo ">>>>>> Waiting for all k8s nodes to be present..."; \
 		sleep 2; \
 	done
 	@echo ">>> Waiting for kube-proxy to be ready..."
@@ -249,12 +260,30 @@ talos-install: talos-install-prepare
 		$(MAKE) install-chart
 
 .PHONY: talos-install-%
-talos-install-%:
-	@NET_STACK=$* $(MAKE) talos-install
+talos-install-%: talos-install-overlay-$*
 
 .PHONY: talos-install-dev
 talos-install-dev: talos-install-dev-ipv4
 
 .PHONY: talos-install-dev-%
 talos-install-dev-%:
 	@VERSION=$(DEV_TAG) $(MAKE) talos-install-$*
+
+.PHONY: talos-install-overlay
+talos-install-overlay: talos-install-overlay-ipv4
+
+.PHONY: talos-install-overlay-%
+talos-install-overlay-%:
+	@NET_STACK=$* $(MAKE) talos-install
+
+.PHONY: talos-install-underlay
+talos-install-underlay: talos-install-underlay-ipv4
+
+.PHONY: talos-install-underlay-%
+talos-install-underlay-%:
+	@$(eval UNDERLAY_VAR_SUFFIX = $(shell echo $* | tr 'a-z' 'A-Z'))
+	@NET_STACK=$* NETWORK_TYPE=vlan VLAN_INTERFACE_NAME=enp0s5f1 VLAN_ID=0 \
+		POD_CIDR=$(TALOS_UNDERLAY_CIDR_$(UNDERLAY_VAR_SUFFIX)) \
+		POD_GATEWAY=$(TALOS_UNDERLAY_GATEWAY_$(UNDERLAY_VAR_SUFFIX)) \
+		EXCLUDE_IPS=$(TALOS_UNDERLAY_EXCLUDE_IPS_$(UNDERLAY_VAR_SUFFIX)) \
+		$(MAKE) talos-install

talos/machine-patch.yaml.j2

@@ -25,7 +25,7 @@ machine:
       - deviceSelector:
           busPath: 0000:00:05.1
         mtu: 1500
-        dhcp: false
+        ignore: true
   kubelet:
     nodeIP:
       validSubnets: