add controller-driven kube-vip load balancer for K3S_BASE clusters

This PR implements controller-driven Kubernetes LoadBalancer services
for CLUSTER_TYPE_K3S_BASE in eve-k.

  - pkg/pillar/types/clustertypes.go: Add LBInterfaceConfig (interface +
    CIDR string) and LBInterfaces []LBInterfaceConfig to both
    EdgeNodeClusterConfig and EdgeNodeClusterStatus.
  - pkg/pillar/cmd/zedagent/parseconfig.go: Parse LoadBalancerService
    from the controller proto and populate
    EdgeNodeClusterConfig.LBInterfaces (K3S_BASE only; first
    interface/CIDR entry applied).
  - pkg/pillar/cmd/zedkube/clusterstatus.go: Relay LBInterfaces from
    EdgeNodeClusterConfig into EdgeNodeClusterStatus on the bootstrap
    node only; non-bootstrap nodes publish an empty list so they do not
    trigger kube-vip setup.
  - pkg/pillar/dpcmanager/dns.go: Filter kube-vip VIPs out of
    DeviceNetworkStatus.AddrInfoList using the LBInterfaces CIDR range,
    preventing VIPs from being used as source addresses for
    controller-bound traffic.
  - pkg/kube/cluster-init.sh: Add check_kubevip_lb loop that reads
    EdgeNodeClusterStatus JSON each iteration and calls kubevip-apply.sh
    or kubevip-delete.sh when the LB config changes. Persists
    last-applied state to avoid redundant re-applies across restarts.
  - pkg/kube/kubevip-apply.sh / kubevip-delete.sh: Scripts to
    install/remove the kube-vip DaemonSet and kube-vip-cloud-provider
    Deployment, configuring the IP pool via a kubevip ConfigMap.
  - pkg/kube/kubevip-ds.yaml: kube-vip DaemonSet manifest (ARP mode,
    control-plane nodes).
  - pkg/kube/config.yaml: Disable k3s built-in ServiceLB (servicelb) and
    Traefik for K3S_BASE — kube-vip replaces ServiceLB; users bring
    their own ingress.
  - pkg/pillar/docs/zedkube.md: Document the feature with an overview
    diagram, data-flow, EVE-API proto, and DeviceNetworkStatus filtering
    notes.

Signed-off-by: naiming-zededa <naiming@zededa.com>

Author: naiming-zededa

pkg/kube/cluster-init.sh

@@ -847,6 +847,51 @@ monitor_cluster_config_change() {
     done
 }
 
+# Persistent state file for kube-vip: stores "<iface> <cidr>" of the last
+# successfully applied config so that container/device restarts do not
+# trigger a redundant re-apply (which causes "configmaps kubevip already
+# exists" errors from kube-vip-cloud-provider).
+KUBEVIP_STATE_FILE="/var/lib/kubevip-applied"
+
+# check_kubevip_lb is called once per main loop iteration. It reads the
+# EdgeNodeClusterStatus published by zedkube and applies or removes the kube-vip
+# load balancer configuration when LBInterfaces[0] changes. On the bootstrap
+# node this field is set by zedkube; on other nodes it is empty so no action
+# is taken. prev_lb_state is seeded from KUBEVIP_STATE_FILE on first call so
+# that restarts skip re-applying an already-configured kube-vip.
+prev_lb_state=""
+check_kubevip_lb() {
+    # Seed in-memory state from the persistent file on first call.
+    if [ -z "$prev_lb_state" ] && [ -f "$KUBEVIP_STATE_FILE" ]; then
+        prev_lb_state=$(cat "$KUBEVIP_STATE_FILE")
+    fi
+    local lb_iface="" lb_cidr="" lb=""
+    if [ -f "$enc_status_file" ]; then
+        enc_data=$(cat "$enc_status_file")
+        lb_iface=$(echo "$enc_data" | jq -r '.LBInterfaces[0].Interface // ""')
+        lb_cidr=$(echo "$enc_data" | jq -r '.LBInterfaces[0].IPPrefix // ""')
+    fi
+    lb=$(printf '%s %s' "$lb_iface" "$lb_cidr" | xargs)  # trim whitespace
+    if [ "$lb" = "$prev_lb_state" ]; then
+        return
+    fi
+    if [ -n "$lb_iface" ] && [ -n "$lb_cidr" ]; then
+        logmsg "check_kubevip_lb: applying kube-vip with iface=$lb_iface cidr=$lb_cidr"
+        if /usr/bin/kubevip-apply.sh "$lb_iface" "$lb_cidr"; then
+            prev_lb_state="$lb"
+            echo "$prev_lb_state" > "$KUBEVIP_STATE_FILE"
+        fi
+    elif [ -n "$prev_lb_state" ]; then
+        logmsg "check_kubevip_lb: removing kube-vip"
+        if /usr/bin/kubevip-delete.sh; then
+            prev_lb_state="$lb"
+            rm -f "$KUBEVIP_STATE_FILE"
+        fi
+    else
+        prev_lb_state="$lb"
+    fi
+}
+
 # started when we detect registration addition
 # start cleaning up some components
 # these are cluster-wide operations, only one nodes initiates it
@@ -928,7 +973,7 @@ EOF
     elif [ $cluster_type -eq $CLUSTER_TYPE_REPLICATED_STORAGE ]; then
             cp "${KUBE_MANIFESTS_SRC_DIR}/${K3S_CONFIG_FILE_DISABLE_LOCAL_PATH}" "${K3S_CONFIG_DIR}/${K3S_CONFIG_FILE_DISABLE_LOCAL_PATH}"
     elif [ $cluster_type -eq $CLUSTER_TYPE_K3S_BASE ]; then
-            rm "${K3S_CONFIG_DIR}/${K3S_CONFIG_FILE_DISABLE_LOCAL_PATH}"
+            rm -f "${K3S_CONFIG_DIR}/${K3S_CONFIG_FILE_DISABLE_LOCAL_PATH}"
     else
             logmsg "possible unhandled cluster type $cluster_type in (provision_cluster_config_file)"
     fi
@@ -1408,6 +1453,7 @@ fi
         check_log_file_size "containerd-user.log"
         check_kubeconfig_yaml_files
         check_and_remove_excessive_k3s_logs
+        check_kubevip_lb
         wait_for_item "wait"
         sleep 15
 done

pkg/kube/config.yaml

@@ -12,6 +12,9 @@ etcd-expose-metrics: true
 container-runtime-endpoint: "/run/containerd-user/containerd.sock"
 disable-network-policy: true
 disable-cloud-controller: true
+disable:
+  - servicelb
+  - traefik
 kubelet-arg:
   - "node-status-update-frequency=2s"
   - "cpu-manager-policy=static"

pkg/kube/kubevip-apply.sh

@@ -6,7 +6,6 @@
 # kubevip-apply.sh
 # This script creates a Kube-VIP ConfigMap with the specified interface and CIDR range
 # and applies the necessary Kubernetes resources.
-# This script is for testing only, not for production use.
 
 # Function to display usage information
 show_usage() {
@@ -15,7 +14,6 @@ show_usage() {
     echo ""
     echo "This script creates a Kube-VIP ConfigMap with the specified interface and CIDR range"
     echo "and applies the necessary Kubernetes resources."
-    echo "This script is for testing only, not for production use."
 }
 
 # Check if we have two arguments
@@ -53,10 +51,8 @@ metadata:
   namespace: kube-system
 data:
   # Global settings for all LoadBalancer services
-  cidr-default: "${CIDR}" # Default CIDR for LoadBalancer services
-  cidr-global: "${CIDR}" # Default CIDR for LoadBalancer services
-  interface-default: "${INTERFACE}" # All global LoadBalancer IPs will be advertised on ${INTERFACE}
-  interface-global: "${INTERFACE}" # All global LoadBalancer IPs will be advertised on ${INTERFACE}
+  cidr-global: "${CIDR}"
+  interface-global: "${INTERFACE}"
 EOF
 
 echo "Created Kube-VIP ConfigMap with interface ${INTERFACE} and CIDR ${CIDR}"

pkg/kube/kubevip-delete.sh

@@ -5,7 +5,6 @@
 #
 # kubevip-delete.sh
 # This script deletes the Kube-VIP ConfigMap and associated resources.
-# This script is for testing only, not for production use.
 
 if kubectl delete -f /etc/kubevip-ds.yaml && \
    kubectl delete -f /etc/kubevip-cm.yaml && \

pkg/kube/kubevip-ds.yaml

@@ -8,7 +8,7 @@ kind: DaemonSet
 metadata:
   labels:
     app.kubernetes.io/name: kube-vip-ds
-    app.kubernetes.io/version: v0.8.9
+    app.kubernetes.io/version: v1.1.0
   name: kube-vip-ds
   namespace: kube-system
 spec:
@@ -19,7 +19,7 @@ spec:
     metadata:
       labels:
         app.kubernetes.io/name: kube-vip-ds
-        app.kubernetes.io/version: v0.8.9
+        app.kubernetes.io/version: v1.1.0
     spec:
       affinity:
         nodeAffinity:
@@ -69,7 +69,7 @@ spec:
               value: "2"
             - name: prometheus_server
               value: 0.0.0.0:2112
-          image: ghcr.io/kube-vip/kube-vip:v0.8.9
+          image: ghcr.io/kube-vip/kube-vip:v1.1.0
           imagePullPolicy: IfNotPresent
           name: kube-vip
           resources: {}

pkg/kube/kubevirt-utils.sh

@@ -24,6 +24,9 @@ Kubevirt_install() {
 # Uses a targeted merge patch on developerConfiguration.featureGates only, so that
 # permittedHostDevices (PCIe/GPU/USB passthrough) is never disturbed.
 Kubevirt_migrate_feature_gates() {
+    if [ -f /var/lib/base-k3s-mode ]; then
+        return 0
+    fi
     current_gates=$(kubectl get kubevirt kubevirt -n kubevirt \
         -o jsonpath='{.spec.configuration.developerConfiguration.featureGates[*]}' 2>/dev/null)
     if echo "$current_gates" | tr ' ' '\n' | grep -qx "CPUManager"; then

pkg/pillar/cmd/zedagent/parseconfig.go

@@ -3513,6 +3513,25 @@ func parseEdgeNodeClusterConfig(getconfigCtx *getconfigContext,
 		enClusterConfig.TieBreakerNodeID = types.UUIDandVersion{UUID: tieBreakerNodeID}
 	}
 
+	// Parse LoadBalancerService — only supported for ClusterTypeK3sBase clusters.
+	// Each proto interface contributes one LBInterfaceConfig entry using its first CIDR.
+	lbSvc := zcfgCluster.GetLoadBalancerService()
+	if lbSvc != nil && enClusterConfig.ClusterType == types.ClusterTypeK3sBase {
+		for _, iface := range lbSvc.GetInterfaces() {
+			ifName := iface.GetInterfaceName()
+			cidrs := iface.GetAddressCidrs()
+			if ifName == "" || len(cidrs) == 0 {
+				continue
+			}
+			if _, _, lbErr := net.ParseCIDR(cidrs[0]); lbErr != nil {
+				log.Errorf("parseEdgeNodeClusterConfig: invalid LB CIDR %s: %v", cidrs[0], lbErr)
+				continue
+			}
+			enClusterConfig.LBInterfaces = append(enClusterConfig.LBInterfaces,
+				types.LBInterfaceConfig{Interface: ifName, IPPrefix: cidrs[0]})
+		}
+	}
+
 	log.Functionf("parseEdgeNodeClusterConfig: ENCluster API, Config %+v, %v", zcfgCluster, enClusterConfig)
 	ctx.pubEdgeNodeClusterConfig.Publish("global", enClusterConfig)
 }

pkg/pillar/cmd/zedkube/clusterstatus.go

@@ -156,6 +156,12 @@ func (z *zedkube) publishKubeConfigStatus() {
 		log.Errorf("publishKubeConfigStatus: cluster token is not from configitme or encrypted")
 	}
 
+	// Only the bootstrap node manages kube-vip load balancing; other nodes leave
+	// LBInterfaces empty so cluster-init.sh does not apply kubevip.
+	if z.clusterConfig.BootstrapNode {
+		status.LBInterfaces = z.clusterConfig.LBInterfaces
+	}
+
 	// publish the cluster status for the kube container
 	log.Functionf("publishKubeConfigStatus: publishing")
 	z.pubEdgeNodeClusterStatus.Publish("global", status)

pkg/pillar/cmd/zedkube/kubeservice.go

@@ -58,10 +58,14 @@ func (z *zedkube) collectKubeSvcs() {
 		// Continue anyway, we might have some services
 	}
 
+	// Collect kube-vip load balancer pool status if kubevip is deployed
+	lbPoolStatus := collectLBPoolStatus(clientset, serviceInfoList)
+
 	// Create new KubeUserServices struct with collected data
 	newKubeUserServices := types.KubeUserServices{
-		UserService: serviceInfoList,
-		UserIngress: ingressInfoList,
+		UserService:  serviceInfoList,
+		UserIngress:  ingressInfoList,
+		LBPoolStatus: lbPoolStatus,
 	}
 
 	// Get previous published data to compare
@@ -323,6 +327,38 @@ func (z *zedkube) GetAllKubeIngresses(clientset *kubernetes.Clientset, serviceIn
 	return ingressInfoList, nil
 }
 
+// collectLBPoolStatus reads the kubevip ConfigMap from kube-system to get the configured
+// load balancer pool, and gathers IPs currently allocated to LoadBalancer-type services.
+// Returns nil if the kubevip ConfigMap does not exist (kubevip not yet deployed).
+func collectLBPoolStatus(clientset *kubernetes.Clientset, services []types.KubeServiceInfo) *types.KubeLBPoolStatus {
+	cm, err := clientset.CoreV1().ConfigMaps("kube-system").Get(
+		context.Background(), "kubevip", metav1.GetOptions{})
+	if err != nil {
+		// kubevip ConfigMap not present — not deployed yet
+		return nil
+	}
+
+	iface := cm.Data["interface-global"]
+	cidr := cm.Data["cidr-global"]
+	if iface == "" || cidr == "" {
+		return nil
+	}
+
+	var allocatedIPs []string
+	for _, svc := range services {
+		if svc.Type == corev1.ServiceTypeLoadBalancer && svc.LoadBalancerIP != "" {
+			allocatedIPs = append(allocatedIPs, svc.LoadBalancerIP)
+		}
+	}
+	sort.Strings(allocatedIPs)
+
+	return &types.KubeLBPoolStatus{
+		Interface:    iface,
+		IPPrefix:     cidr,
+		AllocatedIPs: allocatedIPs,
+	}
+}
+
 // isDeviceInterfaceIP checks if the given IP is assigned to any of the device's network interfaces
 func (z *zedkube) isDeviceInterfaceIP(ipStr string) bool {
 	ip := net.ParseIP(ipStr)