Fix: cluster-ip should not be part of the source ip for SendOnIntf

- fix an issue where in eve-k cluster mode, the cluster-intf can also be
  the management port, and the cluster-ip is in part of the AddrInfoList
  list on the interface to be used as the source ip address for
  communicating to the controller or northbound servers.
- this patch introduces a separate ClusterIPAddr for zedkube to use for
  kubernetes cluster purpose

Signed-off-by: naiming-zededa <naiming@zededa.com>

Author: naiming-zededa

pkg/pillar/cmd/zedagent/reportinfo.go

@@ -917,6 +917,9 @@ func encodeNetworkPortStatus(ctx *zedagentContext,
 	for _, ipAddr := range port.AddrInfoList {
 		devicePort.IPAddrs = append(devicePort.IPAddrs, ipAddr.Addr.String())
 	}
+	if port.ClusterIPAddr != nil {
+		devicePort.IPAddrs = append(devicePort.IPAddrs, port.ClusterIPAddr.String())
+	}
 	// devicePort.Gateway is deprecated - replaced by DefaultRouters
 	devicePort.DefaultRouters = utils.ToStrings(port.DefaultRouters)
 	// devicePort.DNSServers is deprecated - replaced by Dns

pkg/pillar/cmd/zedkube/clusterstatus.go

@@ -211,13 +211,9 @@ func (z *zedkube) updateClusterIPReadiness() (changed bool) {
 			if port.Logicallabel != z.clusterConfig.ClusterInterface {
 				continue
 			}
-			for _, addr := range port.AddrInfoList {
-				if addr.Addr.Equal(z.clusterConfig.ClusterIPPrefix.IP) {
-					ready = true
-					break
-				}
-			}
-			if ready {
+			if port.ClusterIPAddr != nil &&
+				port.ClusterIPAddr.Equal(z.clusterConfig.ClusterIPPrefix.IP) {
+				ready = true
 				break
 			}
 		}
@@ -251,7 +247,6 @@ func (z *zedkube) startClusterStatusServer() {
 	mux.HandleFunc("/cluster-app/", func(w http.ResponseWriter, r *http.Request) {
 		z.clusterAppIDHandler(w, r)
 	})
-
 	serverAddr := net.JoinHostPort(
 		z.clusterConfig.ClusterIPPrefix.IP.String(), types.ClusterStatusPort)
 	z.statusServer = &http.Server{

pkg/pillar/cmd/zedkube/kubeservice.go

@@ -333,7 +333,7 @@ func (z *zedkube) isDeviceInterfaceIP(ipStr string) bool {
 
 	// Check each port in deviceNetworkStatus
 	for _, port := range z.deviceNetworkStatus.Ports {
-		// Check each address in the port's AddrInfoList
+		// Check regular addresses, and not include the ClusterIPAddr
 		for _, addrInfo := range port.AddrInfoList {
 			if addrInfo.Addr.Equal(ip) {
 				log.Functionf("Found matching interface IP %s on port %s", ipStr, port.IfName)

pkg/pillar/dpcmanager/dns.go

@@ -138,10 +138,12 @@ func (m *DpcManager) updateDNS() {
 		// Below this point we collect L3-specific info for the port.
 		if !port.IsL3Port {
 			m.deviceNetStatus.Ports[ix].AddrInfoList = nil
+			m.deviceNetStatus.Ports[ix].ClusterIPAddr = nil
 			continue
 		}
 
 		addrInfoList := make([]types.AddrInfo, 0, len(ipAddrs))
+		var clusterIPAddr net.IP
 		if len(ipAddrs) == 0 {
 			m.Log.Functionf("updateDNS: interface %s has NO IP addresses", port.IfName)
 		}
@@ -151,9 +153,18 @@ func (m *DpcManager) updateDNS() {
 				// IP address received over DHCP is ignored.
 				continue
 			}
+			// Cluster IP is kept separate from AddrInfoList — it is only
+			// for inter-node communication and must not be used as a source
+			// IP for controller-bound traffic.
+			if m.clusterStatus.ClusterIPPrefix != nil &&
+				m.clusterStatus.ClusterIPPrefix.Contains(addr.IP) {
+				clusterIPAddr = addr.IP
+				continue
+			}
 			addrInfoList = append(addrInfoList, types.AddrInfo{Addr: addr.IP})
 		}
 		m.deviceNetStatus.Ports[ix].AddrInfoList = addrInfoList
+		m.deviceNetStatus.Ports[ix].ClusterIPAddr = clusterIPAddr
 
 		// Get DNS etc info from dhcpcd. Updates DomainName and DNSServers.
 		err = m.getDHCPInfo(&m.deviceNetStatus.Ports[ix], port)

pkg/pillar/types/dns.go

@@ -59,6 +59,7 @@ type NetworkPortStatus struct {
 	DNSServers       []net.IP                // from DHCP + static combined (with Gateway as fallback)
 	NtpServers       []netutils.HostnameOrIP // from DHCP + static combined
 	AddrInfoList     []AddrInfo              // from DHCP + static combined
+	ClusterIPAddr    net.IP                  // ClusterIPAddr is the cluster IP address assigned to this port
 	DefaultRouters   []net.IP                // from DHCP + static combined
 	Up               bool
 	MacAddr          net.HardwareAddr
@@ -262,6 +263,9 @@ func (status DeviceNetworkStatus) MostlyEqual(status2 DeviceNetworkStatus) bool
 			}) {
 			return false
 		}
+		if !p1.ClusterIPAddr.Equal(p2.ClusterIPAddr) {
+			return false
+		}
 		if p1.Up != p2.Up ||
 			!bytes.Equal(p1.MacAddr, p2.MacAddr) {
 			return false