fix: cniprovisioner: remove stale chassis-id from tenant cluster node

Detect when the local OVS system-id differs from the host node's
k8s.ovn.org/node-chassis-id annotation and remove the stale annotation

This lets the ovnkube-node to correctly set the value later in case of dpu
reprovision etc.

Also mount host-cluster credentials into cniprovisioner so it has the credentials
to patch the host cluster node directly.

Fixes: RM#4933365

Signed-off-by: Hareesh Puthalath <hareeshp@nvidia.com>

Author: hareeshpc

dpf-utils/cmd/dpucniprovisioner/main.go

@@ -27,6 +27,7 @@ import (
 	"os"
 	"os/signal"
 	"strconv"
+	"strings"
 	"sync"
 
 	"github.com/nvidia/doca-platform/pkg/ipallocator"
@@ -37,9 +38,12 @@ import (
 
 	"github.com/vishvananda/netlink"
 	"k8s.io/client-go/kubernetes"
+	k8sscheme "k8s.io/client-go/kubernetes/scheme"
+	"k8s.io/client-go/rest"
 	"k8s.io/klog/v2"
 	"k8s.io/utils/clock"
 	kexec "k8s.io/utils/exec"
+	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/client/config"
 )
 
@@ -50,6 +54,10 @@ const (
 	// pfIPAllocationFilePath is the path to the file that contains the PF IP allocation done by the IP Allocator.
 	// We should ensure that the IP Allocation request name is pf to have this file created correctly.
 	pfIPAllocationFilePath = "/tmp/ips/pf"
+	// hostClusterTokenFilePath is where cniprovisioner expects the host-cluster token to be mounted.
+	hostClusterTokenFilePath = "/host-cluster-access/token"
+	// hostClusterCAFilePath is where cniprovisioner expects the host-cluster CA bundle to be mounted.
+	hostClusterCAFilePath = "/host-cluster-access/ca.crt"
 )
 
 func main() {
@@ -129,6 +137,11 @@ func main() {
 
 	provisioner := dpucniprovisioner.New(ctx, mode, c, ovsClient, networkhelper.New(), exec, clientset, vtepIPNet, gateway, vtepCIDR, hostCIDR, pfIPNet, node, gatewayDiscoveryNetwork, ovnMTU)
 	provisioner.K8sAPIServer = os.Getenv("K8S_APISERVER")
+	hostClusterClient, err := newHostClusterClient(provisioner.K8sAPIServer)
+	if err != nil {
+		klog.Fatal(err)
+	}
+	provisioner.SetHostKubernetesClient(hostClusterClient)
 
 	err = provisioner.RunOnce()
 	if err != nil {
@@ -246,6 +259,40 @@ func getHostCIDR() (*net.IPNet, error) {
 	return hostCIDR, nil
 }
 
+func newHostClusterClient(apiServer string) (client.Client, error) {
+	if strings.TrimSpace(apiServer) == "" {
+		return nil, errors.New("K8S_APISERVER must be set to initialize host-cluster access")
+	}
+
+	if _, err := os.Stat(hostClusterTokenFilePath); err != nil {
+		if os.IsNotExist(err) {
+			return nil, fmt.Errorf("missing host-cluster access token at %s; required to reconcile host node chassis annotations", hostClusterTokenFilePath)
+		}
+		return nil, fmt.Errorf("error while checking host cluster token file %s: %w", hostClusterTokenFilePath, err)
+	}
+	if _, err := os.Stat(hostClusterCAFilePath); err != nil {
+		if os.IsNotExist(err) {
+			return nil, fmt.Errorf("missing host-cluster CA bundle at %s; required to reconcile host node chassis annotations", hostClusterCAFilePath)
+		}
+		return nil, fmt.Errorf("error while checking host cluster CA file %s: %w", hostClusterCAFilePath, err)
+	}
+
+	hostConfig := &rest.Config{
+		Host:            apiServer,
+		BearerTokenFile: hostClusterTokenFilePath,
+		TLSClientConfig: rest.TLSClientConfig{
+			CAFile: hostClusterCAFilePath,
+		},
+	}
+
+	hostClient, err := client.New(hostConfig, client.Options{Scheme: k8sscheme.Scheme})
+	if err != nil {
+		return nil, fmt.Errorf("error while creating host cluster client: %w", err)
+	}
+
+	return hostClient, nil
+}
+
 // getGatewayDiscoveryNetwork returns the Network to be used by the provisioner to discover the gateway
 func getGatewayDiscoveryNetwork() (*net.IPNet, error) {
 	gatewayDiscoveryNetworkRaw := os.Getenv("GATEWAY_DISCOVERY_NETWORK")

dpf-utils/internal/cniprovisioner/dpu/provisioner.go

@@ -33,12 +33,15 @@ import (
 	"github.com/nvidia/ovn-kubernetes-components/internal/constants"
 	"github.com/nvidia/ovn-kubernetes-components/internal/utils/ovsclient"
 
+	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	k8stypes "k8s.io/apimachinery/pkg/types"
 	"k8s.io/client-go/kubernetes"
 	"k8s.io/klog/v2"
 	"k8s.io/utils/clock"
 	kexec "k8s.io/utils/exec"
 	"k8s.io/utils/ptr"
+	"sigs.k8s.io/controller-runtime/pkg/client"
 )
 
 type Mode string
@@ -90,16 +93,19 @@ const (
 	hostBootstrapKubeconfigPath = "/host-kubernetes/kubelet.conf"
 	// hostNodeNameFilePath is the path used to publish mapped host node name for other containers in the pod.
 	hostNodeNameFilePath = "/var/run/ovn-kubernetes/host-node-name"
+	// hostNodeChassisIDAnnotationKey is the host-cluster node annotation used by OVN to track the chassis identity.
+	hostNodeChassisIDAnnotationKey = "k8s.ovn.org/node-chassis-id"
 )
 
 type DPUCNIProvisioner struct {
-	ctx                       context.Context
-	clock                     clock.Clock
-	ensureConfigurationTicker clock.Ticker
-	ovsClient                 ovsclient.OVSClient
-	networkHelper             networkhelper.NetworkHelper
-	exec                      kexec.Interface
-	kubernetesClient          kubernetes.Interface
+	ctx                        context.Context
+	clock                      clock.Clock
+	ensureConfigurationTicker  clock.Ticker
+	ovsClient                  ovsclient.OVSClient
+	networkHelper              networkhelper.NetworkHelper
+	exec                       kexec.Interface
+	dpuClusterKubernetesClient kubernetes.Interface
+	hostKubernetesClient       client.Client
 
 	// FileSystemRoot controls the file system root. It's used for enabling easier testing of the package. Defaults to
 	// empty.
@@ -159,29 +165,33 @@ func New(ctx context.Context,
 	ovnMTU int,
 ) *DPUCNIProvisioner {
 	return &DPUCNIProvisioner{
-		ctx:                       ctx,
-		clock:                     clock,
-		ensureConfigurationTicker: clock.NewTicker(30 * time.Second),
-		ovsClient:                 ovsClient,
-		networkHelper:             networkHelper,
-		exec:                      exec,
-		kubernetesClient:          kubernetesClient,
-		FileSystemRoot:            "",
-		K8sAPIServer:              "",
-		BootstrapKubeconfigPath:   hostBootstrapKubeconfigPath,
-		HostNodeNameFilePath:      hostNodeNameFilePath,
-		vtepIPNet:                 vtepIPNet,
-		gateway:                   gateway,
-		vtepCIDR:                  vtepCIDR,
-		hostCIDR:                  hostCIDR,
-		pfIP:                      pfIP,
-		dpuHostName:               dpuHostName,
-		mode:                      mode,
-		gatewayDiscoveryNetwork:   gatewayDiscoveryNetwork,
-		ovnMTU:                    ovnMTU,
+		ctx:                        ctx,
+		clock:                      clock,
+		ensureConfigurationTicker:  clock.NewTicker(30 * time.Second),
+		ovsClient:                  ovsClient,
+		networkHelper:              networkHelper,
+		exec:                       exec,
+		dpuClusterKubernetesClient: kubernetesClient,
+		FileSystemRoot:             "",
+		K8sAPIServer:               "",
+		BootstrapKubeconfigPath:    hostBootstrapKubeconfigPath,
+		HostNodeNameFilePath:       hostNodeNameFilePath,
+		vtepIPNet:                  vtepIPNet,
+		gateway:                    gateway,
+		vtepCIDR:                   vtepCIDR,
+		hostCIDR:                   hostCIDR,
+		pfIP:                       pfIP,
+		dpuHostName:                dpuHostName,
+		mode:                       mode,
+		gatewayDiscoveryNetwork:    gatewayDiscoveryNetwork,
+		ovnMTU:                     ovnMTU,
 	}
 }
 
+func (p *DPUCNIProvisioner) SetHostKubernetesClient(c client.Client) {
+	p.hostKubernetesClient = c
+}
+
 // RunOnce runs the provisioning flow once and exits
 func (p *DPUCNIProvisioner) RunOnce() error {
 	if err := p.configure(); err != nil {
@@ -231,6 +241,9 @@ func (p *DPUCNIProvisioner) configure() error {
 	if err := p.writeHostIdentityBootstrapArtifacts(hostName); err != nil {
 		return fmt.Errorf("error while writing host identity bootstrap artifacts: %w", err)
 	}
+	if err := p.reconcileHostNodeChassisID(hostName); err != nil {
+		return fmt.Errorf("error while reconciling host node chassis annotation: %w", err)
+	}
 
 	if p.mode == ExternalIPAM {
 		klog.Info("Configuring br-ovn")
@@ -257,9 +270,46 @@ func (p *DPUCNIProvisioner) configure() error {
 	return nil
 }
 
+// reconcileHostNodeChassisID removes a stale host-cluster node chassis annotation when it differs from the local OVS
+// system-id. This allows ovnkube-node to re-register after DPU reprovisioning.
+func (p *DPUCNIProvisioner) reconcileHostNodeChassisID(hostName string) error {
+	systemID, err := p.ovsClient.GetSystemID()
+	if err != nil {
+		return fmt.Errorf("error while reading local OVS system-id: %w", err)
+	}
+	if systemID == "" {
+		return fmt.Errorf("OVS system-id is empty for DPU node %s (host node %s)", p.dpuHostName, hostName)
+	}
+
+	node := &corev1.Node{}
+	if err := p.hostKubernetesClient.Get(p.ctx, k8stypes.NamespacedName{Name: hostName}, node); err != nil {
+		return fmt.Errorf("error while getting host cluster node %s: %w", hostName, err)
+	}
+
+	current := strings.TrimSpace(node.Annotations[hostNodeChassisIDAnnotationKey])
+	switch {
+	case current == "":
+		klog.Infof("Host cluster node %s has no %s annotation; no cleanup needed", hostName, hostNodeChassisIDAnnotationKey)
+		return nil
+	case current == systemID:
+		klog.Infof("Host cluster node %s already has matching %s=%s", hostName, hostNodeChassisIDAnnotationKey, systemID)
+		return nil
+	}
+
+	klog.Infof("Removing stale %s=%s from host cluster node %s to allow reprovisioned DPU system-id %s to register", hostNodeChassisIDAnnotationKey, current, hostName, systemID)
+	base := node.DeepCopy()
+	delete(node.Annotations, hostNodeChassisIDAnnotationKey)
+	if err := p.hostKubernetesClient.Patch(p.ctx, node, client.MergeFromWithOptions(base, client.MergeFromWithOptimisticLock{})); err != nil {
+		return fmt.Errorf("error while removing stale %s annotation from host node %s: %w", hostNodeChassisIDAnnotationKey, hostName, err)
+	}
+	klog.Infof("Removed stale %s=%s from host cluster node %s", hostNodeChassisIDAnnotationKey, current, hostName)
+
+	return nil
+}
+
 // findAndSetKubernetesHostNameInOVS discovers and sets the Kubernetes Host Name in OVS
 func (p *DPUCNIProvisioner) findAndSetKubernetesHostNameInOVS() (string, error) {
-	nodeClient := p.kubernetesClient.CoreV1().Nodes()
+	nodeClient := p.dpuClusterKubernetesClient.CoreV1().Nodes()
 	n, err := nodeClient.Get(p.ctx, p.dpuHostName, metav1.GetOptions{})
 	if err != nil {
 		return "", fmt.Errorf("error while getting Kubernetes Node: %w", err)