feat: RDMA, SR-IOV, Multus auto-restart pods on config changes (#2035)

Fixes #1831 

Adds automatic pod restart when device plugin ConfigMap content changes.
A SHA256 hash of the config is added to the DaemonSet pod template
annotation (nvidia.network-operator.config-hash). When the config
changes, the hash changes, triggering a Kubernetes rolling update.
Changes:
- pkg/utils/utils.go - Added GetStringHash() function
- pkg/consts/consts.go - Added ConfigHashAnnotation constant
- pkg/state/state_skel.go - Added SetConfigHashAnnotation() helper
- pkg/state/state_rdma_shared_device_plugin.go - Uses config hash
- pkg/state/state_sriov_dp.go - Uses config hash
- pkg/state/state_multus_cni.go - Uses config hash
- Unit tests added

Author: rollandf

pkg/consts/consts.go

@@ -50,4 +50,6 @@ const (
 	OfedDriverSkipDrainLabelSelector = "nvidia.com/ofed-driver-upgrade-drain.skip!=true"
 	// ControllerRevisionAnnotation is the key for annotations used to store revision information on Kubernetes objects.
 	ControllerRevisionAnnotation = "nvidia.network-operator.revision"
+	// ConfigHashAnnotation is the key for annotations used to store config hash on pod templates.
+	ConfigHashAnnotation = "nvidia.network-operator.config-hash"
 )

pkg/state/state_multus_cni.go

@@ -165,6 +165,13 @@ func (s *stateMultusCNI) GetManifestObjects(
 		return nil, errors.Wrap(err, "failed to render objects")
 	}
 
+	if cr.Spec.SecondaryNetwork.Multus.Config != nil {
+		configHash := utils.GetStringHash(*cr.Spec.SecondaryNetwork.Multus.Config)
+		if err := SetConfigHashAnnotation(objs, configHash); err != nil {
+			return nil, errors.Wrap(err, "failed to set config hash annotation")
+		}
+	}
+
 	reqLogger.V(consts.LogLevelDebug).Info("Rendered", "objects:", objs)
 	return objs, nil
 }

pkg/state/state_rdma_shared_device_plugin.go

@@ -147,6 +147,7 @@ func (s *stateRDMASharedDevicePlugin) GetManifestObjects(
 	if clusterInfo == nil {
 		return nil, errors.New("clusterInfo provider required")
 	}
+
 	renderData := &stateRDMASharedDevicePluginManifestRenderData{
 		CrSpec:              cr.Spec.RdmaSharedDevicePlugin,
 		Tolerations:         cr.Spec.Tolerations,
@@ -164,6 +165,14 @@ func (s *stateRDMASharedDevicePlugin) GetManifestObjects(
 	if err != nil {
 		return nil, errors.Wrap(err, "failed to render objects")
 	}
+
+	if cr.Spec.RdmaSharedDevicePlugin.Config != nil {
+		configHash := utils.GetStringHash(*cr.Spec.RdmaSharedDevicePlugin.Config)
+		if err := SetConfigHashAnnotation(objs, configHash); err != nil {
+			return nil, errors.Wrap(err, "failed to set config hash annotation")
+		}
+	}
+
 	reqLogger.V(consts.LogLevelDebug).Info("Rendered", "objects:", objs)
 	return objs, nil
 }

pkg/state/state_skel.go

@@ -471,3 +471,33 @@ func (s *stateSkel) isDaemonSetReady(uds *unstructured.Unstructured, reqLogger l
 func (s *stateSkel) SetRenderer(renderer render.Renderer) {
 	s.renderer = renderer
 }
+
+// SetConfigHashAnnotation sets config hash annotation on DaemonSet pod templates.
+func SetConfigHashAnnotation(objs []*unstructured.Unstructured, configHash string) error {
+	if configHash == "" {
+		return nil
+	}
+
+	for _, obj := range objs {
+		if obj.GetKind() != "DaemonSet" {
+			continue
+		}
+
+		annotations, found, err := unstructured.NestedStringMap(obj.Object,
+			"spec", "template", "metadata", "annotations")
+		if err != nil {
+			return errors.Wrap(err, "failed to get pod template annotations")
+		}
+		if !found || annotations == nil {
+			annotations = make(map[string]string)
+		}
+
+		annotations[consts.ConfigHashAnnotation] = configHash
+
+		if err := unstructured.SetNestedStringMap(obj.Object, annotations,
+			"spec", "template", "metadata", "annotations"); err != nil {
+			return errors.Wrap(err, "failed to set pod template annotations")
+		}
+	}
+	return nil
+}

pkg/state/state_skel_test.go

@@ -80,3 +80,135 @@ var _ = Describe("stateSkel", func() {
 		})
 	})
 })
+
+var _ = Describe("SetConfigHashAnnotation", func() {
+	Context("when configHash is empty", func() {
+		It("should not modify any objects", func() {
+			ds := createTestDaemonSet("test-ds")
+			objs := []*unstructured.Unstructured{ds}
+			err := SetConfigHashAnnotation(objs, "")
+			Expect(err).NotTo(HaveOccurred())
+
+			annotations, found, err := unstructured.NestedStringMap(ds.Object,
+				"spec", "template", "metadata", "annotations")
+			Expect(err).NotTo(HaveOccurred())
+			Expect(found).To(BeFalse())
+			Expect(annotations).To(BeNil())
+		})
+	})
+
+	Context("when configHash is provided", func() {
+		It("should add annotation to DaemonSet pod template", func() {
+			ds := createTestDaemonSet("test-ds")
+			objs := []*unstructured.Unstructured{ds}
+			configHash := "abc123hash"
+
+			err := SetConfigHashAnnotation(objs, configHash)
+			Expect(err).NotTo(HaveOccurred())
+
+			annotations, found, err := unstructured.NestedStringMap(ds.Object,
+				"spec", "template", "metadata", "annotations")
+			Expect(err).NotTo(HaveOccurred())
+			Expect(found).To(BeTrue())
+			Expect(annotations[consts.ConfigHashAnnotation]).To(Equal(configHash))
+		})
+
+		It("should preserve existing annotations", func() {
+			ds := createTestDaemonSet("test-ds")
+			// Add existing annotation
+			err := unstructured.SetNestedStringMap(ds.Object,
+				map[string]string{"existing": "annotation"},
+				"spec", "template", "metadata", "annotations")
+			Expect(err).NotTo(HaveOccurred())
+
+			objs := []*unstructured.Unstructured{ds}
+			configHash := "abc123hash"
+
+			err = SetConfigHashAnnotation(objs, configHash)
+			Expect(err).NotTo(HaveOccurred())
+
+			annotations, found, err := unstructured.NestedStringMap(ds.Object,
+				"spec", "template", "metadata", "annotations")
+			Expect(err).NotTo(HaveOccurred())
+			Expect(found).To(BeTrue())
+			Expect(annotations["existing"]).To(Equal("annotation"))
+			Expect(annotations[consts.ConfigHashAnnotation]).To(Equal(configHash))
+		})
+
+		It("should not modify non-DaemonSet objects", func() {
+			configMap := &unstructured.Unstructured{
+				Object: map[string]interface{}{
+					"apiVersion": "v1",
+					"kind":       "ConfigMap",
+					"metadata": map[string]interface{}{
+						"name":      "test-cm",
+						"namespace": "test-ns",
+					},
+				},
+			}
+			objs := []*unstructured.Unstructured{configMap}
+			configHash := "abc123hash"
+
+			err := SetConfigHashAnnotation(objs, configHash)
+			Expect(err).NotTo(HaveOccurred())
+
+			// ConfigMap should not have the annotation path
+			_, found, _ := unstructured.NestedStringMap(configMap.Object,
+				"spec", "template", "metadata", "annotations")
+			Expect(found).To(BeFalse())
+		})
+
+		It("should handle multiple DaemonSets", func() {
+			ds1 := createTestDaemonSet("test-ds-1")
+			ds2 := createTestDaemonSet("test-ds-2")
+			objs := []*unstructured.Unstructured{ds1, ds2}
+			configHash := "abc123hash"
+
+			err := SetConfigHashAnnotation(objs, configHash)
+			Expect(err).NotTo(HaveOccurred())
+
+			for _, ds := range objs {
+				annotations, found, err := unstructured.NestedStringMap(ds.Object,
+					"spec", "template", "metadata", "annotations")
+				Expect(err).NotTo(HaveOccurred())
+				Expect(found).To(BeTrue())
+				Expect(annotations[consts.ConfigHashAnnotation]).To(Equal(configHash))
+			}
+		})
+	})
+})
+
+func createTestDaemonSet(name string) *unstructured.Unstructured {
+	return &unstructured.Unstructured{
+		Object: map[string]interface{}{
+			"apiVersion": "apps/v1",
+			"kind":       "DaemonSet",
+			"metadata": map[string]interface{}{
+				"name":      name,
+				"namespace": "test-ns",
+			},
+			"spec": map[string]interface{}{
+				"selector": map[string]interface{}{
+					"matchLabels": map[string]interface{}{
+						"app": name,
+					},
+				},
+				"template": map[string]interface{}{
+					"metadata": map[string]interface{}{
+						"labels": map[string]interface{}{
+							"app": name,
+						},
+					},
+					"spec": map[string]interface{}{
+						"containers": []interface{}{
+							map[string]interface{}{
+								"name":  "test-container",
+								"image": "test-image",
+							},
+						},
+					},
+				},
+			},
+		},
+	}
+}

pkg/state/state_sriov_dp.go

@@ -149,6 +149,7 @@ func (s *stateSriovDp) GetManifestObjects(
 	if clusterInfo == nil {
 		return nil, errors.New("clusterInfo provider required")
 	}
+
 	renderData := &sriovDpManifestRenderData{
 		CrSpec:              cr.Spec.SriovDevicePlugin,
 		Tolerations:         cr.Spec.Tolerations,
@@ -166,6 +167,14 @@ func (s *stateSriovDp) GetManifestObjects(
 	if err != nil {
 		return nil, errors.Wrap(err, "failed to render objects")
 	}
+
+	if cr.Spec.SriovDevicePlugin.Config != nil {
+		configHash := utils.GetStringHash(*cr.Spec.SriovDevicePlugin.Config)
+		if err := SetConfigHashAnnotation(objs, configHash); err != nil {
+			return nil, errors.Wrap(err, "failed to set config hash annotation")
+		}
+	}
+
 	reqLogger.V(consts.LogLevelDebug).Info("Rendered", "objects:", objs)
 	return objs, nil
 }

pkg/utils/utils.go

@@ -18,6 +18,8 @@ limitations under the License.
 package utils
 
 import (
+	"crypto/sha256"
+	"encoding/base64"
 	"fmt"
 	"os"
 	"path/filepath"
@@ -91,3 +93,9 @@ func GetCniNetworkDirectory(staticInfo staticconfig.Provider, _ clustertype.Prov
 	}
 	return consts.DefaultCniNetworkDirectory
 }
+
+// GetStringHash returns a base64-encoded SHA256 hash of the input string.
+func GetStringHash(data string) string {
+	hash := sha256.Sum256([]byte(data))
+	return base64.StdEncoding.EncodeToString(hash[:])
+}

pkg/utils/utils_test.go

@@ -83,4 +83,33 @@ var _ = Describe("Utils tests", func() {
 			Expect(result).To(Equal(consts.DefaultCniNetworkDirectory))
 		})
 	})
+
+	Context("Testing GetStringHash", func() {
+		It("Should return consistent hash for same input", func() {
+			input := `{"resourceName": "rdma_shared_device_a"}`
+			hash1 := GetStringHash(input)
+			hash2 := GetStringHash(input)
+			Expect(hash1).To(Equal(hash2))
+		})
+
+		It("Should return different hash for different input", func() {
+			input1 := `{"resourceName": "rdma_shared_device_a"}`
+			input2 := `{"resourceName": "rdma_shared_device_b"}`
+			hash1 := GetStringHash(input1)
+			hash2 := GetStringHash(input2)
+			Expect(hash1).NotTo(Equal(hash2))
+		})
+
+		It("Should return non-empty hash for empty string", func() {
+			hash := GetStringHash("")
+			Expect(hash).NotTo(BeEmpty())
+		})
+
+		It("Should return valid base64 encoded string", func() {
+			input := `{"config": "test"}`
+			hash := GetStringHash(input)
+			// Base64 encoded SHA256 hash should be 44 characters
+			Expect(len(hash)).To(Equal(44))
+		})
+	})
 })