Changes for enabling ovnkube identity

dpf-utils/cmd/dpucniprovisioner/main.go

@@ -128,6 +128,7 @@ func main() {
 	}
 
 	provisioner := dpucniprovisioner.New(ctx, mode, c, ovsClient, networkhelper.New(), exec, clientset, vtepIPNet, gateway, vtepCIDR, hostCIDR, pfIPNet, node, gatewayDiscoveryNetwork, ovnMTU)
+	provisioner.K8sAPIServer = os.Getenv("K8S_APISERVER")
 
 	err = provisioner.RunOnce()
 	if err != nil {

dpf-utils/internal/cniprovisioner/dpu/provisioner.go

@@ -25,6 +25,7 @@ import (
 	"os"
 	"path/filepath"
 	"strconv"
+	"strings"
 	"time"
 
 	"github.com/nvidia/doca-platform/pkg/utils/networkhelper"
@@ -84,6 +85,10 @@ const (
 	// netplanApplyCooldownDuration determines the cooldown period of a successful netplan apply command before a
 	// subsequent netplan apply command is executed.
 	netplanApplyCooldownDuration = time.Minute * 2
+	// hostBootstrapKubeconfigPath is the path where bootstrap kubeconfig is written for ovnkube identity.
+	hostBootstrapKubeconfigPath = "/host-kubernetes/kubelet.conf"
+	// hostNodeNameFilePath is the path used to publish mapped host node name for other containers in the pod.
+	hostNodeNameFilePath = "/var/run/ovn-kubernetes/host-node-name"
 )
 
 type DPUCNIProvisioner struct {
@@ -98,6 +103,13 @@ type DPUCNIProvisioner struct {
 	// FileSystemRoot controls the file system root. It's used for enabling easier testing of the package. Defaults to
 	// empty.
 	FileSystemRoot string
+	// K8sAPIServer is the host cluster API server endpoint used in the generated bootstrap kubeconfig.
+	// Leave empty to skip bootstrap kubeconfig generation.
+	K8sAPIServer string
+	// BootstrapKubeconfigPath is where the bootstrap kubeconfig is written. Defaults to hostBootstrapKubeconfigPath.
+	BootstrapKubeconfigPath string
+	// HostNodeNameFilePath is where the mapped host node name is written. Defaults to hostNodeNameFilePath.
+	HostNodeNameFilePath string
 
 	// vtepIPNet is the IP that should be added to the VTEP interface.
 	vtepIPNet *net.IPNet
@@ -154,6 +166,9 @@ func New(ctx context.Context,
 		exec:                      exec,
 		kubernetesClient:          kubernetesClient,
 		FileSystemRoot:            "",
+		K8sAPIServer:              "",
+		BootstrapKubeconfigPath:   hostBootstrapKubeconfigPath,
+		HostNodeNameFilePath:      hostNodeNameFilePath,
 		vtepIPNet:                 vtepIPNet,
 		gateway:                   gateway,
 		vtepCIDR:                  vtepCIDR,
@@ -208,9 +223,13 @@ func (p *DPUCNIProvisioner) EnsureConfiguration() {
 // configure runs the provisioning flow once
 func (p *DPUCNIProvisioner) configure() error {
 	klog.Info("Configuring Kubernetes host name in OVS")
-	if err := p.findAndSetKubernetesHostNameInOVS(); err != nil {
+	hostName, err := p.findAndSetKubernetesHostNameInOVS()
+	if err != nil {
 		return fmt.Errorf("error while setting the Kubernetes Host Name in OVS: %w", err)
 	}
+	if err := p.writeHostIdentityBootstrapArtifacts(hostName); err != nil {
+		return fmt.Errorf("error while writing host identity bootstrap artifacts: %w", err)
+	}
 
 	if p.mode == ExternalIPAM {
 		klog.Info("Configuring br-ovn")
@@ -238,22 +257,83 @@ func (p *DPUCNIProvisioner) configure() error {
 }
 
 // findAndSetKubernetesHostNameInOVS discovers and sets the Kubernetes Host Name in OVS
-func (p *DPUCNIProvisioner) findAndSetKubernetesHostNameInOVS() error {
+func (p *DPUCNIProvisioner) findAndSetKubernetesHostNameInOVS() (string, error) {
 	nodeClient := p.kubernetesClient.CoreV1().Nodes()
 	n, err := nodeClient.Get(p.ctx, p.dpuHostName, metav1.GetOptions{})
 	if err != nil {
-		return fmt.Errorf("error while getting Kubernetes Node: %w", err)
+		return "", fmt.Errorf("error while getting Kubernetes Node: %w", err)
 	}
 	hostName, ok := n.Labels[constants.HostNameDPULabelKey]
 	if !ok {
-		return fmt.Errorf("required label %s is not set on node %s in the DPU cluster", constants.HostNameDPULabelKey, p.dpuHostName)
+		return "", fmt.Errorf("required label %s is not set on node %s in the DPU cluster", constants.HostNameDPULabelKey, p.dpuHostName)
+	}
+	hostName = strings.TrimSpace(hostName)
+	if hostName == "" {
+		return "", fmt.Errorf("label %s on node %s cannot be empty", constants.HostNameDPULabelKey, p.dpuHostName)
 	}
 
 	if err := p.ovsClient.SetKubernetesHostNodeName(hostName); err != nil {
-		return fmt.Errorf("error while setting the Kubernetes Host Name in OVS: %w", err)
+		return "", fmt.Errorf("error while setting the Kubernetes Host Name in OVS: %w", err)
 	}
 	if err := p.ovsClient.SetHostName(hostName); err != nil {
-		return fmt.Errorf("error while setting the hostname external ID in OVS: %w", err)
+		return "", fmt.Errorf("error while setting the hostname external ID in OVS: %w", err)
+	}
+	return hostName, nil
+}
+
+// writeHostIdentityBootstrapArtifacts writes host identity data for other pod containers.
+// It writes artifacts only when K8sAPIServer is set, which enables the bootstrap flow.
+func (p *DPUCNIProvisioner) writeHostIdentityBootstrapArtifacts(hostName string) error {
+	if strings.TrimSpace(p.K8sAPIServer) == "" {
+		return nil
+	}
+
+	hostNodeNamePath := p.HostNodeNameFilePath
+	if hostNodeNamePath == "" {
+		hostNodeNamePath = hostNodeNameFilePath
+	}
+	hostNodeNamePath = filepath.Join(p.FileSystemRoot, hostNodeNamePath)
+	if err := os.MkdirAll(filepath.Dir(hostNodeNamePath), 0755); err != nil {
+		return fmt.Errorf("error while creating directory for host node name file %s: %w", hostNodeNamePath, err)
+	}
+	if err := os.WriteFile(hostNodeNamePath, []byte(hostName+"\n"), 0644); err != nil {
+		return fmt.Errorf("error while writing host node name file %s: %w", hostNodeNamePath, err)
+	}
+
+	bootstrapPath := p.BootstrapKubeconfigPath
+	if bootstrapPath == "" {
+		bootstrapPath = hostBootstrapKubeconfigPath
+	}
+	bootstrapPath = filepath.Join(p.FileSystemRoot, bootstrapPath)
+	if err := os.MkdirAll(filepath.Dir(bootstrapPath), 0700); err != nil {
+		return fmt.Errorf("error while creating directory for bootstrap kubeconfig %s: %w", bootstrapPath, err)
+	}
+
+	bootstrapKubeconfig := fmt.Sprintf(`apiVersion: v1
+kind: Config
+clusters:
+- cluster:
+    certificate-authority: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
+    server: %s
+  name: host-cluster
+contexts:
+- context:
+    cluster: host-cluster
+    user: ovn-dpu-bootstrap
+  name: ovn-dpu-bootstrap
+current-context: ovn-dpu-bootstrap
+users:
+- name: ovn-dpu-bootstrap
+  user:
+    tokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token
+    as: system:node:%s
+    as-groups:
+    - system:nodes
+    - system:authenticated
+`, strings.TrimSpace(p.K8sAPIServer), hostName)
+
+	if err := os.WriteFile(bootstrapPath, []byte(bootstrapKubeconfig), 0600); err != nil {
+		return fmt.Errorf("error while writing bootstrap kubeconfig %s: %w", bootstrapPath, err)
 	}
 	return nil
 }

helm/ovn-kubernetes-dpf/templates/common.yaml

@@ -87,6 +87,16 @@ rules:
   - egressservices
   - adminpolicybasedexternalroutes
   verbs: [ "get", "list", "watch" ]
+{{- if eq (hasKey .Values.global "enableOvnKubeIdentity" | ternary .Values.global.enableOvnKubeIdentity false) true }}
+- apiGroups: ["certificates.k8s.io"]
+  resources:
+  - certificatesigningrequests
+  verbs:
+  - create
+  - get
+  - list
+  - watch
+{{- end }}
 - apiGroups: [""]
   resources:
   - events
@@ -129,4 +139,4 @@ rules:
   verbs:
   - get
   - create
-{{- end }}
+{{- end }}

helm/ovn-kubernetes-dpf/templates/dpu-manifests.yaml

@@ -1,3 +1,5 @@
+{{- $identityEnabled := hasKey .Values.global "enableOvnKubeIdentity" | ternary .Values.global.enableOvnKubeIdentity false -}}
+{{- $useSecretBootstrap := and $identityEnabled (ne (default "" .Values.dpuManifests.kubernetesSecretName) "") -}}
 {{- if .Values.dpuManifests.enabled }}
 ---
 apiVersion: v1
@@ -53,9 +55,15 @@ roleRef:
   kind: ClusterRole
   apiGroup: rbac.authorization.k8s.io
 subjects:
+{{- if eq (hasKey .Values.global "enableOvnKubeIdentity" | ternary .Values.global.enableOvnKubeIdentity false) true }}
+- kind: Group
+  name: system:ovn-nodes
+  apiGroup: rbac.authorization.k8s.io
+{{- else }}
 - kind: ServiceAccount
   name: {{ include "ovn-kubernetes.fullname" . }}-node
   namespace: {{ .Release.Namespace }}
+{{- end }}
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRoleBinding
@@ -80,9 +88,15 @@ roleRef:
   kind: Role
   apiGroup: rbac.authorization.k8s.io
 subjects:
+{{- if eq (hasKey .Values.global "enableOvnKubeIdentity" | ternary .Values.global.enableOvnKubeIdentity false) true }}
+- kind: Group
+  name: system:ovn-nodes
+  apiGroup: rbac.authorization.k8s.io
+{{- else }}
 - kind: ServiceAccount
   name: {{ include "ovn-kubernetes.fullname" . }}-node
   namespace: {{ .Release.Namespace }}
+{{- end }}
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: RoleBinding
@@ -274,6 +288,13 @@ spec:
           valueFrom:
             fieldRef:
               fieldPath: spec.nodeName
+        {{- if $useSecretBootstrap }}
+        - name: K8S_APISERVER
+          valueFrom:
+            configMapKeyRef:
+              name: {{ include "ovn-kubernetes.fullname" . }}-config
+              key: k8s_apiserver
+        {{- end }}
         {{- if .Values.dpuManifests.externalDHCP }}
         # Needed so that systemctl can run inside the container
         - name: SYSTEMCTL_FORCE_BUS
@@ -327,6 +348,12 @@ spec:
           readOnly: true
         - mountPath: /var/run/openvswitch/
           name: host-var-run-ovs
+        - mountPath: /var/run/ovn-kubernetes
+          name: host-var-run-ovn-kubernetes
+        {{- if $useSecretBootstrap }}
+        - mountPath: /host-kubernetes
+          name: host-kubeconfig
+        {{- end }}
       containers:
       - name: nb-ovsdb
         image: {{ .Values.dpuManifests.image.repository }}:{{ .Values.dpuManifests.image.tag }}
@@ -501,7 +528,15 @@ spec:
       - name: doca-ovnkube-controller
         image: {{ .Values.dpuManifests.image.repository }}:{{ .Values.dpuManifests.image.tag }}
         imagePullPolicy: {{ .Values.dpuManifests.image.pullPolicy }}
-        command: ["/root/ovnkube.sh", "ovnkube-controller-with-node"]
+        command:
+        - /bin/sh
+        - -ec
+        - |
+          host_node_name_file=/var/run/ovn-kubernetes/host-node-name
+          if [ -s "${host_node_name_file}" ]; then
+            export K8S_NODE_DPU="$(tr -d '[:space:]' < "${host_node_name_file}")"
+          fi
+          exec /root/ovnkube.sh ovnkube-controller-with-node
         securityContext:
           runAsUser: 0
           privileged: true
@@ -604,6 +639,10 @@ spec:
           valueFrom:
             fieldRef:
               fieldPath: spec.nodeName
+        - name: K8S_NODE_DPU
+          valueFrom:
+            fieldRef:
+              fieldPath: spec.nodeName
         - name: K8S_NODE_IP
           valueFrom:
             fieldRef:
@@ -702,7 +741,7 @@ spec:
         - name: OVN_ENABLE_MULTI_EXTERNAL_GATEWAY
           value: "false"
         - name: OVN_ENABLE_OVNKUBE_IDENTITY
-          value: "false"
+          value: {{ hasKey .Values.global "enableOvnKubeIdentity" | ternary .Values.global.enableOvnKubeIdentity false | quote }}
         - name: OVN_DISABLE_REQUESTEDCHASSIS
           value: {{ default "false" .Values.dpuManifests.ovnDisableRequestedchassis | quote }}
         - name: OVN_ENABLE_SVC_TEMPLATE_SUPPORT
@@ -726,7 +765,15 @@ spec:
       - name: ovn-controller
         image: {{ .Values.dpuManifests.image.repository }}:{{ .Values.dpuManifests.image.tag }}
         imagePullPolicy: {{ .Values.dpuManifests.image.pullPolicy }}
-        command: ["/root/ovnkube.sh", "ovn-controller"]
+        command:
+        - /bin/sh
+        - -ec
+        - |
+          host_node_name_file=/var/run/ovn-kubernetes/host-node-name
+          if [ -s "${host_node_name_file}" ]; then
+            export K8S_NODE_DPU="$(tr -d '[:space:]' < "${host_node_name_file}")"
+          fi
+          exec /root/ovnkube.sh ovn-controller
         securityContext:
           runAsUser: 0
           capabilities:
@@ -744,6 +791,8 @@ spec:
           name: host-var-run-ovs
         - mountPath: /var/run/ovn/
           name: host-var-run-ovs
+        - mountPath: /var/run/ovn-kubernetes
+          name: host-var-run-ovn-kubernetes
         - mountPath: /ovn-cert
           name: host-ovn-cert
           readOnly: true
@@ -761,6 +810,10 @@ spec:
             configMapKeyRef:
               name: {{ include "ovn-kubernetes.fullname" . }}-config
               key: k8s_apiserver
+        - name: K8S_NODE_DPU
+          valueFrom:
+            fieldRef:
+              fieldPath: spec.nodeName
         - name: OVN_KUBERNETES_NAMESPACE
           valueFrom:
             fieldRef:
@@ -845,8 +898,12 @@ spec:
         hostPath:
           path: /var/run/dbus
       - name: host-kubeconfig
+      {{- if $useSecretBootstrap }}
+        emptyDir: {}
+      {{- else }}
         hostPath:
           path: /etc/kubernetes/
+      {{- end }}
       - name: host-kubelet
         hostPath:
           path: /var/lib/kubelet