Add DCGM, DCGM Exporter, and DRA validation operands to GPUCluster

Signed-off-by: Karthik Vetrivel <kvetrivel@nvidia.com>

Author: karthikvetrivel

internal/state/configurable_state.go

@@ -0,0 +1,107 @@
+/**
+# Copyright (c) NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package state
+
+import (
+	"context"
+	"fmt"
+
+	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
+
+	nvidiav1alpha1 "github.com/NVIDIA/gpu-operator/api/nvidia/v1alpha1"
+	"github.com/NVIDIA/gpu-operator/internal/image"
+)
+
+// configurableState is a State implementation shared by the GPUCluster operands
+// that reconcile with the same shape: skip when disabled, resolve the DRA apiVersion,
+// label the namespace for adminAccess, resolve the operand image, build the operand's
+// render data, and render. Per-operand behavior is injected through the function fields
+// so each operand file only declares what actually differs.
+type configurableState struct {
+	stateSkel
+
+	// isEnabled reports whether this operand should be deployed for the given CR.
+	isEnabled func(cr *nvidiav1alpha1.GPUCluster) bool
+
+	// imageOverride resolves the (repository, image, version) overrides from the CR,
+	// which feed image.ImagePath together with imageEnvName. It may be nil for operands
+	// that carry no per-CR image overrides (e.g. those running the gpu-operator image).
+	imageOverride func(cr *nvidiav1alpha1.GPUCluster) (repository, image, version string)
+
+	// imageEnvName is the fallback env var for the operand image.
+	imageEnvName string
+
+	// buildRenderData builds the operand-specific templating data from the resolved
+	// image path and DRA apiVersion. It receives ctx and the skeleton so operands that
+	// need the client or logging (e.g. dcgm-exporter's ServiceMonitor CRD probe) can use them.
+	buildRenderData func(ctx context.Context, s *configurableState, cr *nvidiav1alpha1.GPUCluster, imagePath, apiVersion string) (interface{}, error)
+}
+
+var _ State = (*configurableState)(nil)
+
+func (s *configurableState) Sync(ctx context.Context, customResource interface{}, infoCatalog InfoCatalog) (SyncState, error) {
+	cr, ok := customResource.(*nvidiav1alpha1.GPUCluster)
+	if !ok {
+		return SyncStateError, fmt.Errorf("GPUCluster CR not provided as input to Sync()")
+	}
+
+	objs, err := s.getManifestObjects(ctx, cr, infoCatalog)
+	if err != nil {
+		return SyncStateNotReady, fmt.Errorf("failed to create k8s objects from manifests: %w", err)
+	}
+
+	if len(objs) == 0 {
+		return s.handleStateObjectsDeletion(ctx)
+	}
+
+	return s.syncObjects(ctx, cr, objs)
+}
+
+func (s *configurableState) GetWatchSources(mgr ctrlManager) map[string]SyncingSource {
+	return gpuClusterDaemonSetSource(mgr)
+}
+
+func (s *configurableState) getManifestObjects(ctx context.Context, cr *nvidiav1alpha1.GPUCluster, infoCatalog InfoCatalog) ([]*unstructured.Unstructured, error) {
+	if !s.isEnabled(cr) {
+		return []*unstructured.Unstructured{}, nil
+	}
+
+	apiVersion, err := draResourceAPIVersion(infoCatalog)
+	if err != nil {
+		return nil, err
+	}
+
+	if err := ensureAdminAccessLabel(ctx, s.client, s.namespace); err != nil {
+		return nil, fmt.Errorf("failed to label namespace for admin access: %w", err)
+	}
+
+	var repository, img, version string
+	if s.imageOverride != nil {
+		repository, img, version = s.imageOverride(cr)
+	}
+	imagePath, err := image.ImagePath(repository, img, version, s.imageEnvName)
+	if err != nil {
+		return nil, fmt.Errorf("failed to construct %s image path: %w", s.name, err)
+	}
+
+	renderData, err := s.buildRenderData(ctx, s, cr, imagePath, apiVersion)
+	if err != nil {
+		return nil, err
+	}
+
+	return s.renderObjects(ctx, renderData)
+}

internal/state/dcgm.go

@@ -0,0 +1,65 @@
+/**
+# Copyright (c) NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package state
+
+import (
+	"context"
+
+	"k8s.io/apimachinery/pkg/runtime"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+
+	nvidiav1alpha1 "github.com/NVIDIA/gpu-operator/api/nvidia/v1alpha1"
+)
+
+const (
+	// dcgmImageEnvName is the fallback env var for the DCGM image when the CR does not
+	// specify repository/image/version.
+	dcgmImageEnvName = "DCGM_IMAGE"
+)
+
+func NewStateDCGM(
+	k8sClient client.Client,
+	namespace string,
+	scheme *runtime.Scheme,
+	manifestDir string) (State, error) {
+
+	skel, err := newStateSkel(k8sClient, namespace, scheme, manifestDir,
+		"state-dcgm", "NVIDIA DCGM hostengine deployed in the cluster")
+	if err != nil {
+		return nil, err
+	}
+	return &configurableState{
+		stateSkel: skel,
+		isEnabled: dcgmEnabled,
+		imageOverride: func(cr *nvidiav1alpha1.GPUCluster) (string, string, string) {
+			spec := cr.Spec.DCGM
+			return spec.Repository, spec.Image, spec.Version
+		},
+		imageEnvName:    dcgmImageEnvName,
+		buildRenderData: buildDCGMRenderData,
+	}, nil
+}
+
+func buildDCGMRenderData(_ context.Context, s *configurableState, cr *nvidiav1alpha1.GPUCluster, imagePath, apiVersion string) (interface{}, error) {
+	daemonsets := cr.Spec.Daemonsets
+	return &dcgmRenderData{
+		DCGM:                    &dcgmSpec{Spec: cr.Spec.DCGM, ImagePath: imagePath},
+		Daemonsets:              &daemonsets,
+		Namespace:               s.namespace,
+		ResourceClaimAPIVersion: apiVersion,
+	}, nil
+}

internal/state/dcgm_exporter.go

@@ -0,0 +1,152 @@
+/**
+# Copyright (c) NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package state
+
+import (
+	"context"
+	"path/filepath"
+	"strings"
+
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/runtime/schema"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/log"
+
+	nvidiav1alpha1 "github.com/NVIDIA/gpu-operator/api/nvidia/v1alpha1"
+	"github.com/NVIDIA/gpu-operator/internal/consts"
+)
+
+const (
+	// dcgmExporterImageEnvName is the fallback env var for the dcgm-exporter image when the
+	// CR does not specify repository/image/version.
+	dcgmExporterImageEnvName = "DCGM_EXPORTER_IMAGE"
+
+	// dcgmRemoteHostEngine points dcgm-exporter at the standalone nvidia-dcgm hostengine
+	// Service (manifests/state-dcgm/0600_service.yaml).
+	dcgmRemoteHostEngine = "nvidia-dcgm:5555"
+
+	dcgmExporterDefaultCollectors     = "/etc/dcgm-exporter/dcp-metrics-included.csv"
+	dcgmExporterCustomCollectors      = "/etc/dcgm-exporter/dcgm-metrics.csv"
+	dcgmExporterDefaultKubeletRootDir = "/var/lib/kubelet"
+	dcgmExporterDefaultJobMappingDir  = "/var/lib/dcgm-exporter/job-mapping"
+)
+
+func NewStateDCGMExporter(
+	k8sClient client.Client,
+	namespace string,
+	scheme *runtime.Scheme,
+	manifestDir string) (State, error) {
+
+	skel, err := newStateSkel(k8sClient, namespace, scheme, manifestDir,
+		"state-dcgm-exporter", "NVIDIA DCGM Exporter deployed in the cluster")
+	if err != nil {
+		return nil, err
+	}
+	return &configurableState{
+		stateSkel: skel,
+		isEnabled: func(cr *nvidiav1alpha1.GPUCluster) bool {
+			return cr.Spec.DCGMExporter != nil && cr.Spec.DCGMExporter.IsEnabled()
+		},
+		imageOverride: func(cr *nvidiav1alpha1.GPUCluster) (string, string, string) {
+			spec := cr.Spec.DCGMExporter
+			return spec.Repository, spec.Image, spec.Version
+		},
+		imageEnvName:    dcgmExporterImageEnvName,
+		buildRenderData: buildDCGMExporterRenderData,
+	}, nil
+}
+
+func buildDCGMExporterRenderData(ctx context.Context, s *configurableState, cr *nvidiav1alpha1.GPUCluster, imagePath, apiVersion string) (interface{}, error) {
+	spec := cr.Spec.DCGMExporter
+
+	// When standalone DCGM is enabled the exporter targets it; otherwise it runs embedded.
+	remoteHostEngine := ""
+	if dcgmEnabled(cr) {
+		remoteHostEngine = dcgmRemoteHostEngine
+	}
+
+	collectors := dcgmExporterDefaultCollectors
+	metricsConfigName := ""
+	if spec.MetricsConfig != nil && spec.MetricsConfig.Name != "" {
+		metricsConfigName = spec.MetricsConfig.Name
+		collectors = dcgmExporterCustomCollectors
+	}
+
+	hpcJobMappingDir := ""
+	if spec.IsHPCJobMappingEnabled() {
+		hpcJobMappingDir = spec.GetHPCJobMappingDirectory()
+		if hpcJobMappingDir == "" {
+			hpcJobMappingDir = dcgmExporterDefaultJobMappingDir
+		}
+	}
+
+	kubeletRootDir := cr.Spec.HostPaths.KubeletRootDir
+	if kubeletRootDir == "" {
+		kubeletRootDir = dcgmExporterDefaultKubeletRootDir
+	}
+
+	// Skip the ServiceMonitor when its CRD is absent so a default install without the
+	// Prometheus Operator does not fail (matches the ClusterPolicy path).
+	serviceMonitorEnabled := spec.ServiceMonitor != nil &&
+		spec.ServiceMonitor.Enabled != nil && *spec.ServiceMonitor.Enabled
+	if serviceMonitorEnabled && !serviceMonitorCRDServed(s.client) {
+		log.FromContext(ctx).V(consts.LogLevelInfo).Info(
+			"ServiceMonitor CRD not served; skipping dcgm-exporter ServiceMonitor creation")
+		serviceMonitorEnabled = false
+	}
+
+	serviceType := "ClusterIP"
+	serviceInternalTrafficPolicy := ""
+	if spec.ServiceSpec != nil {
+		if spec.ServiceSpec.Type != "" {
+			serviceType = string(spec.ServiceSpec.Type)
+		}
+		if spec.ServiceSpec.InternalTrafficPolicy != nil {
+			serviceInternalTrafficPolicy = string(*spec.ServiceSpec.InternalTrafficPolicy)
+		}
+	}
+
+	daemonsets := cr.Spec.Daemonsets
+	return &dcgmExporterRenderData{
+		DCGMExporter:                 &dcgmExporterSpec{Spec: spec, ImagePath: imagePath},
+		Daemonsets:                   &daemonsets,
+		Namespace:                    s.namespace,
+		ResourceClaimAPIVersion:      apiVersion,
+		RemoteHostEngine:             remoteHostEngine,
+		Collectors:                   collectors,
+		HPCJobMappingDir:             hpcJobMappingDir,
+		PodLabelAllowlistRegex:       strings.Join(spec.PodLabelAllowlistRegex, ","),
+		PodMetadataEnabled:           spec.IsKubernetesPodMetadataEnabled(),
+		EnablePodLabels:              spec.IsPodLabelsEnabled(),
+		EnablePodUID:                 spec.IsPodUIDEnabled(),
+		HostPID:                      spec.IsHostPIDEnabled(),
+		HostNetwork:                  spec.IsHostNetworkEnabled(),
+		MetricsConfigName:            metricsConfigName,
+		ServiceMonitorEnabled:        serviceMonitorEnabled,
+		PodResourcesDir:              filepath.Join(kubeletRootDir, "pod-resources"),
+		ServiceType:                  serviceType,
+		ServiceInternalTrafficPolicy: serviceInternalTrafficPolicy,
+	}, nil
+}
+
+// serviceMonitorCRDServed reports whether the cluster serves the monitoring.coreos.com
+// ServiceMonitor kind (i.e. the Prometheus Operator CRDs are installed).
+func serviceMonitorCRDServed(k8sClient client.Client) bool {
+	_, err := k8sClient.RESTMapper().RESTMapping(
+		schema.GroupKind{Group: "monitoring.coreos.com", Kind: "ServiceMonitor"}, "v1")
+	return err == nil
+}