Auto-MNNVL: add MNNVL configuration and startup validation (#346)

shmuel-runai · web-flow · commit 0411a1ea330c · 2026-01-22T14:47:53.000+02:00
Ref: GREP-270

Add MNNVL configuration support to the Grove operator.
This includes configuration, startup validation, and Helm chart integration.

Changes:
- Add MNNVLConfiguration struct with Enabled field (default: false) and to the OperatorConfiguration
- Add validateMNNVLPrerequisites() to check for ComputeDomain CRD at startup
- Update Helm chart values.yaml and _helpers.tpl with mnnvl.enabled
- Add unit tests for config parsing and CRD detection logic

When MNNVL is enabled, but the ComputeDomain CRD is not installed, the operator exits with a non-zero exit code.

Tested
- UT
diff --git a/docs/api-reference/operator-api.md b/docs/api-reference/operator-api.md
@@ -781,6 +781,22 @@ _Appears in:_
 | `error` | ErrorLevel is a log level where only errors are logged.<br /> |
 
 
+#### NetworkAcceleration
+
+
+
+NetworkAcceleration defines the configuration for network acceleration features.
+
+
+
+_Appears in:_
+- [OperatorConfiguration](#operatorconfiguration)
+
+| Field | Description | Default | Validation |
+| --- | --- | --- | --- |
+| `autoMNNVLEnabled` _boolean_ | AutoMNNVLEnabled indicates whether automatic MNNVL (Multi-Node NVLink) support is enabled.<br />When enabled, the operator will automatically create and manage ComputeDomain resources<br />for GPU workloads. If the cluster doesn't have the NVIDIA DRA driver installed,<br />the operator will exit with a non-zero exit code.<br />Default: false |  |  |
+
+
 
 
 #### PodCliqueControllerConfiguration
diff --git a/operator/api/config/v1alpha1/types.go b/operator/api/config/v1alpha1/types.go
@@ -65,6 +65,8 @@ type OperatorConfiguration struct {
 	LogFormat               LogFormat                            `json:"logFormat"`
 	Authorizer              AuthorizerConfig                     `json:"authorizer"`
 	TopologyAwareScheduling TopologyAwareSchedulingConfiguration `json:"topologyAwareScheduling"`
+	// +optional
+	Network NetworkAcceleration `json:"network,omitempty"` // Network is the configuration for network acceleration features like MNNVL.
 }
 
 // LeaderElectionConfiguration defines the configuration for the leader election.
@@ -200,3 +202,13 @@ type TopologyAwareSchedulingConfiguration struct {
 	// +optional
 	Levels []corev1alpha1.TopologyLevel `json:"levels,omitempty"`
 }
+
+// NetworkAcceleration defines the configuration for network acceleration features.
+type NetworkAcceleration struct {
+	// AutoMNNVLEnabled indicates whether automatic MNNVL (Multi-Node NVLink) support is enabled.
+	// When enabled, the operator will automatically create and manage ComputeDomain resources
+	// for GPU workloads. If the cluster doesn't have the NVIDIA DRA driver installed,
+	// the operator will exit with a non-zero exit code.
+	// Default: false
+	AutoMNNVLEnabled bool `json:"autoMNNVLEnabled"`
+}
diff --git a/operator/api/config/v1alpha1/zz_generated.deepcopy.go b/operator/api/config/v1alpha1/zz_generated.deepcopy.go
diff --git a/operator/charts/templates/_helpers.tpl b/operator/charts/templates/_helpers.tpl
@@ -55,6 +55,10 @@ config.yaml: |
       []
     {{- end }}
   {{- end }}
+  {{- if .Values.config.network }}
+  network:
+    autoMNNVLEnabled: {{ .Values.config.network.autoMNNVLEnabled | default false }}
+  {{- end }}
 
 {{- end -}}
 
diff --git a/operator/charts/values.yaml b/operator/charts/values.yaml
@@ -85,6 +85,8 @@ config:
       - system:kube-controller-manager
       - system:serviceaccount:kai-scheduler:pod-grouper
       - system:serviceaccount:kai-scheduler:binder
+  network:
+    autoMNNVLEnabled: false
 
 configMap:
   labels:
diff --git a/operator/cmd/cli/cli.go b/operator/cmd/cli/cli.go
@@ -43,6 +43,8 @@ const (
 	ExitErrInitializeManager
 	// ExitErrStart indicates that the application exited due to an error when starting the application.
 	ExitErrStart
+	// ExitErrMNNVLPrerequisites indicates that the application exited because MNNVL prerequisites are not met.
+	ExitErrMNNVLPrerequisites
 )
 
 var (
diff --git a/operator/cmd/cli/cli_test.go b/operator/cmd/cli/cli_test.go
@@ -130,6 +130,15 @@ func TestLoadAndValidateOperatorConfig(t *testing.T) {
 				assert.Equal(t, 9445, config.Server.Metrics.Port)
 				assert.NotNil(t, config.Controllers.PodCliqueSet.ConcurrentSyncs)
 				assert.Equal(t, 3, *config.Controllers.PodCliqueSet.ConcurrentSyncs)
+				assert.False(t, config.Network.AutoMNNVLEnabled, "MNNVL should be disabled by default")
+			},
+		},
+		{
+			name:       "valid config with MNNVL enabled",
+			configFile: "testdata/valid-config-mnnvl-enabled.yaml",
+			validateFunc: func(t *testing.T, config *configv1alpha1.OperatorConfiguration) {
+				require.NotNil(t, config)
+				assert.True(t, config.Network.AutoMNNVLEnabled, "MNNVL should be enabled")
 			},
 		},
 		{
diff --git a/operator/cmd/cli/testdata/valid-config-mnnvl-enabled.yaml b/operator/cmd/cli/testdata/valid-config-mnnvl-enabled.yaml
@@ -0,0 +1,40 @@
+apiVersion: operator.config.grove.io/v1alpha1
+kind: OperatorConfiguration
+runtimeClientConnection:
+  qps: 100
+  burst: 150
+leaderElection:
+  enabled: true
+  leaseDuration: 15s
+  renewDeadline: 10s
+  retryPeriod: 2s
+  resourceLock: leases
+  resourceName: grove-operator-leader-election
+  resourceNamespace: default
+server:
+  webhooks:
+    bindAddress: 0.0.0.0
+    port: 9443
+    serverCertDir: /etc/grove-operator/webhook-certs
+  healthProbes:
+    bindAddress: 0.0.0.0
+    port: 9444
+  metrics:
+    bindAddress: 0.0.0.0
+    port: 9445
+controllers:
+  podCliqueSet:
+    concurrentSyncs: 3
+  podClique:
+    concurrentSyncs: 3
+  podCliqueScalingGroup:
+    concurrentSyncs: 2
+logLevel: info
+logFormat: json
+authorizer:
+  enabled: false
+clusterTopology:
+  enabled: false
+network:
+  autoMNNVLEnabled: true
+
diff --git a/operator/cmd/cli/testdata/valid-config.yaml b/operator/cmd/cli/testdata/valid-config.yaml
@@ -35,4 +35,6 @@ authorizer:
   enabled: false
 clusterTopology:
   enabled: false
+network:
+  autoMNNVLEnabled: false
 
diff --git a/operator/cmd/main.go b/operator/cmd/main.go
@@ -31,6 +31,7 @@ import (
 	grovectrl "github.com/ai-dynamo/grove/operator/internal/controller"
 	"github.com/ai-dynamo/grove/operator/internal/controller/cert"
 	grovelogger "github.com/ai-dynamo/grove/operator/internal/logger"
+	"github.com/ai-dynamo/grove/operator/internal/mnnvl"
 	groveversion "github.com/ai-dynamo/grove/operator/internal/version"
 
 	"github.com/spf13/pflag"
@@ -63,6 +64,12 @@ func main() {
 	logger.Info("Starting grove operator", "grove-info", groveInfo.Verbose())
 	printFlags()
 
+	// Run MNNVL preflight checks if the feature is enabled
+	if err := mnnvl.Preflight(operatorConfig); err != nil {
+		logger.Error(err, "MNNVL preflight check failed")
+		handleErrorAndExit(err, cli.ExitErrMNNVLPrerequisites)
+	}
+
 	mgr, err := grovectrl.CreateManager(operatorConfig)
 	if err != nil {
 		logger.Error(err, "failed to create grove controller manager")
diff --git a/operator/internal/mnnvl/constants.go b/operator/internal/mnnvl/constants.go
@@ -0,0 +1,45 @@
+// /*
+// Copyright 2025 The Grove Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+// */
+
+// Package mnnvl provides utilities and constants for Multi-Node NVLink (MNNVL) support.
+package mnnvl
+
+// Constants for NVIDIA ComputeDomain (used by MNNVL support)
+const (
+	// ComputeDomainGroup is the API group for NVIDIA ComputeDomain resources.
+	ComputeDomainGroup = "resource.nvidia.com"
+	// ComputeDomainVersion is the API version for NVIDIA ComputeDomain resources.
+	ComputeDomainVersion = "v1beta1"
+	// ComputeDomainKind is the Kind for NVIDIA ComputeDomain resources.
+	ComputeDomainKind = "ComputeDomain"
+	// ComputeDomainResource is the plural resource name for ComputeDomain.
+	ComputeDomainResource = "computedomains"
+	// ComputeDomainCRDName is the full CRD name for ComputeDomain.
+	ComputeDomainCRDName = ComputeDomainResource + "." + ComputeDomainGroup
+)
+
+// MNNVL annotation and finalizer constants
+const (
+	// AnnotationAutoMNNVL is the annotation key used to indicate whether automatic MNNVL
+	// support should be enabled for a PodCliqueSet. When set to "true", the operator will
+	// automatically create and manage ComputeDomain resources for the workload.
+	AnnotationAutoMNNVL = "grove.io/auto-mnnvl"
+
+	// FinalizerComputeDomain is the finalizer added to ComputeDomains to prevent accidental
+	// deletion while workloads are using them. This finalizer is removed by the PCS controller
+	// during scale-in or PCS deletion.
+	FinalizerComputeDomain = "grove.io/computedomain.resource.nvidia.com"
+)
diff --git a/operator/internal/mnnvl/preflight.go b/operator/internal/mnnvl/preflight.go
@@ -0,0 +1,95 @@
+// /*
+// Copyright 2025 The Grove Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+// */
+
+package mnnvl
+
+import (
+	"fmt"
+
+	configv1alpha1 "github.com/ai-dynamo/grove/operator/api/config/v1alpha1"
+
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/client-go/discovery"
+	ctrl "sigs.k8s.io/controller-runtime"
+)
+
+var logger = ctrl.Log.WithName("mnnvl")
+
+// Preflight checks if MNNVL prerequisites are met when the feature is enabled.
+// If MNNVL is enabled, it verifies that the ComputeDomain CRD is installed in the cluster.
+func Preflight(operatorCfg *configv1alpha1.OperatorConfiguration) error {
+	if !operatorCfg.Network.AutoMNNVLEnabled {
+		return nil
+	}
+
+	logger.Info("MNNVL support is enabled, validating prerequisites")
+
+	discoveryClient, err := newDiscoveryClient()
+	if err != nil {
+		return err
+	}
+
+	return validateComputeDomainCRD(discoveryClient)
+}
+
+// newDiscoveryClient creates a new Kubernetes discovery client using the in-cluster
+// or kubeconfig configuration.
+func newDiscoveryClient() (discovery.DiscoveryInterface, error) {
+	cfg, err := ctrl.GetConfig()
+	if err != nil {
+		return nil, fmt.Errorf("failed to get cluster config for MNNVL validation: %w", err)
+	}
+
+	discoveryClient, err := discovery.NewDiscoveryClientForConfig(cfg)
+	if err != nil {
+		return nil, fmt.Errorf("failed to create discovery client for MNNVL validation: %w", err)
+	}
+
+	return discoveryClient, nil
+}
+
+// validateComputeDomainCRD checks if the ComputeDomain CRD is installed in the cluster.
+// It uses the provided discovery interface to query the API server.
+func validateComputeDomainCRD(discoveryClient discovery.DiscoveryInterface) error {
+	_, apiResourceList, err := discoveryClient.ServerGroupsAndResources()
+	if err != nil {
+		// Handle partial discovery errors gracefully
+		if !discovery.IsGroupDiscoveryFailedError(err) {
+			return fmt.Errorf("failed to discover API resources: %w", err)
+		}
+	}
+
+	if !isComputeDomainCRDPresent(apiResourceList) {
+		return fmt.Errorf("MNNVL is enabled but ComputeDomain CRD (%s) is not installed. "+
+			"Please install the NVIDIA DRA driver or disable MNNVL in the operator configuration", ComputeDomainCRDName)
+	}
+
+	logger.Info("ComputeDomain CRD found, MNNVL prerequisites validated")
+	return nil
+}
+
+// isComputeDomainCRDPresent checks if the ComputeDomain CRD is present in the API resource list.
+func isComputeDomainCRDPresent(apiResourceList []*metav1.APIResourceList) bool {
+	computeDomainGroupVersion := ComputeDomainGroup + "/" + ComputeDomainVersion
+	for _, resourceList := range apiResourceList {
+		for _, resource := range resourceList.APIResources {
+			if resource.Name == ComputeDomainResource && resourceList.GroupVersion == computeDomainGroupVersion {
+				return true
+			}
+		}
+	}
+	return false
+}
diff --git a/operator/internal/mnnvl/preflight_test.go b/operator/internal/mnnvl/preflight_test.go

Original file line number	Diff line number	Diff line change
`@@ -43,6 +43,8 @@ const (`
`43`	`43`	`ExitErrInitializeManager`
`44`	`44`	`// ExitErrStart indicates that the application exited due to an error when starting the application.`
`45`	`45`	`ExitErrStart`
	`46`	`+ // ExitErrMNNVLPrerequisites indicates that the application exited because MNNVL prerequisites are not met.`
	`47`	`+ ExitErrMNNVLPrerequisites`
`46`	`48`	`)`
`47`	`49`
`48`	`50`	`var (`