fix(containerd): detect NVIDIA runtime under containerd v2 plugin path and via imports (#1242)

The nvidia runtime check searches /etc/containerd/config.toml for the
literal
`plugins."io.containerd.grpc.v1.cri".containerd.runtimes.nvidia`, which
is the containerd 1.x plugin path. containerd 2.x uses
`io.containerd.cri.v1.runtime`, so the check raises a false positive on
every v2 host even when the nvidia runtime is configured correctly.

Two changes:

  - Accept either plugin path (1.x grpc.v1.cri or 2.x cri.v1.runtime).
- Follow `imports = [...]` directives in the main config so drop-in
files (e.g. /etc/containerd/conf.d/99-nvidia.toml written by
nvidia-container-toolkit-daemonset on recent gpu-operator versions) are
inspected too. Read errors on individual imports are ignored so the
existing substring checks still run against whatever could be read.

Adds unit tests for the v2 plugin path, the new constant, and the
import-following helper.

The error:
```
May 07 12:01:00 cluster-2 gpud[3478]: {"level":"warn","ts":"2026-05-07T12:01:00.934Z","caller":"log/log.go:175","msg":"nvidia GPUs found but nvidia-container-toolkit pod is not found"}
```

- [ ] Marking as draft I also haven't tested this (yet) and this is
claude coding.

With this kind of containerd setup:

```
$ containerd --version
containerd containerd.io v2.2.3 77c84241c7cbdd9b4eca2591793e3d4f4317c590
$ find /etc/containerd/ -type f
/etc/containerd/config.toml
/etc/containerd/config.d/root-on-nvme.toml

$ find /etc/containerd/ -type f|xargs cat
imports = ["/etc/containerd/config.d/*.toml", "/etc/containerd/conf.d/*.toml"]
version = 3

[metrics]
  address = "0.0.0.0:1338"
  grpc_histogram = true
root = "/mnt/local_disk/containerd"
```

Co-authored-by: Gyuho Lee <6799218+gyuho@users.noreply.github.com>

Author: martbhell

components/containerd/component.go

@@ -7,6 +7,8 @@ import (
 	"encoding/json"
 	"fmt"
 	"os"
+	"path/filepath"
+	"regexp"
 	"strings"
 	"sync"
 	"time"
@@ -34,11 +36,19 @@ const (
 
 	socketMissingConsecutiveThreshold = 5
 
-	// Containerd config strings to check for NVIDIA runtime configuration
-	containerdConfigNvidiaDefaultRuntime = `default_runtime_name = "nvidia"`
-	containerdConfigNvidiaRuntimePlugin  = `plugins."io.containerd.grpc.v1.cri".containerd.runtimes.nvidia`
+	// Containerd config strings to check for NVIDIA runtime configuration.
+	// containerd 1.x uses the "io.containerd.grpc.v1.cri" plugin path; 2.x
+	// uses "io.containerd.cri.v1.runtime". Either is acceptable.
+	containerdConfigNvidiaDefaultRuntime  = `default_runtime_name = "nvidia"`
+	containerdConfigNvidiaRuntimePlugin   = `plugins."io.containerd.grpc.v1.cri".containerd.runtimes.nvidia`
+	containerdConfigNvidiaRuntimePluginV2 = `plugins."io.containerd.cri.v1.runtime".containerd.runtimes.nvidia`
 )
 
+// containerdConfigImportsRe matches `imports = [...]` directives in a
+// containerd config so any drop-in files (e.g. /etc/containerd/conf.d/*.toml
+// written by nvidia-container-toolkit-daemonset) can be inspected too.
+var containerdConfigImportsRe = regexp.MustCompile(`(?m)^\s*imports\s*=\s*\[([^\]]*)\]`)
+
 var _ components.Component = &component{}
 
 type component struct {
@@ -363,13 +373,17 @@ func (c *component) Check() components.CheckResult {
 			// been running long enough
 			if elapsed > c.containerToolkitCreationThreshold {
 				config, err := c.getContainerdConfigFunc()
+				if err == nil {
+					config = appendImportedContainerdConfigs(config)
+				}
+				hasNvidiaPlugin := bytes.Contains(config, []byte(containerdConfigNvidiaRuntimePlugin)) ||
+					bytes.Contains(config, []byte(containerdConfigNvidiaRuntimePluginV2))
 				switch {
 				case err != nil:
 					reason := "error getting containerd config"
 					cr.appendReason(reason)
 					log.Logger.Warnw(reason)
-				case !bytes.Contains(config, []byte(containerdConfigNvidiaDefaultRuntime)) ||
-					!bytes.Contains(config, []byte(containerdConfigNvidiaRuntimePlugin)):
+				case !bytes.Contains(config, []byte(containerdConfigNvidiaDefaultRuntime)) || !hasNvidiaPlugin:
 					reason := fmt.Sprintf("nvidia-container-toolkit pod is running but %s is missing NVIDIA runtime configuration", defaultContainerdConfigPath)
 					cr.appendReason(reason)
 					log.Logger.Warnw(reason)
@@ -499,6 +513,37 @@ func (cr *checkResult) HealthStates() apiv1.HealthStates {
 	return apiv1.HealthStates{state}
 }
 
+// appendImportedContainerdConfigs returns config with the contents of any
+// files referenced by an "imports = [...]" directive appended. Errors
+// reading individual imports are ignored so the substring checks still
+// run against whatever could be read.
+func appendImportedContainerdConfigs(config []byte) []byte {
+	m := containerdConfigImportsRe.FindSubmatch(config)
+	if len(m) < 2 {
+		return config
+	}
+	out := config
+	for _, raw := range strings.Split(string(m[1]), ",") {
+		pattern := strings.Trim(strings.TrimSpace(raw), `"'`)
+		if pattern == "" {
+			continue
+		}
+		matches, err := filepath.Glob(pattern)
+		if err != nil {
+			continue
+		}
+		for _, f := range matches {
+			b, err := os.ReadFile(f)
+			if err != nil {
+				continue
+			}
+			out = append(out, '\n')
+			out = append(out, b...)
+		}
+	}
+	return out
+}
+
 func danglingPodCount(containerdPods []PodSandbox, kubeletPods []componentkubelet.PodStatus) int {
 	var danglingCount int
 	if len(kubeletPods) == 0 {

components/containerd/component_config_test.go

@@ -1,6 +1,8 @@
 package containerd
 
 import (
+	"os"
+	"path/filepath"
 	"testing"
 
 	"github.com/stretchr/testify/assert"
@@ -11,17 +13,50 @@ func TestContainerdConfigConstants(t *testing.T) {
 	// Test that the constants match the expected values
 	assert.Equal(t, `default_runtime_name = "nvidia"`, containerdConfigNvidiaDefaultRuntime)
 	assert.Equal(t, `plugins."io.containerd.grpc.v1.cri".containerd.runtimes.nvidia`, containerdConfigNvidiaRuntimePlugin)
+	assert.Equal(t, `plugins."io.containerd.cri.v1.runtime".containerd.runtimes.nvidia`, containerdConfigNvidiaRuntimePluginV2)
 
 	// Test that the constants can be used as byte slices
 	assert.NotNil(t, []byte(containerdConfigNvidiaDefaultRuntime))
 	assert.NotNil(t, []byte(containerdConfigNvidiaRuntimePlugin))
+	assert.NotNil(t, []byte(containerdConfigNvidiaRuntimePluginV2))
 
 	// Test that the constants have the expected content
 	assert.Contains(t, containerdConfigNvidiaDefaultRuntime, "nvidia")
 	assert.Contains(t, containerdConfigNvidiaRuntimePlugin, "nvidia")
 	assert.Contains(t, containerdConfigNvidiaRuntimePlugin, "plugins")
 	assert.Contains(t, containerdConfigNvidiaRuntimePlugin, "containerd")
 	assert.Contains(t, containerdConfigNvidiaRuntimePlugin, "runtimes")
+	assert.Contains(t, containerdConfigNvidiaRuntimePluginV2, "nvidia")
+	assert.Contains(t, containerdConfigNvidiaRuntimePluginV2, "io.containerd.cri.v1.runtime")
+}
+
+// TestAppendImportedContainerdConfigs verifies that imports referenced in a
+// containerd config are read and appended so substring checks see drop-in files.
+func TestAppendImportedContainerdConfigs(t *testing.T) {
+	dir := t.TempDir()
+	dropIn := filepath.Join(dir, "99-nvidia.toml")
+	dropInBody := `[plugins."io.containerd.cri.v1.runtime".containerd]
+  default_runtime_name = "nvidia"
+[plugins."io.containerd.cri.v1.runtime".containerd.runtimes.nvidia]
+  runtime_type = "io.containerd.runc.v2"
+`
+	if err := os.WriteFile(dropIn, []byte(dropInBody), 0o644); err != nil {
+		t.Fatalf("write drop-in: %v", err)
+	}
+
+	main := []byte("version = 3\nimports = [\"" + filepath.Join(dir, "*.toml") + "\"]\n")
+	out := appendImportedContainerdConfigs(main)
+
+	assert.Contains(t, string(out), containerdConfigNvidiaDefaultRuntime)
+	assert.Contains(t, string(out), containerdConfigNvidiaRuntimePluginV2)
+}
+
+// TestAppendImportedContainerdConfigs_NoImports returns the input unchanged
+// when no imports directive is present.
+func TestAppendImportedContainerdConfigs_NoImports(t *testing.T) {
+	in := []byte("version = 3\n[metrics]\n  address = \"0.0.0.0:1338\"\n")
+	out := appendImportedContainerdConfigs(in)
+	assert.Equal(t, string(in), string(out))
 }
 
 // TestContainerdConfigWithBothNvidiaSettings tests configs that should contain both nvidia settings

components/containerd/component_test.go

@@ -2063,6 +2063,36 @@ func TestNVMLValidationWithContainerToolkit(t *testing.T) {
 			expectedHealth:                    apiv1.HealthStateTypeHealthy,
 			expectedReason:                    "ok",
 		},
+		{
+			name: "nvml with containerd v2 nvidia plugin path",
+			nvmlInstance: &mockNVMLInstance{
+				nvmlExists:  true,
+				productName: "Tesla V100",
+			},
+			getContainerdConfigFunc: func() ([]byte, error) {
+				config := `default_runtime_name = "nvidia"
+[plugins."io.containerd.cri.v1.runtime".containerd.runtimes.nvidia]
+  runtime_type = "io.containerd.runc.v2"`
+				return []byte(config), nil
+			},
+			pods: []PodSandbox{
+				{
+					Name:  "nvidia-container-toolkit-daemonset-v2",
+					State: "SANDBOX_READY",
+					Containers: []PodSandboxContainerStatus{
+						{
+							Name:      "nvidia-container-toolkit-ctr",
+							State:     "CONTAINER_RUNNING",
+							CreatedAt: time.Now().Add(-15 * time.Minute).UnixNano(),
+						},
+					},
+				},
+			},
+			getTimeNowFunc:                    time.Now,
+			containerToolkitCreationThreshold: 10 * time.Minute,
+			expectedHealth:                    apiv1.HealthStateTypeHealthy,
+			expectedReason:                    "ok",
+		},
 		{
 			name: "nvml without nvidia config but container toolkit present and running long enough",
 			nvmlInstance: &mockNVMLInstance{