using configmap to inject runtime and csi sideacr

Signed-off-by: jicheng.sk <jicheng.sk@alibaba-inc.com>

Author: BH4AWS

api/v1alpha1/sandbox_types.go

@@ -35,6 +35,16 @@ const (
 	// Note: SandboxSet creates sandboxes with priority 0 by default.
 	// Sandbox Manager or Sandbox Claim creates high-priority sandboxes by default.
 	SandboxAnnotationPriority = "agents.kruise.io/sandbox-priority"
+
+	// ShouldInjectCsiMount is the annotation key for inject csi mount plugin container.
+	// If set, the csi sidecar will be injected into the pod when the sandbox is created.
+	// The csi mount sidecar is used to mount the remote oss/nas storage to the sandbox container.
+	ShouldInjectCsiMount = "agents.kruise.io/should-inject-csi-mount-plugin"
+
+	// ShouldInjectAgentRuntime is the annotation key for inject agent runtime sidecar in init container.
+	// If set, the agent runtime sidecar will be injected into the pod when the sandbox is created.
+	// Some binary tools which are contained in the init agent runtime container. These are the basic tools for sandbox running.
+	ShouldInjectAgentRuntime = "agents.kruise.io/should-inject-agent-runtime"
 )
 
 // EDIT THIS FILE!  THIS IS SCAFFOLDING FOR YOU TO OWN!

go.mod

@@ -21,6 +21,7 @@ require (
 	golang.org/x/time v0.11.0
 	google.golang.org/grpc v1.75.1
 	google.golang.org/protobuf v1.36.11
+	gopkg.in/evanphx/json-patch.v4 v4.12.0
 	k8s.io/api v0.33.0
 	k8s.io/apimachinery v0.33.0
 	k8s.io/apiserver v0.33.0
@@ -103,7 +104,6 @@ require (
 	gomodules.xyz/jsonpatch/v2 v2.4.0 // indirect
 	google.golang.org/genproto/googleapis/api v0.0.0-20250908214217-97024824d090 // indirect
 	google.golang.org/genproto/googleapis/rpc v0.0.0-20250908214217-97024824d090 // indirect
-	gopkg.in/evanphx/json-patch.v4 v4.12.0 // indirect
 	gopkg.in/inf.v0 v0.9.1 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 	k8s.io/apiextensions-apiserver v0.33.0 // indirect

pkg/controller/sandbox/core/config_types.go

@@ -0,0 +1,21 @@
+package core
+
+import (
+	corev1 "k8s.io/api/core/v1"
+)
+
+const (
+	KEY_CSI_INJECTION_CONFIG     = "csi-config"
+	KEY_RUNTIME_INJECTION_CONFIG = "agent-runtime-config"
+	sandboxInjectionConfigName   = "sandbox-injection-config"
+)
+
+type SidecarInjectConfig struct {
+	// Configuration injection for the main container (by convention, one container is designated as the main container)
+	// Injection configuration items for business-specified main containers, such as volumeMount, environment variables, etc. Format: corev1.Container
+	MainContainer corev1.Container `json:"mainContainer"`
+	// Support injection for multiple independent sidecar containers; CSI container plugins are all injected from this
+	Sidecars []corev1.Container `json:"csiSidecar"`
+	// Support injection for volume mount configurations
+	Volumes []corev1.Volume `json:"volume"`
+}

pkg/controller/sandbox/core/sidecar_config_inject.go

@@ -0,0 +1,254 @@
+package core
+
+import (
+	"context"
+	"encoding/json"
+
+	corev1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/api/errors"
+	"k8s.io/apimachinery/pkg/types"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	logf "sigs.k8s.io/controller-runtime/pkg/log"
+
+	agentsv1alpha1 "github.com/openkruise/agents/api/v1alpha1"
+	"github.com/openkruise/agents/pkg/utils/webhookutils"
+)
+
+func enableInjectCsiMountConfig(sandbox *agentsv1alpha1.Sandbox) bool {
+	return sandbox.Annotations[agentsv1alpha1.ShouldInjectCsiMount] == "true"
+}
+
+func enableInjectAgentRuntimeConfig(sandbox *agentsv1alpha1.Sandbox) bool {
+	return sandbox.Annotations[agentsv1alpha1.ShouldInjectAgentRuntime] == "true"
+}
+
+// fetchInjectionConfiguration retrieves the sidecar injection configuration from the ConfigMap.
+// It attempts to fetch the ConfigMap named sandboxInjectionConfigName from the sandbox-system namespace.
+// If the ConfigMap exists, it returns the data map containing configuration keys like
+// KEY_CSI_INJECTION_CONFIG and KEY_RUNTIME_INJECTION_CONFIG. If the ConfigMap is not found
+// or any error occurs during retrieval, it returns nil data and no error.
+//
+// Parameters:
+//   - ctx: context.Context for the operation
+//   - client: Kubernetes client.Interface used to retrieve the ConfigMap
+//
+// Returns:
+//   - map[string]string: The configuration data from the ConfigMap, or nil if not found or on error
+//   - error: Always returns nil (errors are logged but not propagated)
+func fetchInjectionConfiguration(ctx context.Context, cli client.Client) (map[string]string, error) {
+	logger := logf.FromContext(ctx)
+	config := &corev1.ConfigMap{}
+	err := cli.Get(ctx, types.NamespacedName{
+		Namespace: webhookutils.GetNamespace(), // Todo considering the security concern and rbac issue
+		Name:      sandboxInjectionConfigName,
+	}, config)
+	if err != nil {
+		if errors.IsNotFound(err) {
+			logger.Info("no found the injection configuration, using default")
+			return map[string]string{}, nil
+		}
+		return map[string]string{}, err
+	}
+	return config.Data, nil
+}
+
+// parseCSIMountConfig parses the CSI sidecar injection configuration from raw ConfigMap data.
+// It extracts the value associated with KEY_CSI_INJECTION_CONFIG key and unmarshals it into
+// a SidecarInjectConfig struct. If the key is not present in the configRaw map, it returns
+// an empty SidecarInjectConfig without error. The configuration includes the main container
+// settings, CSI sidecar containers list, and volumes to be injected.
+//
+// Parameters:
+//   - ctx: context.Context for the operation (currently unused but kept for future extensibility)
+//   - configRaw: map[string]string containing the raw ConfigMap data with potential CSI config
+//
+// Returns:
+//   - SidecarInjectConfig: The parsed configuration containing main container, sidecars, and volumes
+//   - error: An error if JSON unmarshaling fails, nil otherwise
+func parseCSIMountConfig(ctx context.Context, configRaw map[string]string) (SidecarInjectConfig, error) {
+	log := logf.FromContext(ctx)
+	csiMountConfig := SidecarInjectConfig{}
+
+	configValue, exists := configRaw[KEY_CSI_INJECTION_CONFIG]
+	if !exists || configValue == "" {
+		log.Info("csi container config not found or empty, using default configuration")
+		return csiMountConfig, nil
+	}
+
+	err := json.Unmarshal([]byte(configRaw[KEY_CSI_INJECTION_CONFIG]), &csiMountConfig)
+	if err != nil {
+		log.Error(err, "failed to json unmarshal csi mount config")
+		return csiMountConfig, err
+	}
+	return csiMountConfig, nil
+}
+
+// parseAgentRuntimeConfig parses the agent runtime injection configuration from raw ConfigMap data.
+// It extracts the value associated with KEY_RUNTIME_INJECTION_CONFIG key and unmarshals it into
+// a SidecarInjectConfig struct. If the key is not present in the configRaw map, it returns
+// an empty SidecarInjectConfig without error. The configuration includes the main container
+// settings with lifecycle hooks, init runtime containers list, and volumes to be injected.
+//
+// Parameters:
+//   - ctx: context.Context for the operation (currently unused but kept for future extensibility)
+//   - configRaw: map[string]string containing the raw ConfigMap data with potential runtime config
+//
+// Returns:
+//   - SidecarInjectConfig: The parsed configuration containing main container, sidecars (init containers), and volumes
+//   - error: An error if JSON unmarshaling fails, nil otherwise
+func parseAgentRuntimeConfig(ctx context.Context, configRaw map[string]string) (SidecarInjectConfig, error) {
+	log := logf.FromContext(ctx)
+	csiMountConfig := SidecarInjectConfig{}
+
+	configValue, exists := configRaw[KEY_RUNTIME_INJECTION_CONFIG]
+	if !exists || configValue == "" {
+		log.Info("agent runtime config not found or empty, using default configuration")
+		return csiMountConfig, nil
+	}
+
+	err := json.Unmarshal([]byte(configValue), &csiMountConfig)
+	if err != nil {
+		log.Error(err, "failed to json unmarshal agent runtime config")
+		return csiMountConfig, err
+	}
+	return csiMountConfig, nil
+}
+
+// setCSIMountContainer injects CSI mount configurations into the SandboxTemplate's pod spec.
+// It configures the main container (first container in the spec) with CSI sidecar settings,
+// appends additional CSI sidecar containers, and mounts shared volumes.
+// Volumes are only added if they don't already exist in the template.
+func setCSIMountContainer(ctx context.Context, obj *corev1.PodTemplateSpec, config SidecarInjectConfig) {
+	log := logf.FromContext(ctx)
+
+	// set main container, the first container is the main container
+	if len(obj.Spec.Containers) == 0 {
+		log.Info("no found the template containers")
+		return
+	}
+
+	mainContainer := &obj.Spec.Containers[0]
+	setMainContainerWhenInjectCSISidecar(mainContainer, config)
+
+	// set csi sidecars
+	for _, csiSidecar := range config.Sidecars {
+		obj.Spec.Containers = append(obj.Spec.Containers, csiSidecar)
+	}
+
+	// set share volume
+	if len(config.Volumes) > 0 {
+		if obj.Spec.Volumes == nil {
+			obj.Spec.Volumes = make([]corev1.Volume, 0, len(config.Volumes))
+		}
+		for _, vol := range config.Volumes {
+			if findVolumeByName(obj.Spec.Volumes, vol.Name) {
+				continue
+			}
+			obj.Spec.Volumes = append(obj.Spec.Volumes, vol)
+		}
+	}
+}
+
+// setMainContainerWhenInjectCSISidecar configures the main container with environment variables and volume mounts from the CSI sidecar configuration.
+// It appends environment variables and volume mounts to the main container, skipping any that already exist (matched by name) to avoid duplicates.
+func setMainContainerWhenInjectCSISidecar(mainContainer *corev1.Container, config SidecarInjectConfig) {
+	// append some envs in main container when processing csi mount
+	if mainContainer.Env == nil {
+		mainContainer.Env = make([]corev1.EnvVar, 0, 1)
+	}
+	for _, env := range config.MainContainer.Env {
+		if findEnvByName(mainContainer.Env, env.Name) {
+			continue
+		}
+		mainContainer.Env = append(mainContainer.Env, env)
+	}
+
+	// append some volumeMounts config in main container
+	if config.MainContainer.VolumeMounts != nil {
+		if mainContainer.VolumeMounts == nil {
+			mainContainer.VolumeMounts = make([]corev1.VolumeMount, 0, len(config.MainContainer.VolumeMounts))
+		}
+		for _, volMount := range config.MainContainer.VolumeMounts {
+			if findVolumeMountByName(mainContainer.VolumeMounts, volMount.Name) {
+				continue
+			}
+			mainContainer.VolumeMounts = append(mainContainer.VolumeMounts, volMount)
+		}
+	}
+}
+
+// setAgentRuntimeContainer injects agent runtime configurations into the SandboxTemplate's pod spec.
+// It appends agent runtime containers as init containers and configures the main container (first container) with runtime settings.
+// The init containers run before the main containers to prepare the runtime environment.
+func setAgentRuntimeContainer(ctx context.Context, obj *corev1.PodTemplateSpec, config SidecarInjectConfig) {
+	log := logf.FromContext(ctx)
+
+	// append init agent runtime container
+	if obj.Spec.InitContainers == nil {
+		obj.Spec.InitContainers = make([]corev1.Container, 0, 1)
+	}
+	obj.Spec.InitContainers = append(obj.Spec.InitContainers, config.Sidecars...)
+
+	if len(obj.Spec.Containers) == 0 {
+		log.Info("no found the template container, default main container is the first container")
+		return
+	}
+	mainContainer := &obj.Spec.Containers[0]
+	setMainContainerConfigWhenInjectRuntimeSidecar(mainContainer, config)
+
+	obj.Spec.Volumes = append(obj.Spec.Volumes, config.Volumes...)
+}
+
+func setMainContainerConfigWhenInjectRuntimeSidecar(mainContainer *corev1.Container, config SidecarInjectConfig) {
+	// set main container lifecycle
+	if mainContainer.Lifecycle == nil {
+		mainContainer.Lifecycle = &corev1.Lifecycle{}
+	}
+	if mainContainer.Lifecycle.PostStart == nil {
+		mainContainer.Lifecycle.PostStart = &corev1.LifecycleHandler{}
+	}
+
+	// using config to override the main container lifecycle post start
+	if config.MainContainer.Lifecycle != nil && config.MainContainer.Lifecycle.PostStart != nil {
+		mainContainer.Lifecycle.PostStart = config.MainContainer.Lifecycle.PostStart
+	}
+
+	// set main container env
+	if mainContainer.Env == nil {
+		mainContainer.Env = make([]corev1.EnvVar, 0, len(config.MainContainer.Env))
+	}
+	mainContainer.Env = append(mainContainer.Env, config.MainContainer.Env...)
+
+	// set main container volumeMounts
+	if mainContainer.VolumeMounts == nil {
+		mainContainer.VolumeMounts = make([]corev1.VolumeMount, 0, len(config.MainContainer.VolumeMounts))
+	}
+	mainContainer.VolumeMounts = append(mainContainer.VolumeMounts, config.MainContainer.VolumeMounts...)
+}
+
+func findVolumeMountByName(volumeMounts []corev1.VolumeMount, name string) bool {
+	for _, volumeMount := range volumeMounts {
+		if volumeMount.Name == name {
+			return true
+		}
+	}
+	return false
+}
+
+func findVolumeByName(volumes []corev1.Volume, name string) bool {
+	for _, volume := range volumes {
+		if volume.Name == name {
+			return true
+		}
+	}
+	return false
+}
+
+func findEnvByName(envs []corev1.EnvVar, name string) bool {
+	for _, env := range envs {
+		if env.Name == name {
+			return true
+		}
+	}
+	return false
+}