✨ Console integration followups (#1498)

* ✨ Add CronJob name generation functions and tests for container and k8s scans
✨ Implement cleanup of stale CronJobs in container and k8s scan handlers
✨ Add Workload Identity Federation support for container registry authentication and refactor related constants
✨ Update user manual with Workload Identity Federation setup instructions for GKE, EKS, and AKS
✨ Update wifInitContainer to generate kubeconfig with bearer token for GKE clusters
🧹 Refactor audit config tests to use dynamic cron job names for Kubernetes and container scans
✨ Enhance cron job name generation to handle long integration and cluster names
🧹 Fix cron job name validation to check for cluster name inclusion when not truncated

* ✨ Refactor: Remove legacy CronJob name functions and update label selectors for consistency

* 🐛 fix: skip WIF registry credentials when static pull secrets are configured for container image scanning

Author: slntopp

.github/actions/spelling/expect.txt

@@ -1,8 +1,9 @@
 AAD
 AADSTS
-acr
+Acr
 artifactory
 artifactregistry
+auditconfig
 bak
 bitnami
 containerregistry
@@ -28,7 +29,6 @@ psat
 rolearn
 selfsigned
 servicemonitor
-servicemonitors
 spiffe
 SResources
 SVIDs

controllers/container_image/deployment_handler.go

@@ -29,14 +29,8 @@ type DeploymentHandler struct {
 }
 
 func (n *DeploymentHandler) Reconcile(ctx context.Context) (ctrl.Result, error) {
-	// TODO: remove in next version
-	// Delete the old container scanning cronjob if it exists
-	if err := k8s.DeleteIfExists(ctx,
-		n.KubeClient,
-		&batchv1.CronJob{ObjectMeta: metav1.ObjectMeta{
-			Name:      OldCronJobName(n.Mondoo.Name),
-			Namespace: n.Mondoo.Namespace,
-		}}); err != nil {
+	// Clean up CronJobs with stale names (from old naming schemes)
+	if err := n.cleanupStaleCronJobs(ctx); err != nil {
 		return ctrl.Result{}, err
 	}
 
@@ -235,3 +229,26 @@ func (n *DeploymentHandler) cleanupWIFServiceAccount(ctx context.Context) error
 	}
 	return nil
 }
+
+// cleanupStaleCronJobs removes CronJobs from old naming schemes by label selection.
+func (n *DeploymentHandler) cleanupStaleCronJobs(ctx context.Context) error {
+	cronJobs := &batchv1.CronJobList{}
+	listOpts := &client.ListOptions{
+		Namespace:     n.Mondoo.Namespace,
+		LabelSelector: labels.SelectorFromSet(CronJobLabels(*n.Mondoo)),
+	}
+	if err := n.KubeClient.List(ctx, cronJobs, listOpts); err != nil {
+		return err
+	}
+
+	expectedName := CronJobName(n.Mondoo.Name)
+	for i := range cronJobs.Items {
+		if cronJobs.Items[i].Name != expectedName {
+			logger.Info("Deleting stale container scan CronJob", "name", cronJobs.Items[i].Name)
+			if err := k8s.DeleteIfExists(ctx, n.KubeClient, &cronJobs.Items[i]); err != nil {
+				return err
+			}
+		}
+	}
+	return nil
+}

controllers/container_image/resources.go

@@ -11,15 +11,13 @@ import (
 	// That's the mod k8s relies on https://github.com/kubernetes/kubernetes/blob/master/go.mod#L63
 
 	"go.mondoo.com/mondoo-operator/api/v1alpha2"
-	k8s_scan "go.mondoo.com/mondoo-operator/controllers/k8s_scan"
 	"go.mondoo.com/mondoo-operator/pkg/constants"
 	"go.mondoo.com/mondoo-operator/pkg/feature_flags"
 	"go.mondoo.com/mondoo-operator/pkg/utils/k8s"
 	mondoo "go.mondoo.com/mondoo-operator/pkg/utils/mondoo"
 	"go.mondoo.com/mql/v13/providers-sdk/v1/inventory"
 	batchv1 "k8s.io/api/batch/v1"
 	corev1 "k8s.io/api/core/v1"
-	"k8s.io/apimachinery/pkg/api/resource"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/utils/ptr"
 	"sigs.k8s.io/yaml"
@@ -193,7 +191,7 @@ func CronJob(image, integrationMrn, clusterUid, privateRegistrySecretName string
 		)
 
 		// Add init container for registry credential generation
-		podSpec.InitContainers = append(podSpec.InitContainers, registryWIFInitContainer(wif))
+		podSpec.InitContainers = append(podSpec.InitContainers, k8s.RegistryWIFInitContainer(wif))
 
 		// AKS Workload Identity webhook requires this label on the pod template only.
 		// Copy labels so we don't mutate the CronJob/Job metadata.
@@ -232,7 +230,7 @@ func OldCronJobName(prefix string) string {
 }
 
 func CronJobName(prefix string) string {
-	return fmt.Sprintf("%s%s", prefix, CronJobNameSuffix)
+	return k8s.CronJobName("container-scan", prefix)
 }
 
 func ConfigMap(integrationMRN, clusterUID string, m v1alpha2.MondooAuditConfig, cfg v1alpha2.MondooOperatorConfig) (*corev1.ConfigMap, error) {
@@ -380,175 +378,3 @@ func validateContainerRegistryWIF(wif *v1alpha2.WorkloadIdentityConfig) error {
 
 	return nil
 }
-
-// registryWIFInitContainer creates an init container that generates docker config credentials
-// using cloud-native Workload Identity Federation
-func registryWIFInitContainer(wif *v1alpha2.WorkloadIdentityConfig) corev1.Container {
-	var image, shell, script string
-	var env []corev1.EnvVar
-
-	// Common retry wrapper for transient failures
-	retryWrapper := `set -euo pipefail
-# Retry wrapper for transient failures
-retry() {
-  local max_attempts=3
-  local delay=5
-  local attempt=1
-  while [ $attempt -le $max_attempts ]; do
-    if "$@"; then
-      return 0
-    fi
-    echo "Attempt $attempt failed, retrying in ${delay}s..."
-    sleep $delay
-    attempt=$((attempt + 1))
-  done
-  echo "All $max_attempts attempts failed"
-  return 1
-}
-`
-
-	switch wif.Provider {
-	case v1alpha2.CloudProviderGKE:
-		image = k8s_scan.GCloudSDKImage
-		shell = "/bin/bash"
-		script = retryWrapper + `
-# Use WIF identity to get an access token for Artifact Registry / GCR
-TOKEN=$(retry gcloud auth print-access-token)
-AUTH=$(echo -n "oauth2accesstoken:${TOKEN}" | base64 -w0)
-
-# All GCP regions and multi-region locations that host Artifact Registry.
-# Docker config requires exact hostname matches, so we enumerate them all.
-AR_LOCATIONS="
-africa-south1 asia-east1 asia-east2 asia-northeast1 asia-northeast2 asia-northeast3
-asia-south1 asia-south2 asia-southeast1 asia-southeast2
-australia-southeast1 australia-southeast2
-europe-central2 europe-north1 europe-southwest1 europe-west1 europe-west2
-europe-west3 europe-west4 europe-west6 europe-west8 europe-west9 europe-west10 europe-west12
-me-central1 me-central2 me-west1
-northamerica-northeast1 northamerica-northeast2
-southamerica-east1 southamerica-west1
-us-central1 us-east1 us-east4 us-east5 us-south1 us-west1 us-west2 us-west3 us-west4
-asia europe us
-"
-
-AUTHS=""
-add_auth() {
-  [ -n "$AUTHS" ] && AUTHS="${AUTHS},"
-  AUTHS="${AUTHS}\"$1\":{\"auth\":\"${AUTH}\"}"
-}
-
-for loc in $AR_LOCATIONS; do
-  add_auth "${loc}-docker.pkg.dev"
-done
-
-# Legacy GCR endpoints
-for host in gcr.io us.gcr.io eu.gcr.io asia.gcr.io; do
-  add_auth "$host"
-done
-
-cat > /etc/opt/mondoo/docker/config.json <<DOCKEREOF
-{"auths":{${AUTHS}}}
-DOCKEREOF
-echo "Docker config generated for $(echo "$AUTHS" | tr ',' '\n' | wc -l) registry endpoints"
-`
-		env = []corev1.EnvVar{
-			{Name: "HOME", Value: "/tmp"},
-		}
-
-	case v1alpha2.CloudProviderEKS:
-		image = k8s_scan.AWSCLIImage
-		shell = "/bin/bash"
-		script = retryWrapper + `
-# Use IRSA identity to get ECR login password
-PASSWORD=$(retry aws ecr get-login-password --region "$AWS_REGION")
-
-# Derive registry URL from role ARN account ID and region
-ACCOUNT_ID=$(echo "$ROLE_ARN" | cut -d: -f5)
-REGISTRY="${ACCOUNT_ID}.dkr.ecr.${AWS_REGION}.amazonaws.com"
-
-# Write docker config
-AUTH=$(echo -n "AWS:${PASSWORD}" | base64 -w0)
-cat > /etc/opt/mondoo/docker/config.json <<DOCKEREOF
-{
-  "auths": {
-    "${REGISTRY}": { "auth": "${AUTH}" }
-  }
-}
-DOCKEREOF
-echo "Docker config generated for ECR registry: ${REGISTRY}"
-`
-		env = []corev1.EnvVar{
-			{Name: "HOME", Value: "/tmp"},
-			{Name: "AWS_REGION", Value: wif.EKS.Region},
-			{Name: "ROLE_ARN", Value: wif.EKS.RoleARN},
-		}
-
-	case v1alpha2.CloudProviderAKS:
-		image = k8s_scan.AzureCLIImage
-		shell = "/bin/bash"
-		script = retryWrapper + `
-# Azure WIF webhook injects AZURE_CLIENT_ID, AZURE_TENANT_ID, AZURE_FEDERATED_TOKEN_FILE
-retry az login --federated-token "$(cat "$AZURE_FEDERATED_TOKEN_FILE")" \
-  --service-principal \
-  -u "$AZURE_CLIENT_ID" \
-  -t "$AZURE_TENANT_ID"
-
-# Get ACR access token
-TOKEN=$(retry az acr login --name "$ACR_LOGIN_SERVER" --expose-token --output tsv --query accessToken)
-
-# Write docker config
-AUTH=$(echo -n "00000000-0000-0000-0000-000000000000:${TOKEN}" | base64 -w0)
-cat > /etc/opt/mondoo/docker/config.json <<DOCKEREOF
-{
-  "auths": {
-    "${ACR_LOGIN_SERVER}": { "auth": "${AUTH}" }
-  }
-}
-DOCKEREOF
-echo "Docker config generated for ACR: ${ACR_LOGIN_SERVER}"
-`
-		env = []corev1.EnvVar{
-			{Name: "HOME", Value: "/tmp"},
-			{Name: "ACR_LOGIN_SERVER", Value: wif.AKS.LoginServer},
-		}
-
-	default:
-		image = "busybox:1.36"
-		shell = "/bin/sh"
-		script = `echo "ERROR: Unknown workload identity provider"; exit 1`
-		env = []corev1.EnvVar{}
-	}
-
-	return corev1.Container{
-		Name:            "generate-registry-creds",
-		Image:           image,
-		ImagePullPolicy: corev1.PullIfNotPresent,
-		Command:         []string{shell, "-c", script},
-		Env:             env,
-		VolumeMounts: []corev1.VolumeMount{
-			{Name: "docker-config", MountPath: "/etc/opt/mondoo/docker"},
-			{Name: "temp", MountPath: "/tmp"},
-		},
-		Resources: corev1.ResourceRequirements{
-			Requests: corev1.ResourceList{
-				corev1.ResourceCPU:    resource.MustParse("50m"),
-				corev1.ResourceMemory: resource.MustParse("64Mi"),
-			},
-			Limits: corev1.ResourceList{
-				corev1.ResourceCPU:    resource.MustParse("200m"),
-				corev1.ResourceMemory: resource.MustParse("256Mi"),
-			},
-		},
-		TerminationMessagePath:   "/dev/termination-log",
-		TerminationMessagePolicy: corev1.TerminationMessageReadFile,
-		SecurityContext: &corev1.SecurityContext{
-			AllowPrivilegeEscalation: ptr.To(false),
-			ReadOnlyRootFilesystem:   ptr.To(true),
-			RunAsNonRoot:             ptr.To(true),
-			RunAsUser:                ptr.To(int64(101)),
-			Capabilities: &corev1.Capabilities{
-				Drop: []corev1.Capability{"ALL"},
-			},
-		},
-	}
-}

controllers/k8s_scan/deployment_handler.go

@@ -141,6 +141,11 @@ type DeploymentHandler struct {
 }
 
 func (n *DeploymentHandler) Reconcile(ctx context.Context) (ctrl.Result, error) {
+	// Clean up CronJobs with stale names (from old naming schemes)
+	if err := n.cleanupStaleCronJobs(ctx); err != nil {
+		return ctrl.Result{}, err
+	}
+
 	hasExternalClusters := len(n.Mondoo.Spec.KubernetesResources.ExternalClusters) > 0
 
 	if !n.Mondoo.Spec.KubernetesResources.Enable {
@@ -399,10 +404,7 @@ func (n *DeploymentHandler) cleanupOrphanedExternalClusterResources(ctx context.
 	cronJobs := &batchv1.CronJobList{}
 	listOpts := &client.ListOptions{
 		Namespace: n.Mondoo.Namespace,
-		LabelSelector: labels.SelectorFromSet(map[string]string{
-			"app":       "mondoo-k8s-scan",
-			"mondoo_cr": n.Mondoo.Name,
-		}),
+		LabelSelector: labels.SelectorFromSet(CronJobLabels(*n.Mondoo)),
 	}
 	if err := n.KubeClient.List(ctx, cronJobs, listOpts); err != nil {
 		return err
@@ -568,10 +570,7 @@ func (n *DeploymentHandler) garbageCollectIfNeeded(ctx context.Context, clusterU
 	cronJobs := &batchv1.CronJobList{}
 	listOpts := &client.ListOptions{
 		Namespace: n.Mondoo.Namespace,
-		LabelSelector: labels.SelectorFromSet(map[string]string{
-			"app":       "mondoo-k8s-scan",
-			"mondoo_cr": n.Mondoo.Name,
-		}),
+		LabelSelector: labels.SelectorFromSet(CronJobLabels(*n.Mondoo)),
 	}
 	if err := n.KubeClient.List(ctx, cronJobs, listOpts); err != nil {
 		logger.Error(err, "Failed to list CronJobs for garbage collection")
@@ -718,3 +717,40 @@ func (n *DeploymentHandler) syncWIFServiceAccount(ctx context.Context, cluster v
 
 	return nil
 }
+
+// cleanupStaleCronJobs removes CronJobs from old naming schemes by label selection.
+// CronJobs belonging to removed external clusters are skipped here — cleanupOrphanedExternalClusterResources
+// handles those so it can also clean up associated ConfigMaps, ServiceAccounts, and Secrets.
+func (n *DeploymentHandler) cleanupStaleCronJobs(ctx context.Context) error {
+	cronJobs := &batchv1.CronJobList{}
+	listOpts := &client.ListOptions{
+		Namespace: n.Mondoo.Namespace,
+		LabelSelector: labels.SelectorFromSet(CronJobLabels(*n.Mondoo)),
+	}
+	if err := n.KubeClient.List(ctx, cronJobs, listOpts); err != nil {
+		return err
+	}
+
+	expected := map[string]bool{
+		CronJobName(n.Mondoo.Name): true,
+	}
+	configuredClusters := make(map[string]bool)
+	for _, cluster := range n.Mondoo.Spec.KubernetesResources.ExternalClusters {
+		expected[ExternalClusterCronJobName(n.Mondoo.Name, cluster.Name)] = true
+		configuredClusters[cluster.Name] = true
+	}
+
+	for i := range cronJobs.Items {
+		if expected[cronJobs.Items[i].Name] {
+			continue
+		}
+		if clusterName, ok := cronJobs.Items[i].Labels["cluster_name"]; ok && !configuredClusters[clusterName] {
+			continue
+		}
+		logger.Info("Deleting stale k8s scan CronJob", "name", cronJobs.Items[i].Name)
+		if err := k8s.DeleteIfExists(ctx, n.KubeClient, &cronJobs.Items[i]); err != nil {
+			return err
+		}
+	}
+	return nil
+}

controllers/k8s_scan/deployment_handler_test.go

@@ -1530,11 +1530,11 @@ func TestExternalClusterNaming(t *testing.T) {
 		t.Run(tt.prefix+"-"+tt.clusterName, func(t *testing.T) {
 			// Test CronJob name
 			cronJobName := ExternalClusterCronJobName(tt.prefix, tt.clusterName)
-			if !strings.HasPrefix(cronJobName, tt.prefix) {
-				t.Errorf("CronJob name should start with prefix %q, got %q", tt.prefix, cronJobName)
+			if !strings.HasPrefix(cronJobName, "mondoo-k8s-scan-") {
+				t.Errorf("CronJob name should start with %q, got %q", "mondoo-k8s-scan-", cronJobName)
 			}
-			if !strings.HasSuffix(cronJobName, tt.clusterName) {
-				t.Errorf("CronJob name should end with cluster name %q, got %q", tt.clusterName, cronJobName)
+			if !strings.Contains(cronJobName, tt.clusterName) && len(cronJobName) < 52 {
+				t.Errorf("CronJob name should contain cluster name %q when not truncated, got %q", tt.clusterName, cronJobName)
 			}
 
 			// Test ConfigMap name