fix(lmeval): merge multiple CA sources into single bundle for HTTPS pods

The previous CA injection mounted a single key from odh-trusted-ca-bundle,
which contains only public/system CAs. Cluster-internal services using
OpenShift service-serving certificates (*.svc.cluster.local) are signed by
a different CA in the openshift-service-ca.crt ConfigMap, so the pod still
got SSLCertVerificationError.

Additionally, REQUESTS_CA_BUNDLE replaces Python's default trust store
rather than appending, so mounting only one CA source loses trust in all
others.

Fix: replace findCABundle with findAndMergeCABundle, which collects PEM
data from both odh-trusted-ca-bundle and openshift-service-ca.crt
(best-effort, each skipped if absent), concatenates them, and creates a
per-job merged ConfigMap (<jobName>-ca-bundle) with an owner reference
for automatic GC. The pod mounts this merged ConfigMap so
REQUESTS_CA_BUNDLE contains all relevant CAs.

Co-Authored-By: Claude Opus 4.6 <[email protected]>

rh-pre-commit.version: 2.3.2
rh-pre-commit.check-secrets: ENABLED

Author: SudipSinha

controllers/lmes/constants.go

@@ -47,12 +47,20 @@ const (
 	ServiceName                = "LMES"
 
 	// DefaultCABundleConfigMapName is the standard RHOAI ConfigMap that holds the cluster CA bundle.
-	// It is injected into managed namespaces by the RHOAI operator and trusted by cluster-internal
-	// HTTPS services (e.g., KServe external routes using self-signed certs).
+	// It is injected into managed namespaces by the RHOAI operator.
 	DefaultCABundleConfigMapName = "odh-trusted-ca-bundle"
-	// CABundleVolumeName is the volume name used when auto-mounting the cluster CA bundle.
+	// ServiceCAConfigMapName is the well-known ConfigMap that contains the OpenShift
+	// service-serving CA, used by cluster-internal HTTPS services (*.svc.cluster.local).
+	ServiceCAConfigMapName = "openshift-service-ca.crt"
+	// ServiceCAKey is the standard key within the service-serving CA ConfigMap.
+	ServiceCAKey = "service-ca.crt"
+	// MergedCABundleKey is the key used in the per-job merged CA ConfigMap.
+	MergedCABundleKey = "merged-ca-bundle.crt"
+	// MergedCAConfigMapSuffix is appended to the job name to form the merged CA ConfigMap name.
+	MergedCAConfigMapSuffix = "-ca-bundle"
+	// CABundleVolumeName is the volume name used when auto-mounting the merged CA bundle.
 	CABundleVolumeName = "odh-ca-bundle"
-	// CABundleMountPath is the file path at which the CA bundle is mounted inside the lm-eval pod.
+	// CABundleMountPath is the file path at which the merged CA bundle is mounted inside the lm-eval pod.
 	// REQUESTS_CA_BUNDLE is set to this path so Python's requests library picks it up automatically.
 	CABundleMountPath = "/etc/ssl/certs/odh-ca-bundle.crt"
 

controllers/lmes/lmevaljob_controller.go

@@ -31,6 +31,7 @@ import (
 	"github.com/trustyai-explainability/trustyai-service-operator/controllers/utils"
 
 	corev1 "k8s.io/api/core/v1"
+	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/types"
@@ -563,20 +564,16 @@ func (r *LMEvalJobReconciler) handleNewCR(ctx context.Context, log logr.Logger,
 		return ctrl.Result{}, err
 	}
 
-	// Bug 1: Auto-inject the cluster CA bundle when the model endpoint uses HTTPS
-	// and verify_certificate is not already set. This handles the common RHOAI case
-	// where KServe external routes use self-signed certs trusted by the cluster but
-	// not by Python's default trust store inside the pod.
 	var caBundle *corev1.ConfigMap
 	var caBundleKey string
 	if hasHTTPSBaseURL(job) && !hasExplicitVerifyCertificate(job) {
-		if cm, key, err := r.findCABundle(ctx, job.Namespace); err == nil {
+		if cm, key, err := r.findAndMergeCABundle(ctx, job); err == nil {
 			caBundle = cm
 			caBundleKey = key
-			log.Info("auto-injecting cluster CA bundle for HTTPS endpoint",
-				"configmap", DefaultCABundleConfigMapName, "key", key)
+			log.Info("auto-injecting merged CA bundle for HTTPS endpoint",
+				"configmap", cm.Name, "key", key)
 		} else {
-			log.Info("HTTPS base_url detected but cluster CA bundle not found, proceeding without auto-injection",
+			log.Info("HTTPS base_url detected but no CA bundle data found, proceeding without auto-injection",
 				"error", err.Error())
 		}
 	}
@@ -825,11 +822,10 @@ func (r *LMEvalJobReconciler) handleResume(ctx context.Context, log logr.Logger,
 		return r.pullingJobs.addOrUpdate(string(job.GetUID()), Options.PodCheckingInterval), err
 	}
 
-	// Bug 1: Apply the same CA bundle auto-injection on resume as on initial scheduling.
 	var caBundle *corev1.ConfigMap
 	var caBundleKey string
 	if hasHTTPSBaseURL(job) && !hasExplicitVerifyCertificate(job) {
-		if cm, key, err := r.findCABundle(ctx, job.Namespace); err == nil {
+		if cm, key, err := r.findAndMergeCABundle(ctx, job); err == nil {
 			caBundle = cm
 			caBundleKey = key
 		}
@@ -1830,18 +1826,84 @@ func getLastScheduledGeneration(job *lmesv1alpha1.LMEvalJob) int64 {
 	return gen
 }
 
-// findCABundle looks for the standard RHOAI cluster CA bundle ConfigMap in the
-// given namespace. It tries well-known key names and returns the ConfigMap and
-// the matching key on success.
-func (r *LMEvalJobReconciler) findCABundle(ctx context.Context, namespace string) (*corev1.ConfigMap, string, error) {
-	cm := &corev1.ConfigMap{}
-	if err := r.Get(ctx, types.NamespacedName{Namespace: namespace, Name: DefaultCABundleConfigMapName}, cm); err != nil {
-		return nil, "", err
+// findAndMergeCABundle collects CA certificates from well-known cluster sources
+// and creates a per-job ConfigMap containing the merged bundle. This ensures
+// that REQUESTS_CA_BUNDLE (which replaces the default trust store) contains
+// both public CAs and the OpenShift service-serving CA.
+//
+// Sources checked (best-effort, each is skipped if not found):
+//   - odh-trusted-ca-bundle: ca-bundle.crt, odh-ca-bundle.crt (public/system CAs)
+//   - openshift-service-ca.crt: service-ca.crt (service-serving CA)
+func (r *LMEvalJobReconciler) findAndMergeCABundle(ctx context.Context, job *lmesv1alpha1.LMEvalJob) (*corev1.ConfigMap, string, error) {
+	log := log.FromContext(ctx)
+	var pemBlocks []string
+
+	// Source 1: odh-trusted-ca-bundle (public/system CAs)
+	odhCM := &corev1.ConfigMap{}
+	if err := r.Get(ctx, types.NamespacedName{Namespace: job.Namespace, Name: DefaultCABundleConfigMapName}, odhCM); err == nil {
+		for _, key := range []string{"ca-bundle.crt", "odh-ca-bundle.crt"} {
+			if data, ok := odhCM.Data[key]; ok && strings.TrimSpace(data) != "" {
+				pemBlocks = append(pemBlocks, data)
+				log.V(1).Info("collected CA data", "configmap", DefaultCABundleConfigMapName, "key", key)
+			}
+		}
+	} else if !apierrors.IsNotFound(err) {
+		log.Error(err, "error reading CA bundle ConfigMap", "name", DefaultCABundleConfigMapName)
+	}
+
+	// Source 2: openshift-service-ca.crt (service-serving CA)
+	svcCM := &corev1.ConfigMap{}
+	if err := r.Get(ctx, types.NamespacedName{Namespace: job.Namespace, Name: ServiceCAConfigMapName}, svcCM); err == nil {
+		if data, ok := svcCM.Data[ServiceCAKey]; ok && strings.TrimSpace(data) != "" {
+			pemBlocks = append(pemBlocks, data)
+			log.V(1).Info("collected CA data", "configmap", ServiceCAConfigMapName, "key", ServiceCAKey)
+		}
+	} else if !apierrors.IsNotFound(err) {
+		log.Error(err, "error reading service CA ConfigMap", "name", ServiceCAConfigMapName)
+	}
+
+	if len(pemBlocks) == 0 {
+		return nil, "", fmt.Errorf("no CA bundle data found in namespace %s", job.Namespace)
 	}
-	for _, key := range []string{"ca-bundle.crt", "odh-ca-bundle.crt", "service-ca.crt"} {
-		if _, ok := cm.Data[key]; ok {
-			return cm, key, nil
+
+	merged := strings.Join(pemBlocks, "\n")
+	mergedCMName := job.Name + MergedCAConfigMapSuffix
+
+	mergedCM := &corev1.ConfigMap{}
+	err := r.Get(ctx, types.NamespacedName{Namespace: job.Namespace, Name: mergedCMName}, mergedCM)
+	if apierrors.IsNotFound(err) {
+		ownerRefController := true
+		mergedCM = &corev1.ConfigMap{
+			ObjectMeta: v1.ObjectMeta{
+				Name:      mergedCMName,
+				Namespace: job.Namespace,
+				OwnerReferences: []v1.OwnerReference{
+					{
+						APIVersion: job.APIVersion,
+						Kind:       job.Kind,
+						Name:       job.Name,
+						Controller: &ownerRefController,
+						UID:        job.UID,
+					},
+				},
+			},
+			Data: map[string]string{
+				MergedCABundleKey: merged,
+			},
+		}
+		if err := r.Create(ctx, mergedCM); err != nil {
+			return nil, "", fmt.Errorf("failed to create merged CA ConfigMap: %w", err)
+		}
+	} else if err != nil {
+		return nil, "", fmt.Errorf("failed to read merged CA ConfigMap: %w", err)
+	} else {
+		mergedCM.Data = map[string]string{
+			MergedCABundleKey: merged,
+		}
+		if err := r.Update(ctx, mergedCM); err != nil {
+			return nil, "", fmt.Errorf("failed to update merged CA ConfigMap: %w", err)
 		}
 	}
-	return nil, "", fmt.Errorf("ConfigMap %s has no recognised CA bundle key", DefaultCABundleConfigMapName)
+
+	return mergedCM, MergedCABundleKey, nil
 }

controllers/lmes/lmevaljob_controller_suite_test.go

@@ -216,14 +216,14 @@ var _ = Describe("LMEvalJob CA bundle injection", func() {
 	ctx := context.Background()
 	trueB := true
 
-	It("injects CA bundle volume, mount, and env var when base_url uses HTTPS", func() {
+	It("injects merged CA bundle when base_url uses HTTPS", func() {
 		caConfigMap := &corev1.ConfigMap{
 			ObjectMeta: metav1.ObjectMeta{
 				Name:      lmes.DefaultCABundleConfigMapName,
 				Namespace: testNamespace,
 			},
 			Data: map[string]string{
-				"ca-bundle.crt": "-----BEGIN CERTIFICATE-----\ntest\n-----END CERTIFICATE-----",
+				"ca-bundle.crt": "-----BEGIN CERTIFICATE-----\npublic-ca\n-----END CERTIFICATE-----",
 			},
 		}
 		Expect(k8sClient.Create(ctx, caConfigMap)).Should(Succeed())
@@ -252,32 +252,44 @@ var _ = Describe("LMEvalJob CA bundle injection", func() {
 		}
 		Expect(k8sClient.Create(ctx, job)).Should(Succeed())
 
+		mergedCMName := "test-ca-bundle" + lmes.MergedCAConfigMapSuffix
+
+		// Verify the merged ConfigMap was created
+		mergedCM := &corev1.ConfigMap{}
+		WaitFor(func() error {
+			return k8sClient.Get(ctx, types.NamespacedName{
+				Name: mergedCMName, Namespace: testNamespace,
+			}, mergedCM)
+		}, "merged CA ConfigMap was not created")
+		Expect(mergedCM.Data).To(HaveKey(lmes.MergedCABundleKey))
+		Expect(mergedCM.Data[lmes.MergedCABundleKey]).To(ContainSubstring("public-ca"))
+
 		pod := &corev1.Pod{}
 		WaitFor(func() error {
 			return k8sClient.Get(ctx, types.NamespacedName{
 				Name: "test-ca-bundle", Namespace: testNamespace,
 			}, pod)
 		}, "pod was not created for HTTPS job")
 
-		// Verify CA bundle volume
+		// Verify CA bundle volume points to merged ConfigMap
 		foundVolume := false
 		for _, v := range pod.Spec.Volumes {
 			if v.Name == lmes.CABundleVolumeName {
 				foundVolume = true
 				Expect(v.VolumeSource.ConfigMap).NotTo(BeNil())
-				Expect(v.VolumeSource.ConfigMap.Name).To(Equal(lmes.DefaultCABundleConfigMapName))
+				Expect(v.VolumeSource.ConfigMap.Name).To(Equal(mergedCMName))
 			}
 		}
 		Expect(foundVolume).To(BeTrue(), "CA bundle volume not found on pod")
 
-		// Verify CA bundle volume mount on main container
+		// Verify volume mount uses merged key
 		mainContainer := pod.Spec.Containers[0]
 		foundMount := false
 		for _, m := range mainContainer.VolumeMounts {
 			if m.Name == lmes.CABundleVolumeName {
 				foundMount = true
 				Expect(m.MountPath).To(Equal(lmes.CABundleMountPath))
-				Expect(m.SubPath).To(Equal("ca-bundle.crt"))
+				Expect(m.SubPath).To(Equal(lmes.MergedCABundleKey))
 				Expect(m.ReadOnly).To(BeTrue())
 			}
 		}
@@ -294,6 +306,57 @@ var _ = Describe("LMEvalJob CA bundle injection", func() {
 		Expect(foundEnv).To(BeTrue(), "REQUESTS_CA_BUNDLE env var not found")
 	})
 
+	It("merges both odh-trusted-ca-bundle and openshift-service-ca.crt when both exist", func() {
+		svcCAConfigMap := &corev1.ConfigMap{
+			ObjectMeta: metav1.ObjectMeta{
+				Name:      lmes.ServiceCAConfigMapName,
+				Namespace: testNamespace,
+			},
+			Data: map[string]string{
+				lmes.ServiceCAKey: "-----BEGIN CERTIFICATE-----\nservice-ca\n-----END CERTIFICATE-----",
+			},
+		}
+		Expect(k8sClient.Create(ctx, svcCAConfigMap)).Should(Succeed())
+
+		job := &lmesv1alpha1.LMEvalJob{
+			ObjectMeta: metav1.ObjectMeta{
+				Name:      "test-ca-merged",
+				Namespace: testNamespace,
+			},
+			TypeMeta: metav1.TypeMeta{
+				Kind:       lmesv1alpha1.KindName,
+				APIVersion: lmesv1alpha1.Version,
+			},
+			Spec: lmesv1alpha1.LMEvalJobSpec{
+				AllowOnline:        &trueB,
+				AllowCodeExecution: &trueB,
+				Model:              "hf",
+				ModelArgs: []lmesv1alpha1.Arg{
+					{Name: "pretrained", Value: "google/flan-t5-base"},
+					{Name: "base_url", Value: "https://model.svc.cluster.local:8443"},
+				},
+				TaskList: lmesv1alpha1.TaskList{
+					TaskNames: []string{"task1"},
+				},
+			},
+		}
+		Expect(k8sClient.Create(ctx, job)).Should(Succeed())
+
+		mergedCMName := "test-ca-merged" + lmes.MergedCAConfigMapSuffix
+
+		mergedCM := &corev1.ConfigMap{}
+		WaitFor(func() error {
+			return k8sClient.Get(ctx, types.NamespacedName{
+				Name: mergedCMName, Namespace: testNamespace,
+			}, mergedCM)
+		}, "merged CA ConfigMap was not created")
+
+		Expect(mergedCM.Data[lmes.MergedCABundleKey]).To(ContainSubstring("public-ca"),
+			"merged bundle should contain public CAs from odh-trusted-ca-bundle")
+		Expect(mergedCM.Data[lmes.MergedCABundleKey]).To(ContainSubstring("service-ca"),
+			"merged bundle should contain service-serving CA from openshift-service-ca.crt")
+	})
+
 	It("does not inject CA bundle when base_url uses HTTP", func() {
 		job := &lmesv1alpha1.LMEvalJob{
 			ObjectMeta: metav1.ObjectMeta{