Fall back to Rancher CA bundles for HelmOps

Resolve the CA bundle in the HelmOps controller and store it in
BundleHelmOptions.CABundle so the agent can use it without needing
access to cattle-system secrets. The agent service account only has
access to its own namespace.

Also restart helmops pods in dev/update-controller-k3d so that
redeployments pick up the new controller binary.

Author: thardeck

charts/fleet-crd/templates/crds.yaml

@@ -194,6 +194,20 @@ spec:
 
                     should be downloaded from a helm chart'
                   properties:
+                    helmOpCABundle:
+                      description: 'CABundle is a PEM encoded CA bundle used to validate
+                        TLS connections to
+
+                        the Helm registry. It is resolved by the controller (which
+                        has access to
+
+                        Rancher''s cattle-system CA secrets) and stored here so the
+                        agent can use
+
+                        it without requiring access to those secrets.'
+                      format: byte
+                      nullable: true
+                      type: string
                     helmOpInsecureSkipTLSVerify:
                       description: InsecureSkipTLSverify will use insecure HTTPS to
                         clone the helm app resource.
@@ -2067,6 +2081,20 @@ spec:
                     not a git repository.'
                   nullable: true
                   properties:
+                    helmOpCABundle:
+                      description: 'CABundle is a PEM encoded CA bundle used to validate
+                        TLS connections to
+
+                        the Helm registry. It is resolved by the controller (which
+                        has access to
+
+                        Rancher''s cattle-system CA secrets) and stored here so the
+                        agent can use
+
+                        it without requiring access to those secrets.'
+                      format: byte
+                      nullable: true
+                      type: string
                     helmOpInsecureSkipTLSVerify:
                       description: InsecureSkipTLSverify will use insecure HTTPS to
                         clone the helm app resource.
@@ -8269,6 +8297,20 @@ spec:
                     not a git repository.'
                   nullable: true
                   properties:
+                    helmOpCABundle:
+                      description: 'CABundle is a PEM encoded CA bundle used to validate
+                        TLS connections to
+
+                        the Helm registry. It is resolved by the controller (which
+                        has access to
+
+                        Rancher''s cattle-system CA secrets) and stored here so the
+                        agent can use
+
+                        it without requiring access to those secrets.'
+                      format: byte
+                      nullable: true
+                      type: string
                     helmOpInsecureSkipTLSVerify:
                       description: InsecureSkipTLSverify will use insecure HTTPS to
                         clone the helm app resource.

dev/update-controller-k3d

@@ -20,3 +20,4 @@ fleet_ctx=$(kubectl config current-context)
 k3d image import rancher/fleet:dev -m direct -c "${fleet_ctx#k3d-}"
 kubectl delete pod -l app=fleet-controller -n cattle-fleet-system
 kubectl delete pod -l app=gitjob -n cattle-fleet-system
+kubectl delete pod -l app=helmops -n cattle-fleet-system

e2e/single-cluster/helmop_test.go

@@ -520,6 +520,91 @@ var _ = Describe("HelmOp resource tests with tarball source", Label("infra-setup
 	})
 })
 
+var _ = Describe("HelmOp resource falls back to Rancher CA bundle", Label("infra-setup", "helm-registry"), Ordered, func() {
+	// This test mirrors the GitOps E2E test "should succeed when not configuring any CA"
+	// in go_getter_custom_ca_test.go. The dev/create-secrets script places the root CA
+	// into cattle-system/tls-ca-additional. ChartMuseum is served with a cert signed by
+	// that root CA. A HelmOp with a credentials-only secret (no cacerts) and
+	// InsecureSkipTLSVerify=false must therefore succeed via the Rancher CA fallback.
+	const (
+		name       = "rancher-ca-fallback"
+		secretName = "helmop-rancher-ca-creds"
+	)
+
+	var (
+		namespace string
+		k         kubectl.Command
+	)
+
+	BeforeAll(func() {
+		k = env.Kubectl.Namespace(env.Namespace)
+		out, err := k.Create(
+			"secret", "generic", secretName,
+			"--from-literal=username="+os.Getenv("CI_OCI_USERNAME"),
+			"--from-literal=password="+os.Getenv("CI_OCI_PASSWORD"),
+			// no cacerts — TLS trust must come from cattle-system/tls-ca-additional
+		)
+		if strings.Contains(out, "already exists") {
+			err = nil
+		}
+		Expect(err).ToNot(HaveOccurred(), out)
+	})
+
+	JustBeforeEach(func() {
+		namespace = testenv.NewNamespaceName(
+			name,
+			rand.New(rand.NewSource(time.Now().UnixNano())),
+		)
+
+		// URL without embedded credentials so the secret is the only auth source.
+		repo := fmt.Sprintf("https://chartmuseum-service.%s.svc.cluster.local:8081", cmd.InfraNamespace)
+		err := testenv.ApplyTemplate(k, testenv.AssetPath("helmop/helmop.yaml"), struct {
+			Name                  string
+			Namespace             string
+			Repo                  string
+			Chart                 string
+			PollingInterval       time.Duration
+			HelmSecretName        string
+			InsecureSkipTLSVerify bool
+			Version               string
+		}{
+			name,
+			namespace,
+			repo,
+			"sleeper-chart",
+			5 * time.Second,
+			secretName,
+			false, // strict TLS — relies on Rancher CA bundle fallback
+			"0.1.0",
+		})
+		Expect(err).ToNot(HaveOccurred())
+	})
+
+	AfterAll(func() {
+		out, err := k.Delete("helmop", name)
+		Expect(err).ToNot(HaveOccurred(), out)
+		out, err = k.Delete("secret", secretName)
+		Expect(err).ToNot(HaveOccurred(), out)
+	})
+
+	It("deploys the chart using the Rancher CA bundle from cattle-system", func() {
+		Eventually(func(g Gomega) {
+			outPods, _ := k.Namespace(namespace).Get("pods")
+			g.Expect(outPods).To(ContainSubstring("sleeper-"))
+		}).Should(Succeed())
+		Eventually(func(g Gomega) {
+			outDeployments, _ := k.Namespace(namespace).Get("deployments")
+			g.Expect(outDeployments).To(ContainSubstring("sleeper"))
+		}).Should(Succeed())
+
+		By("setting the expected version in the helmop Status")
+		Eventually(func() string {
+			out, _ := k.Get("helmop", name, "-o=jsonpath={.status.version}")
+			return out
+		}).Should(Equal("0.1.0"))
+	})
+})
+
 // getExternalHelmAddr retrieves the external URL where our local Helm registry can be reached.
 func getExternalHelmAddr(k kubectl.Command) (string, error) {
 	if v := os.Getenv("external_ip"); v != "" {

integrationtests/helmops/controller/controller_test.go

@@ -4,6 +4,7 @@ import (
 	"crypto/sha256"
 	"crypto/subtle"
 	"crypto/tls"
+	"encoding/pem"
 	"fmt"
 	"io"
 	"log"
@@ -230,6 +231,26 @@ func checkBundleIsAsExpected(g Gomega, bundle fleet.Bundle, helmop fleet.HelmOp,
 	g.Expect(controllerutil.ContainsFinalizer(&bundle, finalize.BundleFinalizer)).To(BeTrue())
 }
 
+// createRancherCASecret creates a secret in cattle-system using the
+// certificate from svr and registers a DeferCleanup to delete it.
+func createRancherCASecret(svr *httptest.Server, secretName, dataKey string) {
+	certPEM := pem.EncodeToMemory(&pem.Block{
+		Type:  "CERTIFICATE",
+		Bytes: svr.TLS.Certificates[0].Certificate[0],
+	})
+	secret := &v1.Secret{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      secretName,
+			Namespace: "cattle-system",
+		},
+		Data: map[string][]byte{dataKey: certPEM},
+	}
+	Expect(k8sClient.Create(ctx, secret)).ToNot(HaveOccurred())
+	DeferCleanup(func() {
+		_ = k8sClient.Delete(ctx, secret)
+	})
+}
+
 func updateHelmOp(helmop fleet.HelmOp) error {
 	backoff := retry.DefaultBackoff
 	backoff.Steps = 10
@@ -1267,5 +1288,73 @@ var _ = Describe("HelmOps controller", func() {
 				}).Should(Succeed())
 			})
 		})
+
+		When("connecting to a https server with a CA bundle from Rancher tls-ca secret", func() {
+			BeforeEach(func() {
+				targets = []fleet.BundleTarget{}
+				helmop = getRandomHelmOpWithTargets("test-rancher-tlsca", targets)
+				helmop.Spec.Helm.Version = ""
+				helmop.Spec.HelmSecretName = ""
+
+				svr := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+					w.WriteHeader(http.StatusOK)
+					fmt.Fprint(w, helmRepoIndex)
+				}))
+				DeferCleanup(svr.Close)
+
+				helmop.Spec.Helm.Repo = svr.URL
+				helmop.Spec.Helm.Chart = "alpine"
+				helmop.Spec.InsecureSkipTLSverify = false
+				doAfterNamespaceCreated = func() {
+					createRancherCASecret(svr, "tls-ca", "cacerts.pem")
+				}
+			})
+
+			It("creates a bundle with the latest version it got from the index", func() {
+				Eventually(func(g Gomega) {
+					bundle := &fleet.Bundle{}
+					ns := types.NamespacedName{Name: helmop.Name, Namespace: helmop.Namespace}
+					err := k8sClient.Get(ctx, ns, bundle)
+					g.Expect(err).ToNot(HaveOccurred())
+					t := []fleet.BundleTarget{{Name: "default", ClusterGroup: "default"}}
+					helmop.Spec.Helm.Version = "0.2.0"
+					checkBundleIsAsExpected(g, *bundle, helmop, t)
+				}).Should(Succeed())
+			})
+		})
+
+		When("connecting to a https server with a CA bundle from Rancher tls-ca-additional secret", func() {
+			BeforeEach(func() {
+				targets = []fleet.BundleTarget{}
+				helmop = getRandomHelmOpWithTargets("test-rancher-tlsca-additional", targets)
+				helmop.Spec.Helm.Version = ""
+				helmop.Spec.HelmSecretName = ""
+
+				svr := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+					w.WriteHeader(http.StatusOK)
+					fmt.Fprint(w, helmRepoIndex)
+				}))
+				DeferCleanup(svr.Close)
+
+				helmop.Spec.Helm.Repo = svr.URL
+				helmop.Spec.Helm.Chart = "alpine"
+				helmop.Spec.InsecureSkipTLSverify = false
+				doAfterNamespaceCreated = func() {
+					createRancherCASecret(svr, "tls-ca-additional", "ca-additional.pem")
+				}
+			})
+
+			It("creates a bundle with the latest version it got from the index", func() {
+				Eventually(func(g Gomega) {
+					bundle := &fleet.Bundle{}
+					ns := types.NamespacedName{Name: helmop.Name, Namespace: helmop.Namespace}
+					err := k8sClient.Get(ctx, ns, bundle)
+					g.Expect(err).ToNot(HaveOccurred())
+					t := []fleet.BundleTarget{{Name: "default", ClusterGroup: "default"}}
+					helmop.Spec.Helm.Version = "0.2.0"
+					checkBundleIsAsExpected(g, *bundle, helmop, t)
+				}).Should(Succeed())
+			})
+		})
 	})
 })

integrationtests/helmops/controller/suite_test.go

@@ -19,6 +19,8 @@ import (
 	"github.com/rancher/fleet/internal/manifest"
 	v1alpha1 "github.com/rancher/fleet/pkg/apis/fleet.cattle.io/v1alpha1"
 
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/client-go/kubernetes"
 	"k8s.io/client-go/kubernetes/scheme"
 	"k8s.io/client-go/rest"
@@ -128,6 +130,22 @@ var _ = BeforeSuite(func() {
 		err = mgr.Start(ctx)
 		Expect(err).ToNot(HaveOccurred(), "failed to run manager")
 	}()
+
+	// Create Rancher-like namespace for CA bundle secrets
+	err = k8sClient.Create(ctx, &corev1.Namespace{
+		ObjectMeta: metav1.ObjectMeta{
+			Name: "cattle-system",
+		},
+	})
+	Expect(err).ToNot(HaveOccurred())
+
+	DeferCleanup(func() {
+		_ = k8sClient.Delete(ctx, &corev1.Namespace{
+			ObjectMeta: metav1.ObjectMeta{
+				Name: "cattle-system",
+			},
+		})
+	})
 })
 
 var _ = AfterSuite(func() {

internal/bundlereader/helm.go

@@ -32,6 +32,13 @@ func GetManifestFromHelmChart(ctx context.Context, c client.Reader, bd *fleet.Bu
 	}
 	auth.InsecureSkipVerify = bd.Spec.HelmChartOptions.InsecureSkipTLSverify
 
+	// Use the Rancher CA bundle that was pre-resolved by the controller and stored in
+	// HelmChartOptions.CABundle. The agent service account cannot read cattle-system
+	// secrets directly, so the controller must pass the bundle through.
+	if len(auth.CABundle) == 0 {
+		auth.CABundle = bd.Spec.HelmChartOptions.CABundle
+	}
+
 	chartURL, err := ChartURL(ctx, *helm, auth, true)
 	if err != nil {
 		return nil, err