Skip to content

Commit 4992a9b

Browse files
thardeckthardeck
committed
Fall back to Rancher CA bundles for HelmOps (#4724)
* Fall back to Rancher CA bundles for HelmOps Resolve the CA bundle in the HelmOps controller and store it in BundleHelmOptions.CABundle so the agent can use it without needing access to cattle-system secrets. The agent service account only has access to its own namespace. Also restart helmops pods in dev/update-controller-k3d so that redeployments pick up the new controller binary. * Remove obsolete OCI no-TLS negative test The test relied on the OCI registry being untrusted when InsecureSkipTLSVerify is false and no CA bundle is provided. Since the Rancher CA bundle fallback now supplies the fleet CI root CA (stored in cattle-system/tls-ca-additional), the Zot OCI registry is trusted automatically and the chart deploys successfully.
1 parent a72317d commit 4992a9b

13 files changed

Lines changed: 388 additions & 31 deletions

File tree

‎charts/fleet-crd/templates/crds.yaml‎

Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -194,6 +194,20 @@ spec:
194194
195195
should be downloaded from a helm chart'
196196
properties:
197+
helmOpCABundle:
198+
description: 'CABundle is a PEM encoded CA bundle used to validate
199+
TLS connections to
200+
201+
the Helm registry. It is resolved by the controller (which
202+
has access to
203+
204+
Rancher''s cattle-system CA secrets, if any) and stored here
205+
so the agent can use
206+
207+
it without requiring access to those secrets.'
208+
format: byte
209+
nullable: true
210+
type: string
197211
helmOpInsecureSkipTLSVerify:
198212
description: InsecureSkipTLSverify will use insecure HTTPS to
199213
clone the helm app resource.
@@ -2079,6 +2093,20 @@ spec:
20792093
not a git repository.'
20802094
nullable: true
20812095
properties:
2096+
helmOpCABundle:
2097+
description: 'CABundle is a PEM encoded CA bundle used to validate
2098+
TLS connections to
2099+
2100+
the Helm registry. It is resolved by the controller (which
2101+
has access to
2102+
2103+
Rancher''s cattle-system CA secrets, if any) and stored here
2104+
so the agent can use
2105+
2106+
it without requiring access to those secrets.'
2107+
format: byte
2108+
nullable: true
2109+
type: string
20822110
helmOpInsecureSkipTLSVerify:
20832111
description: InsecureSkipTLSverify will use insecure HTTPS to
20842112
clone the helm app resource.
@@ -8282,6 +8310,20 @@ spec:
82828310
not a git repository.'
82838311
nullable: true
82848312
properties:
8313+
helmOpCABundle:
8314+
description: 'CABundle is a PEM encoded CA bundle used to validate
8315+
TLS connections to
8316+
8317+
the Helm registry. It is resolved by the controller (which
8318+
has access to
8319+
8320+
Rancher''s cattle-system CA secrets, if any) and stored here
8321+
so the agent can use
8322+
8323+
it without requiring access to those secrets.'
8324+
format: byte
8325+
nullable: true
8326+
type: string
82858327
helmOpInsecureSkipTLSVerify:
82868328
description: InsecureSkipTLSverify will use insecure HTTPS to
82878329
clone the helm app resource.

‎dev/update-controller-k3d‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,3 +20,4 @@ fleet_ctx=$(kubectl config current-context)
2020
k3d image import rancher/fleet:dev -m direct -c "${fleet_ctx#k3d-}"
2121
kubectl delete pod -l app=fleet-controller -n cattle-fleet-system
2222
kubectl delete pod -l app=gitjob -n cattle-fleet-system
23+
kubectl delete pod -l app=helmops -n cattle-fleet-system

‎e2e/single-cluster/helmop_test.go‎

Lines changed: 80 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -371,21 +371,6 @@ var _ = Describe("HelmOp resource with polling of OCI registry", Label("infra-se
371371
})
372372
})
373373

374-
Context("containing a valid helmop description pointing to an oci registry and not TLS", func() {
375-
BeforeEach(func() {
376-
namespace = "helmop-ns2"
377-
name = "basic-oci-no-tls"
378-
insecure = false
379-
380-
repo = fmt.Sprintf("%s/sleeper-chart", ociRef)
381-
})
382-
It("does not deploy the chart because of TLS", func() {
383-
Consistently(func() string {
384-
out, _ := k.Namespace(namespace).Get("pods")
385-
return out
386-
}, 5*time.Second, time.Second).ShouldNot(ContainSubstring("sleeper-"))
387-
})
388-
})
389374
})
390375

391376
When("applying a helmop resource which cannot be deployed", func() {
@@ -520,6 +505,86 @@ var _ = Describe("HelmOp resource tests with tarball source", Label("infra-setup
520505
})
521506
})
522507

508+
var _ = Describe("HelmOp resource falls back to Rancher CA bundle", Label("infra-setup", "helm-registry"), Ordered, func() {
509+
// This test mirrors the GitOps E2E test "should succeed when not configuring any CA"
510+
// in go_getter_custom_ca_test.go. The dev/create-secrets script places the root CA
511+
// into cattle-system/tls-ca-additional. ChartMuseum is served with a cert signed by
512+
// that root CA. A HelmOp with a credentials-only secret (no cacerts) and
513+
// InsecureSkipTLSVerify=false must therefore succeed via the Rancher CA fallback.
514+
const (
515+
name = "rancher-ca-fallback"
516+
secretName = "helmop-rancher-ca-creds"
517+
)
518+
519+
var (
520+
namespace string
521+
k kubectl.Command
522+
)
523+
524+
BeforeAll(func() {
525+
k = env.Kubectl.Namespace(env.Namespace)
526+
out, err := k.Create(
527+
"secret", "generic", secretName,
528+
"--from-literal=username="+os.Getenv("CI_OCI_USERNAME"),
529+
"--from-literal=password="+os.Getenv("CI_OCI_PASSWORD"),
530+
// no cacerts — TLS trust must come from cattle-system/tls-ca-additional
531+
)
532+
if strings.Contains(out, "already exists") {
533+
err = nil
534+
}
535+
Expect(err).ToNot(HaveOccurred(), out)
536+
})
537+
538+
JustBeforeEach(func() {
539+
namespace = testenv.NewNamespaceName(
540+
name,
541+
rand.New(rand.NewSource(time.Now().UnixNano())),
542+
)
543+
544+
// URL without embedded credentials so the secret is the only auth source.
545+
repo := fmt.Sprintf("https://chartmuseum-service.%s.svc.cluster.local:8081", cmd.InfraNamespace)
546+
err := testenv.ApplyTemplate(k, testenv.AssetPath("helmop/helmop.yaml"), struct {
547+
Name string
548+
Namespace string
549+
Repo string
550+
Chart string
551+
PollingInterval time.Duration
552+
HelmSecretName string
553+
InsecureSkipTLSVerify bool
554+
Version string
555+
}{
556+
name,
557+
namespace,
558+
repo,
559+
"sleeper-chart",
560+
5 * time.Second,
561+
secretName,
562+
false, // strict TLS — relies on Rancher CA bundle fallback
563+
"0.1.0",
564+
})
565+
Expect(err).ToNot(HaveOccurred())
566+
})
567+
568+
AfterAll(func() {
569+
out, err := k.Delete("helmop", name)
570+
Expect(err).ToNot(HaveOccurred(), out)
571+
out, err = k.Delete("secret", secretName)
572+
Expect(err).ToNot(HaveOccurred(), out)
573+
})
574+
575+
It("deploys the chart using the Rancher CA bundle from cattle-system", func() {
576+
Eventually(func(g Gomega) {
577+
outPods, _ := k.Namespace(namespace).Get("pods")
578+
g.Expect(outPods).To(ContainSubstring("sleeper-"))
579+
}).Should(Succeed())
580+
Eventually(func(g Gomega) {
581+
outDeployments, _ := k.Namespace(namespace).Get("deployments")
582+
g.Expect(outDeployments).To(ContainSubstring("sleeper"))
583+
}).Should(Succeed())
584+
585+
})
586+
})
587+
523588
// getExternalHelmAddr retrieves the external URL where our local Helm registry can be reached.
524589
func getExternalHelmAddr(k kubectl.Command) (string, error) {
525590
if v := os.Getenv("external_ip"); v != "" {

‎integrationtests/helmops/controller/controller_test.go‎

Lines changed: 89 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@ import (
44
"crypto/sha256"
55
"crypto/subtle"
66
"crypto/tls"
7+
"encoding/pem"
78
"fmt"
89
"io"
910
"log"
@@ -230,6 +231,26 @@ func checkBundleIsAsExpected(g Gomega, bundle fleet.Bundle, helmop fleet.HelmOp,
230231
g.Expect(controllerutil.ContainsFinalizer(&bundle, finalize.BundleFinalizer)).To(BeTrue())
231232
}
232233

234+
// createRancherCASecret creates a secret in cattle-system using the
235+
// certificate from svr and registers a DeferCleanup to delete it.
236+
func createRancherCASecret(svr *httptest.Server, secretName, dataKey string) {
237+
certPEM := pem.EncodeToMemory(&pem.Block{
238+
Type: "CERTIFICATE",
239+
Bytes: svr.TLS.Certificates[0].Certificate[0],
240+
})
241+
secret := &v1.Secret{
242+
ObjectMeta: metav1.ObjectMeta{
243+
Name: secretName,
244+
Namespace: "cattle-system",
245+
},
246+
Data: map[string][]byte{dataKey: certPEM},
247+
}
248+
Expect(k8sClient.Create(ctx, secret)).ToNot(HaveOccurred())
249+
DeferCleanup(func() {
250+
_ = k8sClient.Delete(ctx, secret)
251+
})
252+
}
253+
233254
func updateHelmOp(helmop fleet.HelmOp) error {
234255
backoff := retry.DefaultBackoff
235256
backoff.Steps = 10
@@ -1267,5 +1288,73 @@ var _ = Describe("HelmOps controller", func() {
12671288
}).Should(Succeed())
12681289
})
12691290
})
1291+
1292+
When("connecting to a https server with a CA bundle from Rancher tls-ca secret", func() {
1293+
BeforeEach(func() {
1294+
targets = []fleet.BundleTarget{}
1295+
helmop = getRandomHelmOpWithTargets("test-rancher-tlsca", targets)
1296+
helmop.Spec.Helm.Version = ""
1297+
helmop.Spec.HelmSecretName = ""
1298+
1299+
svr := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
1300+
w.WriteHeader(http.StatusOK)
1301+
fmt.Fprint(w, helmRepoIndex)
1302+
}))
1303+
DeferCleanup(svr.Close)
1304+
1305+
helmop.Spec.Helm.Repo = svr.URL
1306+
helmop.Spec.Helm.Chart = "alpine"
1307+
helmop.Spec.InsecureSkipTLSverify = false
1308+
doAfterNamespaceCreated = func() {
1309+
createRancherCASecret(svr, "tls-ca", "cacerts.pem")
1310+
}
1311+
})
1312+
1313+
It("creates a bundle with the latest version it got from the index", func() {
1314+
Eventually(func(g Gomega) {
1315+
bundle := &fleet.Bundle{}
1316+
ns := types.NamespacedName{Name: helmop.Name, Namespace: helmop.Namespace}
1317+
err := k8sClient.Get(ctx, ns, bundle)
1318+
g.Expect(err).ToNot(HaveOccurred())
1319+
t := []fleet.BundleTarget{{Name: "default", ClusterGroup: "default"}}
1320+
helmop.Spec.Helm.Version = "0.2.0"
1321+
checkBundleIsAsExpected(g, *bundle, helmop, t)
1322+
}).Should(Succeed())
1323+
})
1324+
})
1325+
1326+
When("connecting to a https server with a CA bundle from Rancher tls-ca-additional secret", func() {
1327+
BeforeEach(func() {
1328+
targets = []fleet.BundleTarget{}
1329+
helmop = getRandomHelmOpWithTargets("test-rancher-tlsca-additional", targets)
1330+
helmop.Spec.Helm.Version = ""
1331+
helmop.Spec.HelmSecretName = ""
1332+
1333+
svr := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
1334+
w.WriteHeader(http.StatusOK)
1335+
fmt.Fprint(w, helmRepoIndex)
1336+
}))
1337+
DeferCleanup(svr.Close)
1338+
1339+
helmop.Spec.Helm.Repo = svr.URL
1340+
helmop.Spec.Helm.Chart = "alpine"
1341+
helmop.Spec.InsecureSkipTLSverify = false
1342+
doAfterNamespaceCreated = func() {
1343+
createRancherCASecret(svr, "tls-ca-additional", "ca-additional.pem")
1344+
}
1345+
})
1346+
1347+
It("creates a bundle with the latest version it got from the index", func() {
1348+
Eventually(func(g Gomega) {
1349+
bundle := &fleet.Bundle{}
1350+
ns := types.NamespacedName{Name: helmop.Name, Namespace: helmop.Namespace}
1351+
err := k8sClient.Get(ctx, ns, bundle)
1352+
g.Expect(err).ToNot(HaveOccurred())
1353+
t := []fleet.BundleTarget{{Name: "default", ClusterGroup: "default"}}
1354+
helmop.Spec.Helm.Version = "0.2.0"
1355+
checkBundleIsAsExpected(g, *bundle, helmop, t)
1356+
}).Should(Succeed())
1357+
})
1358+
})
12701359
})
12711360
})

‎integrationtests/helmops/controller/suite_test.go‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,8 @@ import (
1919
"github.com/rancher/fleet/internal/manifest"
2020
v1alpha1 "github.com/rancher/fleet/pkg/apis/fleet.cattle.io/v1alpha1"
2121

22+
corev1 "k8s.io/api/core/v1"
23+
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
2224
"k8s.io/client-go/kubernetes"
2325
"k8s.io/client-go/kubernetes/scheme"
2426
"k8s.io/client-go/rest"
@@ -129,6 +131,22 @@ var _ = BeforeSuite(func() {
129131
err = mgr.Start(ctx)
130132
Expect(err).ToNot(HaveOccurred(), "failed to run manager")
131133
}()
134+
135+
// Create Rancher-like namespace for CA bundle secrets
136+
err = k8sClient.Create(ctx, &corev1.Namespace{
137+
ObjectMeta: metav1.ObjectMeta{
138+
Name: "cattle-system",
139+
},
140+
})
141+
Expect(err).ToNot(HaveOccurred())
142+
143+
DeferCleanup(func() {
144+
_ = k8sClient.Delete(ctx, &corev1.Namespace{
145+
ObjectMeta: metav1.ObjectMeta{
146+
Name: "cattle-system",
147+
},
148+
})
149+
})
132150
})
133151

134152
var _ = AfterSuite(func() {

‎internal/bundlereader/helm.go‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,13 @@ func GetManifestFromHelmChart(ctx context.Context, c client.Reader, bd *fleet.Bu
3232
}
3333
auth.InsecureSkipVerify = bd.Spec.HelmChartOptions.InsecureSkipTLSverify
3434

35+
// Use the Rancher CA bundle that was pre-resolved by the controller and stored in
36+
// HelmChartOptions.CABundle. The agent service account cannot read cattle-system
37+
// secrets directly, so the controller must pass the CA bundle through.
38+
if len(auth.CABundle) == 0 {
39+
auth.CABundle = bd.Spec.HelmChartOptions.CABundle
40+
}
41+
3542
chartURL, err := ChartURL(ctx, *helm, auth, true)
3643
if err != nil {
3744
return nil, err

0 commit comments

Comments
 (0)