[v0.15] - Use Custom CAs when pulling OCI artifact (#5128)

_typos.toml

@@ -18,4 +18,5 @@ extend-exclude = [
 extend-ignore-re = [
     "-{5}BEGIN RSA PRIVATE KEY-{5}(?:$|[^-]{63,}-{5}END)",
     "-{5}BEGIN PRIVATE KEY-{5}(?:$|[^-]{63,}-{5}END)",
+    "-{5}BEGIN CERTIFICATE-{5}(?:$|[^-]{63,}-{5}END CERTIFICATE)"
 ]

internal/cmd/cli/apply/apply.go

@@ -575,6 +575,7 @@ func shouldStoreInOCIRegistry(ctx context.Context, c client.Reader, ociSecretKey
 	ociOpts.AgentPassword = opts.AgentPassword
 	ociOpts.BasicHTTP = opts.BasicHTTP
 	ociOpts.InsecureSkipTLS = opts.InsecureSkipTLS
+	ociOpts.CABundle = opts.CABundle
 
 	return true, nil
 }
@@ -756,19 +757,27 @@ func newOCISecret(manifestID string, bundle *fleet.Bundle, opts ocistorage.OCIOp
 				},
 			},
 		},
-		Data: map[string][]byte{
-			ocistorage.OCISecretReference:       []byte(opts.Reference),
-			ocistorage.OCISecretUsername:        []byte(opts.Username),
-			ocistorage.OCISecretPassword:        []byte(opts.Password),
-			ocistorage.OCISecretAgentUsername:   []byte(opts.AgentUsername),
-			ocistorage.OCISecretAgentPassword:   []byte(opts.AgentPassword),
-			ocistorage.OCISecretBasicHTTP:       []byte(strconv.FormatBool(opts.BasicHTTP)),
-			ocistorage.OCISecretInsecureSkipTLS: []byte(strconv.FormatBool(opts.InsecureSkipTLS)),
-		},
+		Data: newOCISecretData(opts),
 		Type: fleet.SecretTypeOCIStorage,
 	}
 }
 
+func newOCISecretData(opts ocistorage.OCIOpts) map[string][]byte {
+	data := map[string][]byte{
+		ocistorage.OCISecretReference:       []byte(opts.Reference),
+		ocistorage.OCISecretUsername:        []byte(opts.Username),
+		ocistorage.OCISecretPassword:        []byte(opts.Password),
+		ocistorage.OCISecretAgentUsername:   []byte(opts.AgentUsername),
+		ocistorage.OCISecretAgentPassword:   []byte(opts.AgentPassword),
+		ocistorage.OCISecretBasicHTTP:       []byte(strconv.FormatBool(opts.BasicHTTP)),
+		ocistorage.OCISecretInsecureSkipTLS: []byte(strconv.FormatBool(opts.InsecureSkipTLS)),
+	}
+	if len(opts.CABundle) > 0 {
+		data[ocistorage.OCISecretCABundle] = opts.CABundle
+	}
+	return data
+}
+
 func newValuesSecret(bundle *fleet.Bundle, data map[string][]byte) *corev1.Secret {
 	return &corev1.Secret{
 		Type: fleet.SecretTypeBundleValues,

internal/ocistorage/ociwrapper.go

@@ -4,6 +4,7 @@ import (
 	"bytes"
 	"context"
 	"crypto/tls"
+	"crypto/x509"
 	"encoding/json"
 	"fmt"
 	"io"
@@ -15,13 +16,15 @@ import (
 
 	"github.com/opencontainers/go-digest"
 	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
-	"github.com/rancher/fleet/internal/manifest"
+	"github.com/sirupsen/logrus"
 	"oras.land/oras-go/v2"
 	"oras.land/oras-go/v2/content"
 	"oras.land/oras-go/v2/content/memory"
 	"oras.land/oras-go/v2/registry/remote"
 	"oras.land/oras-go/v2/registry/remote/auth"
 	"oras.land/oras-go/v2/registry/remote/retry"
+
+	"github.com/rancher/fleet/internal/manifest"
 )
 
 const (
@@ -39,6 +42,7 @@ type OCIOpts struct {
 	AgentPassword   string
 	BasicHTTP       bool
 	InsecureSkipTLS bool
+	CABundle        []byte
 }
 
 type OrasOps interface {
@@ -71,20 +75,61 @@ func NewOCIWrapper() *OCIWrapper {
 	}
 }
 
-func getHTTPClient(insecureSkipTLS bool) *http.Client {
-	if !insecureSkipTLS {
+func getHTTPClient(insecureSkipTLS bool, caBundle []byte) *http.Client {
+	// Defensive copy to avoid mutations
+	caBundle = append([]byte(nil), caBundle...)
+
+	// Merge proxy CA bundle if present
+	if proxyCAPEM, ok := os.LookupEnv("PROXY_CA_BUNDLE"); ok && proxyCAPEM != "" {
+		proxyBytes := []byte(proxyCAPEM)
+		tmpPool := x509.NewCertPool()
+		if !tmpPool.AppendCertsFromPEM(proxyBytes) {
+			logrus.Warnf("%s is set but contains no valid PEM certificates; ignoring proxy CA bundle", "PROXY_CA_BUNDLE")
+		} else {
+			if len(caBundle) > 0 && caBundle[len(caBundle)-1] != '\n' {
+				caBundle = append(caBundle, '\n')
+			}
+			caBundle = append(caBundle, proxyBytes...)
+		}
+	}
+
+	// If no custom TLS config needed, use default ORAS client
+	if !insecureSkipTLS && len(caBundle) == 0 {
 		return retry.DefaultClient
 	}
-	return &http.Client{
-		Transport: &http.Transport{
-			TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, // #nosec G402
-		},
+
+	// Clone the default transport to preserve proxy, timeout, and
+	// connection-pooling settings.
+	baseTransport, ok := http.DefaultTransport.(*http.Transport)
+	if !ok {
+		baseTransport = &http.Transport{
+			Proxy: http.ProxyFromEnvironment,
+		}
+	}
+	transport := baseTransport.Clone()
+	transport.TLSClientConfig = &tls.Config{
+		InsecureSkipVerify: insecureSkipTLS, // #nosec G402
+	}
+
+	// Merge custom CA bundle with system cert pool
+	if len(caBundle) > 0 {
+		pool, err := x509.SystemCertPool()
+		if err != nil {
+			pool = x509.NewCertPool()
+		}
+		if !pool.AppendCertsFromPEM(caBundle) {
+			logrus.Warnf("CA bundle contains no valid PEM certificates; proceeding with system cert pool only")
+		}
+		transport.TLSClientConfig.RootCAs = pool
+		transport.TLSClientConfig.MinVersion = tls.VersionTLS12
 	}
+
+	return &http.Client{Transport: retry.NewTransport(transport)}
 }
 
 func getAuthClient(opts OCIOpts) *auth.Client {
 	client := &auth.Client{
-		Client: getHTTPClient(opts.InsecureSkipTLS),
+		Client: getHTTPClient(opts.InsecureSkipTLS, opts.CABundle),
 		Cache:  auth.NewCache(),
 	}
 	if opts.Username != "" {

internal/ocistorage/ociwrapper_test.go

@@ -4,6 +4,7 @@ import (
 	"bytes"
 	"context"
 	"crypto/tls"
+	"crypto/x509"
 	"fmt"
 	"net/http"
 	"os"
@@ -136,17 +137,181 @@ var _ = Describe("OCIUtils tests", func() {
 		Expect(repo.PlainHTTP).To(BeFalse())
 	})
 	It("return the expected tls client", func() {
-		client := getHTTPClient(true)
-		expected := &http.Client{
-			Transport: &http.Transport{
-				TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
-			},
-		}
-		Expect(client).To(Equal(expected))
+		client := getHTTPClient(true, nil)
+
+		// Custom path wraps transport in retry.Transport
+		retryTransport, ok := client.Transport.(*retry.Transport)
+		Expect(ok).To(BeTrue())
+		innerTransport, ok := retryTransport.Base.(*http.Transport)
+		Expect(ok).To(BeTrue())
+		Expect(innerTransport.TLSClientConfig).ToNot(BeNil())
+		Expect(innerTransport.TLSClientConfig.InsecureSkipVerify).To(BeTrue())
+		Expect(innerTransport.Proxy).ToNot(BeNil())
+
+		client = getHTTPClient(false, nil)
+		Expect(client).To(Equal(retry.DefaultClient))
+	})
+	It("should use custom CA bundle when provided", func() {
+		// Use a valid test certificate from the codebase pattern (same as netutils_test.go)
+		caBundle := []byte(`-----BEGIN CERTIFICATE-----
+MIICGTCCAZ+gAwIBAgIQCeCTZaz32ci5PhwLBCou8zAKBggqhkjOPQQDAzBOMQsw
+CQYDVQQGEwJVUzEXMBUGA1UEChMORGlnaUNlcnQsIEluYy4xJjAkBgNVBAMTHURp
+Z2lDZXJ0IFRMUyBFQ0MgUDM4NCBSb290IEc1MB4XDTIxMDExNTAwMDAwMFoXDTQ2
+MDExNDIzNTk1OVowTjELMAkGA1UEBhMCVVMxFzAVBgNVBAoTDkRpZ2lDZXJ0LCBJ
+bmMuMSYwJAYDVQQDEx1EaWdpQ2VydCBUTFMgRUNDIFAzODQgUm9vdCBHNTB2MBAG
+ByqGSM49AgEGBSuBBAAiA2IABMFEoc8Rl1Ca3iOCNQfN0MsYndLxf3c1TzvdlHJS
+7cI7+Oz6e2tYIOyZrsn8aLN1udsJ7MgT9U7GCh1mMEy7H0cKPGEQQil8pQgO4CLp
+0zVozptjn4S1mU1YoI71VOeVyaNCMEAwHQYDVR0OBBYEFMFRRVBZqz7nLFr6ICIS
+B4CIfBFqMA4GA1UdDwEB/wQEAwIBhjAPBgNVHRMBAf8EBTADAQH/MAoGCCqGSM49
+BAMDA2gAMGUCMQCJao1H5+z8blUD2WdsJk6Dxv3J+ysTvLd6jLRl0mlpYxNjOyZQ
+LgGheQaRnUi/wr4CMEfDFXuxoJGZSZOoPHzoRgaLLPIxAJSdYsiJvRmEFOml+wG4
+DXZDjC5Ty3zfDBeWUA==
+-----END CERTIFICATE-----`)
+		client := getHTTPClient(false, caBundle)
+
+		// Verify transport is configured with custom TLS config
+		retryT, ok := client.Transport.(*retry.Transport)
+		Expect(ok).To(BeTrue())
+		transport, ok := retryT.Base.(*http.Transport)
+		Expect(ok).To(BeTrue())
+		Expect(transport.TLSClientConfig).ToNot(BeNil())
+		Expect(transport.TLSClientConfig.RootCAs).ToNot(BeNil())
+		Expect(transport.TLSClientConfig.MinVersion).To(Equal(uint16(tls.VersionTLS12)))
+		Expect(transport.TLSClientConfig.InsecureSkipVerify).To(BeFalse())
+	})
+	It("should merge proxy CA bundle from environment variable", func() {
+		// Use a valid certificate for proxy CA (same as custom test)
+		proxyCA := `-----BEGIN CERTIFICATE-----
+MIICGTCCAZ+gAwIBAgIQCeCTZaz32ci5PhwLBCou8zAKBggqhkjOPQQDAzBOMQsw
+CQYDVQQGEwJVUzEXMBUGA1UEChMORGlnaUNlcnQsIEluYy4xJjAkBgNVBAMTHURp
+Z2lDZXJ0IFRMUyBFQ0MgUDM4NCBSb290IEc1MB4XDTIxMDExNTAwMDAwMFoXDTQ2
+MDExNDIzNTk1OVowTjELMAkGA1UEBhMCVVMxFzAVBgNVBAoTDkRpZ2lDZXJ0LCBJ
+bmMuMSYwJAYDVQQDEx1EaWdpQ2VydCBUTFMgRUNDIFAzODQgUm9vdCBHNTB2MBAG
+ByqGSM49AgEGBSuBBAAiA2IABMFEoc8Rl1Ca3iOCNQfN0MsYndLxf3c1TzvdlHJS
+7cI7+Oz6e2tYIOyZrsn8aLN1udsJ7MgT9U7GCh1mMEy7H0cKPGEQQil8pQgO4CLp
+0zVozptjn4S1mU1YoI71VOeVyaNCMEAwHQYDVR0OBBYEFMFRRVBZqz7nLFr6ICIS
+B4CIfBFqMA4GA1UdDwEB/wQEAwIBhjAPBgNVHRMBAf8EBTADAQH/MAoGCCqGSM49
+BAMDA2gAMGUCMQCJao1H5+z8blUD2WdsJk6Dxv3J+ysTvLd6jLRl0mlpYxNjOyZQ
+LgGheQaRnUi/wr4CMEfDFXuxoJGZSZOoPHzoRgaLLPIxAJSdYsiJvRmEFOml+wG4
+DXZDjC5Ty3zfDBeWUA==
+-----END CERTIFICATE-----`
+		os.Setenv("PROXY_CA_BUNDLE", proxyCA)
+		defer os.Unsetenv("PROXY_CA_BUNDLE")
+
+		// Use the same valid certificate for custom CA (simpler for testing)
+		customCA := []byte(proxyCA)
+		client := getHTTPClient(false, customCA)
+
+		// Should have merged both CAs
+		retryT, ok := client.Transport.(*retry.Transport)
+		Expect(ok).To(BeTrue())
+		transport, ok := retryT.Base.(*http.Transport)
+		Expect(ok).To(BeTrue())
+		Expect(transport.TLSClientConfig).ToNot(BeNil())
+		Expect(transport.TLSClientConfig.RootCAs).ToNot(BeNil())
+		Expect(transport.TLSClientConfig.MinVersion).To(Equal(uint16(tls.VersionTLS12)))
+
+		// Verify both CAs are valid and would have been loaded
+		testPool := x509.NewCertPool()
+		ok = testPool.AppendCertsFromPEM(customCA)
+		Expect(ok).To(BeTrue(), "custom CA bundle should be valid PEM")
+		ok = testPool.AppendCertsFromPEM([]byte(proxyCA))
+		Expect(ok).To(BeTrue(), "proxy CA bundle should be valid PEM")
+	})
+	It("should handle invalid CA bundle gracefully", func() {
+		invalidCA := []byte("not a valid PEM certificate")
+		client := getHTTPClient(false, invalidCA)
+
+		// Should still create a client with TLS config (warning logged)
+		retryT, ok := client.Transport.(*retry.Transport)
+		Expect(ok).To(BeTrue())
+		transport, ok := retryT.Base.(*http.Transport)
+		Expect(ok).To(BeTrue())
+		Expect(transport.TLSClientConfig).ToNot(BeNil())
+		Expect(transport.TLSClientConfig.RootCAs).ToNot(BeNil())
+	})
+	It("should handle invalid proxy CA bundle gracefully", func() {
+		os.Setenv("PROXY_CA_BUNDLE", "invalid proxy PEM")
+		defer os.Unsetenv("PROXY_CA_BUNDLE")
+
+		client := getHTTPClient(false, nil)
 
-		client = getHTTPClient(false)
+		// Should return default client when no valid CA and not insecure
 		Expect(client).To(Equal(retry.DefaultClient))
 	})
+	It("should combine insecureSkipTLS with CA bundle", func() {
+		caBundle := []byte(`-----BEGIN CERTIFICATE-----
+MIICGTCCAZ+gAwIBAgIQCeCTZaz32ci5PhwLBCou8zAKBggqhkjOPQQDAzBOMQsw
+CQYDVQQGEwJVUzEXMBUGA1UEChMORGlnaUNlcnQsIEluYy4xJjAkBgNVBAMTHURp
+Z2lDZXJ0IFRMUyBFQ0MgUDM4NCBSb290IEc1MB4XDTIxMDExNTAwMDAwMFoXDTQ2
+MDExNDIzNTk1OVowTjELMAkGA1UEBhMCVVMxFzAVBgNVBAoTDkRpZ2lDZXJ0LCBJ
+bmMuMSYwJAYDVQQDEx1EaWdpQ2VydCBUTFMgRUNDIFAzODQgUm9vdCBHNTB2MBAG
+ByqGSM49AgEGBSuBBAAiA2IABMFEoc8Rl1Ca3iOCNQfN0MsYndLxf3c1TzvdlHJS
+7cI7+Oz6e2tYIOyZrsn8aLN1udsJ7MgT9U7GCh1mMEy7H0cKPGEQQil8pQgO4CLp
+0zVozptjn4S1mU1YoI71VOeVyaNCMEAwHQYDVR0OBBYEFMFRRVBZqz7nLFr6ICIS
+B4CIfBFqMA4GA1UdDwEB/wQEAwIBhjAPBgNVHRMBAf8EBTADAQH/MAoGCCqGSM49
+BAMDA2gAMGUCMQCJao1H5+z8blUD2WdsJk6Dxv3J+ysTvLd6jLRl0mlpYxNjOyZQ
+LgGheQaRnUi/wr4CMEfDFXuxoJGZSZOoPHzoRgaLLPIxAJSdYsiJvRmEFOml+wG4
+DXZDjC5Ty3zfDBeWUA==
+-----END CERTIFICATE-----`)
+		client := getHTTPClient(true, caBundle)
+
+		retryT, ok := client.Transport.(*retry.Transport)
+		Expect(ok).To(BeTrue())
+		transport, ok := retryT.Base.(*http.Transport)
+		Expect(ok).To(BeTrue())
+		Expect(transport.TLSClientConfig).ToNot(BeNil())
+		Expect(transport.TLSClientConfig.InsecureSkipVerify).To(BeTrue())
+		Expect(transport.TLSClientConfig.RootCAs).ToNot(BeNil())
+		Expect(transport.TLSClientConfig.MinVersion).To(Equal(uint16(tls.VersionTLS12)))
+
+		// Verify the CA bundle is valid PEM (even though InsecureSkipVerify is set)
+		testPool := x509.NewCertPool()
+		ok = testPool.AppendCertsFromPEM(caBundle)
+		Expect(ok).To(BeTrue(), "CA bundle should be valid PEM")
+	})
+	It("should not mutate the original CA bundle slice", func() {
+		// validPEM is a real certificate so that AppendCertsFromPEM succeeds and
+		// getHTTPClient actually reaches the append(caBundle, proxyBytes...) path.
+		// An invalid PEM would make the function return early without appending,
+		// meaning the defensive copy would never be exercised.
+		validPEM := []byte(`-----BEGIN CERTIFICATE-----
+MIICGTCCAZ+gAwIBAgIQCeCTZaz32ci5PhwLBCou8zAKBggqhkjOPQQDAzBOMQsw
+CQYDVQQGEwJVUzEXMBUGA1UEChMORGlnaUNlcnQsIEluYy4xJjAkBgNVBAMTHURp
+Z2lDZXJ0IFRMUyBFQ0MgUDM4NCBSb290IEc1MB4XDTIxMDExNTAwMDAwMFoXDTQ2
+MDExNDIzNTk1OVowTjELMAkGA1UEBhMCVVMxFzAVBgNVBAoTDkRpZ2lDZXJ0LCBJ
+bmMuMSYwJAYDVQQDEx1EaWdpQ2VydCBUTFMgRUNDIFAzODQgUm9vdCBHNTB2MBAG
+ByqGSM49AgEGBSuBBAAiA2IABMFEoc8Rl1Ca3iOCNQfN0MsYndLxf3c1TzvdlHJS
+7cI7+Oz6e2tYIOyZrsn8aLN1udsJ7MgT9U7GCh1mMEy7H0cKPGEQQil8pQgO4CLp
+0zVozptjn4S1mU1YoI71VOeVyaNCMEAwHQYDVR0OBBYEFMFRRVBZqz7nLFr6ICIS
+B4CIfBFqMA4GA1UdDwEB/wQEAwIBhjAPBgNVHRMBAf8EBTADAQH/MAoGCCqGSM49
+BAMDA2gAMGUCMQCJao1H5+z8blUD2WdsJk6Dxv3J+ysTvLd6jLRl0mlpYxNjOyZQ
+LgGheQaRnUi/wr4CMEfDFXuxoJGZSZOoPHzoRgaLLPIxAJSdYsiJvRmEFOml+wG4
+DXZDjC5Ty3zfDBeWUA==
+-----END CERTIFICATE-----`)
+
+		os.Setenv("PROXY_CA_BUNDLE", string(validPEM))
+		defer os.Unsetenv("PROXY_CA_BUNDLE")
+
+		// Allocate with excess capacity so that append can write into the backing
+		// array without reallocating. With len == cap (e.g. []byte("literal")),
+		// append always reallocates and the defensive copy makes no observable
+		// difference — the test would pass even without it.
+		originalCA := make([]byte, len(validPEM), len(validPEM)+512)
+		copy(originalCA, validPEM)
+
+		// backing covers the full allocation so we can detect writes past len(originalCA).
+		backing := originalCA[:cap(originalCA)]
+
+		_ = getHTTPClient(false, originalCA)
+
+		// The slice header and visible content must be unchanged.
+		Expect(originalCA).To(Equal(validPEM))
+		// Without the defensive copy, getHTTPClient would append '\n'+proxyBytes
+		// directly into the excess capacity of the caller's backing array.
+		Expect(backing[len(validPEM):]).To(Equal(make([]byte, 512)),
+			"backing array beyond len(originalCA) must not be written to")
+	})
 	It("return the expected credentials", func() {
 		opts := OCIOpts{
 			Reference: "test.com",

internal/ocistorage/secret.go

@@ -21,6 +21,7 @@ const (
 	OCISecretBasicHTTP       = "basicHTTP"
 	OCISecretInsecureSkipTLS = "insecureSkipTLS"
 	OCISecretInsecure        = "insecure" // legacy alias
+	OCISecretCABundle        = "cacerts"
 )
 
 // ReadOptsFromSecret reads the secret identified by the given NamespacedName and
@@ -84,6 +85,9 @@ func ReadOptsFromSecret(ctx context.Context, c client.Reader, ns client.ObjectKe
 		return OCIOpts{}, err
 	}
 
+	// Read optional CA bundle
+	opts.CABundle = secret.Data[OCISecretCABundle]
+
 	return opts, nil
 }