Store CertificateResource as one unit

certmagic.go

@@ -499,3 +499,27 @@ var (
 
 // Maximum size for the stack trace when recovering from panics.
 const stackTraceBufferSize = 1024 * 128
+
+const (
+	// Storage mode controls the format in which certificates are stored in `Storage`.
+	//
+	// Formats:
+	// - legacy: Store cert, privkey and meta as three separate storage items (.cert, .key, .json).
+	// - bundle: Store cert, privkey and meta as a single, bundled storage item (.bundle).
+	//
+	// Modes:
+	// - legacy:     Store and load certificates in legacy format.
+	// - transition: Store in legacy and bundle format, load as bundle with fallback to legacy format.
+	// - bundle:     Store and load certificates in bundle format.
+	//
+	// In the transition mode, failures around reads and writes of the bundle are soft.
+	// They should only log errors and try to work with the legacy format as fallback.
+	// Operations on the legacy format are hard-failures, implying that errors should be propagated up.
+	//
+	// The storage mode is controlled via the CERTMAGIC_STORAGE_MODE environment variable
+	StorageModeEnv = "CERTMAGIC_STORAGE_MODE"
+
+	StorageModeLegacy     = "legacy"
+	StorageModeTransition = "transition"
+	StorageModeBundle     = "bundle"
+)

config.go

@@ -32,6 +32,7 @@ import (
 	"net"
 	"net/http"
 	"net/url"
+	"os"
 	"strings"
 	"time"
 
@@ -752,15 +753,15 @@ func (cfg *Config) reusePrivateKey(ctx context.Context, domain string) (privKey
 
 	for i, issuer := range issuers {
 		// see if this issuer location in storage has a private key for the domain
-		privateKeyStorageKey := StorageKeys.SitePrivateKey(issuer.IssuerKey(), domain)
-		privKeyPEM, err = cfg.Storage.Load(ctx, privateKeyStorageKey)
+		certRes, err := cfg.loadCertResource(ctx, issuer, domain)
 		if errors.Is(err, fs.ErrNotExist) {
 			err = nil // obviously, it's OK to not have a private key; so don't prevent obtaining a cert
 			continue
 		}
 		if err != nil {
 			return nil, nil, nil, fmt.Errorf("loading existing private key for reuse with issuer %s: %v", issuer.IssuerKey(), err)
 		}
+		privKeyPEM = certRes.PrivateKeyPEM
 
 		// we loaded a private key; try decoding it so we can use it
 		privKey, err = PEMDecodePrivateKey(privKeyPEM)
@@ -1101,7 +1102,8 @@ func (cfg *Config) RevokeCert(ctx context.Context, domain string, reason int, in
 			return err
 		}
 
-		if !cfg.Storage.Exists(ctx, StorageKeys.SitePrivateKey(issuerKey, domain)) {
+		// loadCertResource should already fail if private key is missing.
+		if len(certRes.PrivateKeyPEM) == 0 {
 			return fmt.Errorf("private key not found for %s", certRes.SANs)
 		}
 
@@ -1268,9 +1270,26 @@ func (cfg *Config) checkStorage(ctx context.Context) error {
 
 // storageHasCertResources returns true if the storage
 // associated with cfg's certificate cache has all the
+// resources related to the certificate for domain.
+func (cfg *Config) storageHasCertResources(ctx context.Context, issuer Issuer, domain string) bool {
+	switch os.Getenv(StorageModeEnv) {
+	case StorageModeTransition:
+		if cfg.storageHasCertResourcesBundle(ctx, issuer, domain) {
+			return true
+		}
+		return cfg.storageHasCertResourcesLegacy(ctx, issuer, domain)
+	case StorageModeBundle:
+		return cfg.storageHasCertResourcesBundle(ctx, issuer, domain)
+	default:
+		return cfg.storageHasCertResourcesLegacy(ctx, issuer, domain)
+	}
+}
+
+// storageHasCertResourcesLegacy returns true if the storage
+// associated with cfg's certificate cache has all the
 // resources related to the certificate for domain: the
 // certificate, the private key, and the metadata.
-func (cfg *Config) storageHasCertResources(ctx context.Context, issuer Issuer, domain string) bool {
+func (cfg *Config) storageHasCertResourcesLegacy(ctx context.Context, issuer Issuer, domain string) bool {
 	issuerKey := issuer.IssuerKey()
 	certKey := StorageKeys.SiteCert(issuerKey, domain)
 	keyKey := StorageKeys.SitePrivateKey(issuerKey, domain)
@@ -1280,10 +1299,39 @@ func (cfg *Config) storageHasCertResources(ctx context.Context, issuer Issuer, d
 		cfg.Storage.Exists(ctx, metaKey)
 }
 
+// storageHasCertResourcesBundle returns true if the storage
+// associated with cfg's certificate cache has the
+// certificate resource bundle for domain.
+func (cfg *Config) storageHasCertResourcesBundle(ctx context.Context, issuer Issuer, domain string) bool {
+	issuerKey := issuer.IssuerKey()
+	certBundle := StorageKeys.SiteBundle(issuerKey, domain)
+	return cfg.Storage.Exists(ctx, certBundle)
+}
+
 // deleteSiteAssets deletes the folder in storage containing the
 // certificate, private key, and metadata file for domain from the
 // issuer with the given issuer key.
 func (cfg *Config) deleteSiteAssets(ctx context.Context, issuerKey, domain string) error {
+	switch os.Getenv(StorageModeEnv) {
+	case StorageModeTransition:
+		if err := cfg.deleteSiteAssetsBundle(ctx, issuerKey, domain); err != nil {
+			cfg.Logger.Warn("unable to delete certificate resource bundle",
+				zap.String("issuer", issuerKey),
+				zap.String("domain", domain),
+				zap.Error(err))
+		}
+		return cfg.deleteSiteAssetsLegacy(ctx, issuerKey, domain)
+	case StorageModeBundle:
+		return cfg.deleteSiteAssetsBundle(ctx, issuerKey, domain)
+	default:
+		return cfg.deleteSiteAssetsLegacy(ctx, issuerKey, domain)
+	}
+}
+
+// deleteSiteAssetsLegacy deletes the folder in storage containing the
+// certificate, private key, and metadata file for domain from the
+// issuer with the given issuer key.
+func (cfg *Config) deleteSiteAssetsLegacy(ctx context.Context, issuerKey, domain string) error {
 	err := cfg.Storage.Delete(ctx, StorageKeys.SiteCert(issuerKey, domain))
 	if err != nil {
 		return fmt.Errorf("deleting certificate file: %v", err)
@@ -1303,6 +1351,16 @@ func (cfg *Config) deleteSiteAssets(ctx context.Context, issuerKey, domain strin
 	return nil
 }
 
+// deleteSiteAssetsBundle deletes the folder in storage containing the
+// certificate bundle for domain from the issuer with the given issuer key.
+func (cfg *Config) deleteSiteAssetsBundle(ctx context.Context, issuerKey, domain string) error {
+	err := cfg.Storage.Delete(ctx, StorageKeys.SiteBundle(issuerKey, domain))
+	if err != nil {
+		return fmt.Errorf("deleting certificate bundle: %v", err)
+	}
+	return nil
+}
+
 // lockKey returns a key for a lock that is specific to the operation
 // named op being performed related to domainName and this config's CA.
 func (cfg *Config) lockKey(op, domainName string) string {

config_test.go

@@ -154,3 +154,157 @@ func mustJSON(val any) []byte {
 	}
 	return result
 }
+
+// testStorageModeSetup creates a test config with the specified storage mode
+func testStorageModeSetup(t *testing.T, mode, storagePath string) (*Config, *ACMEIssuer) {
+	t.Helper()
+	t.Setenv(StorageModeEnv, mode)
+
+	am := &ACMEIssuer{CA: "https://example.com/acme/directory"}
+	cfg := &Config{
+		Issuers:   []Issuer{am},
+		Storage:   &FileStorage{Path: storagePath},
+		Logger:    defaultTestLogger,
+		certCache: new(Cache),
+	}
+	am.config = cfg
+
+	t.Cleanup(func() {
+		os.RemoveAll(storagePath)
+	})
+
+	return cfg, am
+}
+
+func makeCertResource(am *ACMEIssuer, domain string, useLegacyContent bool) CertificateResource {
+	return CertificateResource{
+		SANs:           []string{domain},
+		PrivateKeyPEM:  []byte("private key"),
+		CertificatePEM: []byte("certificate"),
+		IssuerData:     mustJSON(acme.Certificate{URL: "https://example.com/cert"}),
+		issuerKey:      am.IssuerKey(),
+	}
+}
+
+func assertFileExists(t *testing.T, ctx context.Context, storage Storage, path string) {
+	t.Helper()
+	if !storage.Exists(ctx, path) {
+		t.Errorf("Expected file to exist at %s", path)
+	}
+}
+
+func assertFileNotExists(t *testing.T, ctx context.Context, storage Storage, path string) {
+	t.Helper()
+	if storage.Exists(ctx, path) {
+		t.Errorf("Expected file NOT to exist at %s", path)
+	}
+}
+
+func assertCertResourceContent(t *testing.T, loaded CertificateResource, expectedKey, expectedCert string) {
+	t.Helper()
+	if string(loaded.PrivateKeyPEM) != expectedKey {
+		t.Errorf("Private key mismatch: expected %q, got %q", expectedKey, string(loaded.PrivateKeyPEM))
+	}
+	if string(loaded.CertificatePEM) != expectedCert {
+		t.Errorf("Certificate mismatch: expected %q, got %q", expectedCert, string(loaded.CertificatePEM))
+	}
+}
+
+func TestStorageModeLegacy(t *testing.T) {
+	ctx := context.Background()
+	cfg, am := testStorageModeSetup(t, StorageModeLegacy, "./_testdata_tmp_legacy")
+
+	domain := "example.com"
+	cert := makeCertResource(am, domain, true)
+
+	if err := cfg.saveCertResource(ctx, am, cert); err != nil {
+		t.Fatalf("Failed to save cert resource: %v", err)
+	}
+
+	issuerKey := am.IssuerKey()
+	assertFileExists(t, ctx, cfg.Storage, StorageKeys.SitePrivateKey(issuerKey, domain))
+	assertFileExists(t, ctx, cfg.Storage, StorageKeys.SiteCert(issuerKey, domain))
+	assertFileExists(t, ctx, cfg.Storage, StorageKeys.SiteMeta(issuerKey, domain))
+	assertFileNotExists(t, ctx, cfg.Storage, StorageKeys.SiteBundle(issuerKey, domain))
+
+	loaded, err := cfg.loadCertResource(ctx, am, domain)
+	if err != nil {
+		t.Fatalf("Failed to load cert resource: %v", err)
+	}
+	assertCertResourceContent(t, loaded, "private key", "certificate")
+}
+
+func TestStorageModeBundle(t *testing.T) {
+	ctx := context.Background()
+	cfg, am := testStorageModeSetup(t, StorageModeBundle, "./_testdata_tmp_bundle")
+
+	domain := "example.com"
+	cert := makeCertResource(am, domain, false)
+
+	if err := cfg.saveCertResource(ctx, am, cert); err != nil {
+		t.Fatalf("Failed to save cert resource: %v", err)
+	}
+
+	issuerKey := am.IssuerKey()
+	assertFileExists(t, ctx, cfg.Storage, StorageKeys.SiteBundle(issuerKey, domain))
+	assertFileNotExists(t, ctx, cfg.Storage, StorageKeys.SitePrivateKey(issuerKey, domain))
+	assertFileNotExists(t, ctx, cfg.Storage, StorageKeys.SiteCert(issuerKey, domain))
+	assertFileNotExists(t, ctx, cfg.Storage, StorageKeys.SiteMeta(issuerKey, domain))
+
+	loaded, err := cfg.loadCertResource(ctx, am, domain)
+	if err != nil {
+		t.Fatalf("Failed to load cert resource: %v", err)
+	}
+	assertCertResourceContent(t, loaded, "private key", "certificate")
+}
+
+func TestStorageModeTransition(t *testing.T) {
+	ctx := context.Background()
+	cfg, am := testStorageModeSetup(t, StorageModeTransition, "./_testdata_tmp_transition")
+
+	domain := "example.com"
+	cert := makeCertResource(am, domain, false)
+
+	if err := cfg.saveCertResource(ctx, am, cert); err != nil {
+		t.Fatalf("Failed to save cert resource: %v", err)
+	}
+
+	// Verify BOTH legacy and bundle files exist
+	issuerKey := am.IssuerKey()
+	assertFileExists(t, ctx, cfg.Storage, StorageKeys.SitePrivateKey(issuerKey, domain))
+	assertFileExists(t, ctx, cfg.Storage, StorageKeys.SiteCert(issuerKey, domain))
+	assertFileExists(t, ctx, cfg.Storage, StorageKeys.SiteMeta(issuerKey, domain))
+	assertFileExists(t, ctx, cfg.Storage, StorageKeys.SiteBundle(issuerKey, domain))
+
+	loaded, err := cfg.loadCertResource(ctx, am, domain)
+	if err != nil {
+		t.Fatalf("Failed to load cert resource: %v", err)
+	}
+	assertCertResourceContent(t, loaded, "private key", "certificate")
+}
+
+func TestStorageModeTransitionFallback(t *testing.T) {
+	ctx := context.Background()
+	cfg, am := testStorageModeSetup(t, StorageModeTransition, "./_testdata_tmp_transition_fallback")
+
+	domain := "example.com"
+	cert := makeCertResource(am, domain, true)
+
+	// Save in legacy mode to simulate existing data
+	os.Setenv(StorageModeEnv, StorageModeLegacy)
+	if err := cfg.saveCertResource(ctx, am, cert); err != nil {
+		t.Fatalf("Failed to save cert in legacy mode: %v", err)
+	}
+
+	issuerKey := am.IssuerKey()
+	assertFileExists(t, ctx, cfg.Storage, StorageKeys.SitePrivateKey(issuerKey, domain))
+	assertFileNotExists(t, ctx, cfg.Storage, StorageKeys.SiteBundle(issuerKey, domain))
+
+	// Switch to transition mode and verify fallback to legacy works
+	os.Setenv(StorageModeEnv, StorageModeTransition)
+	loaded, err := cfg.loadCertResource(ctx, am, domain)
+	if err != nil {
+		t.Fatalf("Failed to load cert in transition mode with fallback: %v", err)
+	}
+	assertCertResourceContent(t, loaded, "private key", "certificate")
+}