feat: add support for extra (custom) extensions

wip

Signed-off-by: Orzelius <33936483+Orzelius@users.noreply.github.com>

Author: Orzelius

cmd/image-factory/cmd/options.go

@@ -502,6 +502,17 @@ type EnterpriseOptions struct {
 
 	// VEX contains configuration for VEX data fetching.
 	VEX VEXOptions `koanf:"vex"`
+
+	// ExtraExtensions contains configuration for extra (custom) extensions.
+	ExtraExtensions ExtraExtensionsOptions `koanf:"extraExtensions"`
+}
+
+// ExtraExtensionsOptions configures custom extensions offered alongside the official ones.
+type ExtraExtensionsOptions struct {
+	// Manifest specifies the OCI repository holding the extra extensions manifest image.
+	//
+	// It may live in a different registry than the official images.
+	Manifest OCIRepositoryOptions `koanf:"manifest"`
 }
 
 // SPDXOptions configures SPDX document generation and caching.

cmd/image-factory/cmd/service.go

@@ -485,6 +485,12 @@ func buildArtifactsManager(logger *zap.Logger, opts Options) (*artifacts.Manager
 		OverlayManifestImage:   opts.Artifacts.Core.Components.OverlayManifest,
 		TalosctlImage:          opts.Artifacts.Core.Components.Talosctl,
 
+		// Extra extensions manifest may live in its own registry, so it's passed as a full reference.
+		ExtraExtensionsManifest: artifacts.RepositoryRef{
+			Image:    opts.Enterprise.ExtraExtensions.Manifest.String(),
+			Insecure: opts.Enterprise.ExtraExtensions.Manifest.Insecure,
+		},
+
 		ExternalURL: opts.HTTP.ExternalURL,
 	})
 	if err != nil {

enterprise/spdx/builder/builder.go

@@ -92,7 +92,7 @@ func (b *Builder) PayloadHash(ctx context.Context, schematicID, versionTag strin
 		return "", fmt.Errorf("failed to get schematic: %w", err)
 	}
 
-	return Hash(sc.Customization.SystemExtensions.OfficialExtensions, versionTag, string(arch)), nil
+	return Hash(sc.Customization.SystemExtensions.AllExtensions(), versionTag, string(arch)), nil
 }
 
 // Build returns an SPDX bundle, building and caching if necessary.
@@ -118,7 +118,7 @@ func (b *Builder) Build(ctx context.Context, schematicID, versionTag string, arc
 	// Compute cache key from only the inputs that affect the SBOM content
 	// (extensions list, version, architecture), so that schematics differing
 	// in other fields share the same cached bundle.
-	sbomHash := Hash(sc.Customization.SystemExtensions.OfficialExtensions, versionTag, string(arch))
+	sbomHash := Hash(sc.Customization.SystemExtensions.AllExtensions(), versionTag, string(arch))
 
 	// Check cache first
 	if err := b.storage.Head(ctx, sbomHash); err == nil {
@@ -177,10 +177,10 @@ func (b *Builder) buildBundle(reqID string, sc *schematicpkg.Schematic, schemati
 	}
 
 	logger.Debug("building SPDX bundle from extensions",
-		zap.Int("extensions", len(sc.Customization.SystemExtensions.OfficialExtensions)))
+		zap.Int("extensions", len(sc.Customization.SystemExtensions.AllExtensions())))
 
-	// Extract SPDX from extensions
-	if len(sc.Customization.SystemExtensions.OfficialExtensions) > 0 {
+	// Extract SPDX from extensions (official and extra)
+	if len(sc.Customization.SystemExtensions.AllExtensions()) > 0 {
 		if err = b.extractExtensionsSPDX(ctx, bundle, sc, versionTag, arch); err != nil {
 			logger.Warn("failed to extract SPDX from some extensions", zap.Error(err))
 		}
@@ -350,12 +350,12 @@ func (b *Builder) extractSPDXFromInitramfs(bundle *Bundle, bootAsset asset.BootA
 func (b *Builder) extractExtensionsSPDX(ctx context.Context, bundle *Bundle, schematicData *schematicpkg.Schematic, versionTag string, arch artifacts.Arch) error {
 	logger := ctxlog.Logger(ctx, b.logger)
 
-	availableExtensions, err := b.artifactsManager.GetOfficialExtensions(ctx, versionTag)
+	availableExtensions, err := b.artifactsManager.GetAllAvailableExtensions(ctx, versionTag)
 	if err != nil {
 		return fmt.Errorf("failed to get official extensions: %w", err)
 	}
 
-	for _, extensionName := range schematicData.Customization.SystemExtensions.OfficialExtensions {
+	for _, extensionName := range schematicData.Customization.SystemExtensions.AllExtensions() {
 		extensionRef := findExtension(availableExtensions, extensionName)
 
 		if value.IsZero(extensionRef) {

hack/dev/config.yaml

@@ -9,19 +9,22 @@ artifacts:
     registry: registry.local:5000
     namespace: image-factory
     repository: schematic
+    insecure: true
 
   installer:
     # internal registry namespace to push installer images to
     internal:
       registry: registry.local:5000
       namespace: siderolabs
+      insecure: true
 
 cache:
   oci:
     # private registry repository for cached assets
     registry: registry.local:5000
     namespace: image-factory
     repository: cache
+    insecure: true
 
   # path to the ECDSA private key (to sign cached assets)
   signingKeyPath: /etc/image-factory/cache-signing-key.key
@@ -30,17 +33,27 @@ http:
   # external URL the Image Factory is available at
   externalURL: http://localhost:8080
 
+containerSignature:
+  disabled: true
+
 authentication:
   enabled: false
   htpasswdPath: /etc/image-factory/htpasswd
 
 enterprise:
+  extraExtensions:
+    manifest:
+      registry: host.docker.internal:5005
+      namespace: extension-testing
+      repository: extensions
+      insecure: true
   spdx:
     cache:
       # private registry repository for cached spdx
       registry: registry.local:5000
       namespace: image-factory
       repository: spdx-cache
+      insecure: true
   vex:
     data:
       registry: registry.ghcr.io:5000

internal/artifacts/artifacts.go

@@ -43,10 +43,23 @@ type Options struct { //nolint:govet
 	OverlayManifestImage   string
 	TalosctlImage          string
 
+	// ExtraExtensionsManifest references the extra extensions manifest. Unlike the official
+	// images it may live in a separate registry, so it is resolved independently of ImageRegistry.
+	// Its zero value disables extra extensions.
+	ExtraExtensionsManifest RepositoryRef
+
 	// External identification.
 	ExternalURL string
 }
 
+// RepositoryRef identifies an image repository that may live in its own registry.
+type RepositoryRef struct {
+	// Image is the full reference (registry/namespace/repository).
+	Image string
+	// Insecure allows connecting to the registry over HTTP or with invalid TLS certificates.
+	Insecure bool
+}
+
 // ImageVerifyOptions are the options for verifying the image signature.
 type ImageVerifyOptions = verify.VerifyOptions
 

internal/artifacts/fetch.go

@@ -14,83 +14,23 @@ import (
 	"path/filepath"
 	"strings"
 
-	"github.com/google/go-containerregistry/pkg/crane"
 	"github.com/google/go-containerregistry/pkg/name"
-	v1 "github.com/google/go-containerregistry/pkg/v1"
-	"github.com/google/go-containerregistry/pkg/v1/empty"
-	"github.com/google/go-containerregistry/pkg/v1/layout"
 	"github.com/siderolabs/talos/pkg/machinery/imager/quirks"
 	"go.uber.org/zap"
-	"golang.org/x/sync/errgroup"
 
+	"github.com/siderolabs/image-factory/internal/artifacts/imagehandler"
 	"github.com/siderolabs/image-factory/internal/image/verify"
 )
 
-type imageHandler func(ctx context.Context, logger *zap.Logger, img v1.Image) error
-
-// imageExportHandler exports the image for further processing.
-func imageExportHandler(exportHandler func(logger *zap.Logger, r io.Reader) error) imageHandler {
-	return func(_ context.Context, logger *zap.Logger, img v1.Image) error {
-		logger.Info("extracting the image")
-
-		r, w := io.Pipe()
-
-		var eg errgroup.Group
-
-		eg.Go(func() error {
-			defer w.Close() //nolint:errcheck
-
-			return crane.Export(img, w)
-		})
-
-		eg.Go(func() error {
-			err := exportHandler(logger, r)
-			if err != nil {
-				r.CloseWithError(err) // signal the exporter to stop
-			}
-
-			return err
-		})
-
-		if err := eg.Wait(); err != nil {
-			return fmt.Errorf("error extracting the image: %w", err)
-		}
-
-		return nil
-	}
-}
-
-// imageOCIHandler exports the image to the OCI format.
-func imageOCIHandler(path string) imageHandler {
-	return func(_ context.Context, logger *zap.Logger, img v1.Image) error {
-		if err := os.RemoveAll(path); err != nil {
-			return fmt.Errorf("error removing the directory %q: %w", path, err)
-		}
-
-		l, err := layout.Write(path, empty.Index)
-		if err != nil {
-			return fmt.Errorf("error creating layout: %w", err)
-		}
-
-		logger.Info("exporting the image", zap.String("destination", path))
-
-		if err = l.AppendImage(img); err != nil {
-			return fmt.Errorf("error exporting the image: %w", err)
-		}
-
-		return nil
-	}
-}
-
 // fetchImageByTag contains combined logic of image handling: heading, downloading, verifying signatures, and exporting.
-func (m *Manager) fetchImageByTag(imageName, tag string, architecture Arch, imageHandler imageHandler) error {
+func (m *Manager) fetchImageByTag(repo name.Repository, tag string, architecture Arch, insecure bool, handler imagehandler.Handler) error {
 	// set a timeout for fetching, but don't bind it to any context, as we want fetch operation to finish
 	ctx, cancel := context.WithTimeout(context.Background(), FetchTimeout)
 	defer cancel()
 
 	// light check first - if the image exists, and resolve the digest
 	// it's important to do further checks by digest exactly
-	repoRef := m.imageRegistry.Repo(imageName).Tag(tag)
+	repoRef := repo.Tag(tag)
 
 	m.logger.Debug("heading the image", zap.Stringer("image", repoRef))
 
@@ -101,11 +41,11 @@ func (m *Manager) fetchImageByTag(imageName, tag string, architecture Arch, imag
 
 	digestRef := repoRef.Digest(descriptor.Digest.String())
 
-	return m.fetchImageByDigest(digestRef, architecture, imageHandler)
+	return m.fetchImageByDigest(digestRef, architecture, insecure, handler)
 }
 
 // fetchImageByDigest fetches an image by digest, verifies signatures, and exports it to the storage.
-func (m *Manager) fetchImageByDigest(digestRef name.Digest, architecture Arch, imageHandler imageHandler) error {
+func (m *Manager) fetchImageByDigest(digestRef name.Digest, architecture Arch, insecure bool, handler imagehandler.Handler) error {
 	var err error
 	// set a timeout for fetching, but don't bind it to any context, as we want fetch operation to finish
 	ctx, cancel := context.WithTimeout(context.Background(), FetchTimeout)
@@ -118,7 +58,7 @@ func (m *Manager) fetchImageByDigest(digestRef name.Digest, architecture Arch, i
 
 	var nameOptions []name.Option
 
-	if m.options.InsecureImageRegistry {
+	if insecure {
 		nameOptions = append(nameOptions, name.Insecure)
 	}
 
@@ -142,14 +82,14 @@ func (m *Manager) fetchImageByDigest(digestRef name.Digest, architecture Arch, i
 		return fmt.Errorf("error creating image from descriptor: %w", err)
 	}
 
-	return imageHandler(ctx, logger, img)
+	return handler(ctx, logger, img)
 }
 
 // fetchImager fetches 'imager' container, and saves to the storage path.
 func (m *Manager) fetchImager(tag string) error {
 	destinationPath := filepath.Join(m.storagePath, tag)
 
-	if err := m.fetchImageByTag(m.options.ImagerImage, tag, ArchAmd64, imageExportHandler(func(logger *zap.Logger, r io.Reader) error {
+	if err := m.fetchImageByTag(m.imageRegistry.Repo(m.options.ImagerImage), tag, ArchAmd64, m.options.InsecureImageRegistry, imagehandler.Export(func(logger *zap.Logger, r io.Reader) error {
 		return untarWithPrefix(logger, r, usrInstallPrefix, destinationPath+tmpSuffix)
 	})); err != nil {
 		return err
@@ -164,7 +104,7 @@ func (m *Manager) extractOverlay(arch Arch, ref OverlayRef) error {
 
 	destinationPath := filepath.Join(m.storagePath, string(arch)+"-"+ref.Digest+"-overlay")
 
-	if err := m.fetchImageByDigest(imageRef, arch, imageExportHandler(func(logger *zap.Logger, r io.Reader) error {
+	if err := m.fetchImageByDigest(imageRef, arch, m.options.InsecureImageRegistry, imagehandler.Export(func(logger *zap.Logger, r io.Reader) error {
 		return untarWithPrefix(logger, r, overlaysPrefix, destinationPath+tmpSuffix)
 	})); err != nil {
 		return err
@@ -175,9 +115,9 @@ func (m *Manager) extractOverlay(arch Arch, ref OverlayRef) error {
 
 // fetchExtensionImage fetches a specified extension image and exports it to the storage as OCI.
 func (m *Manager) fetchExtensionImage(arch Arch, ref ExtensionRef, destPath string) error {
-	imageRef := m.imageRegistry.Repo(ref.TaggedReference.RepositoryStr()).Digest(ref.Digest)
+	imageRef := ref.Registry.Repo(ref.TaggedReference.RepositoryStr()).Digest(ref.Digest)
 
-	if err := m.fetchImageByDigest(imageRef, arch, imageOCIHandler(destPath+tmpSuffix)); err != nil {
+	if err := m.fetchImageByDigest(imageRef, arch, ref.Insecure, imagehandler.OCI(destPath+tmpSuffix)); err != nil {
 		return err
 	}
 
@@ -188,7 +128,7 @@ func (m *Manager) fetchExtensionImage(arch Arch, ref ExtensionRef, destPath stri
 func (m *Manager) fetchOverlayImage(arch Arch, ref OverlayRef, destPath string) error {
 	imageRef := m.imageRegistry.Repo(ref.TaggedReference.RepositoryStr()).Digest(ref.Digest)
 
-	if err := m.fetchImageByDigest(imageRef, arch, imageOCIHandler(destPath+tmpSuffix)); err != nil {
+	if err := m.fetchImageByDigest(imageRef, arch, m.options.InsecureImageRegistry, imagehandler.OCI(destPath+tmpSuffix)); err != nil {
 		return err
 	}
 
@@ -206,7 +146,7 @@ func (m *Manager) InstallerImageName(versionTag string) string {
 
 // fetchInstallerImage fetches a Talos installer image and exports it to the storage.
 func (m *Manager) fetchInstallerImage(arch Arch, versionTag string, destPath string) error {
-	if err := m.fetchImageByTag(m.InstallerImageName(versionTag), versionTag, arch, imageOCIHandler(destPath+tmpSuffix)); err != nil {
+	if err := m.fetchImageByTag(m.imageRegistry.Repo(m.InstallerImageName(versionTag)), versionTag, arch, m.options.InsecureImageRegistry, imagehandler.OCI(destPath+tmpSuffix)); err != nil {
 		return err
 	}
 
@@ -215,7 +155,7 @@ func (m *Manager) fetchInstallerImage(arch Arch, versionTag string, destPath str
 
 // fetchTalosctlImage fetches a Talosctl image and exports it to the storage.
 func (m *Manager) fetchTalosctlImage(versionTag string, destPath string) error {
-	if err := m.fetchImageByTag(m.options.TalosctlImage, versionTag, ArchAmd64, imageExportHandler(func(logger *zap.Logger, r io.Reader) error {
+	if err := m.fetchImageByTag(m.imageRegistry.Repo(m.options.TalosctlImage), versionTag, ArchAmd64, m.options.InsecureImageRegistry, imagehandler.Export(func(logger *zap.Logger, r io.Reader) error {
 		return untarWithPrefix(logger, r, "", destPath+tmpSuffix)
 	})); err != nil {
 		return err