From 3bccbe1286dde26f81fa084206a44e633f3c0324 Mon Sep 17 00:00:00 2001 From: Orzelius Date: Mon, 29 Jun 2026 22:23:57 +0900 Subject: [PATCH] fix: handle single arch images go-containerregistry's partial.Descriptor only copies Platform onto the index entry when the image was resolved from a multi-arch index. Derive platform from the image's config file if present so single arch images keep working. Part of #395 Signed-off-by: Orzelius <33936483+Orzelius@users.noreply.github.com> --- enterprise/spdx/builder/builder.go | 3 +- internal/artifacts/fetch.go | 78 ++---------- .../artifacts/imagehandler/imagehandler.go | 89 ++++++++++++++ .../imagehandler/imagehandler_test.go | 46 +++++++ internal/artifacts/imagehandler/spdx.go | 116 ++++++++++++++++++ internal/artifacts/spdx.go | 112 +---------------- internal/artifacts/versions.go | 8 +- 7 files changed, 271 insertions(+), 181 deletions(-) create mode 100644 internal/artifacts/imagehandler/imagehandler.go create mode 100644 internal/artifacts/imagehandler/imagehandler_test.go create mode 100644 internal/artifacts/imagehandler/spdx.go diff --git a/enterprise/spdx/builder/builder.go b/enterprise/spdx/builder/builder.go index e5df278..ddae1fe 100644 --- a/enterprise/spdx/builder/builder.go +++ b/enterprise/spdx/builder/builder.go @@ -27,6 +27,7 @@ import ( "github.com/siderolabs/image-factory/enterprise/spdx/storage" "github.com/siderolabs/image-factory/internal/artifacts" + "github.com/siderolabs/image-factory/internal/artifacts/imagehandler" "github.com/siderolabs/image-factory/internal/asset" "github.com/siderolabs/image-factory/internal/ctxlog" "github.com/siderolabs/image-factory/internal/profile" @@ -309,7 +310,7 @@ func (b *Builder) extractSPDXFromInitramfs(bundle *Bundle, bootAsset asset.BootA return nil } - if !strings.HasSuffix(path, artifacts.SPDXFileSuffix) { + if !strings.HasSuffix(path, imagehandler.SPDXFileSuffix) { return nil } diff --git a/internal/artifacts/fetch.go b/internal/artifacts/fetch.go index bd44820..566511c 100644 --- a/internal/artifacts/fetch.go +++ b/internal/artifacts/fetch.go @@ -14,76 +14,16 @@ import ( "path/filepath" "strings" - "github.com/google/go-containerregistry/pkg/crane" "github.com/google/go-containerregistry/pkg/name" - v1 "github.com/google/go-containerregistry/pkg/v1" - "github.com/google/go-containerregistry/pkg/v1/empty" - "github.com/google/go-containerregistry/pkg/v1/layout" "github.com/siderolabs/talos/pkg/machinery/imager/quirks" "go.uber.org/zap" - "golang.org/x/sync/errgroup" + "github.com/siderolabs/image-factory/internal/artifacts/imagehandler" "github.com/siderolabs/image-factory/internal/image/verify" ) -type imageHandler func(ctx context.Context, logger *zap.Logger, img v1.Image) error - -// imageExportHandler exports the image for further processing. -func imageExportHandler(exportHandler func(logger *zap.Logger, r io.Reader) error) imageHandler { - return func(_ context.Context, logger *zap.Logger, img v1.Image) error { - logger.Info("extracting the image") - - r, w := io.Pipe() - - var eg errgroup.Group - - eg.Go(func() error { - defer w.Close() //nolint:errcheck - - return crane.Export(img, w) - }) - - eg.Go(func() error { - err := exportHandler(logger, r) - if err != nil { - r.CloseWithError(err) // signal the exporter to stop - } - - return err - }) - - if err := eg.Wait(); err != nil { - return fmt.Errorf("error extracting the image: %w", err) - } - - return nil - } -} - -// imageOCIHandler exports the image to the OCI format. -func imageOCIHandler(path string) imageHandler { - return func(_ context.Context, logger *zap.Logger, img v1.Image) error { - if err := os.RemoveAll(path); err != nil { - return fmt.Errorf("error removing the directory %q: %w", path, err) - } - - l, err := layout.Write(path, empty.Index) - if err != nil { - return fmt.Errorf("error creating layout: %w", err) - } - - logger.Info("exporting the image", zap.String("destination", path)) - - if err = l.AppendImage(img); err != nil { - return fmt.Errorf("error exporting the image: %w", err) - } - - return nil - } -} - // fetchImageByTag contains combined logic of image handling: heading, downloading, verifying signatures, and exporting. -func (m *Manager) fetchImageByTag(imageName, tag string, architecture Arch, imageHandler imageHandler) error { +func (m *Manager) fetchImageByTag(imageName, tag string, architecture Arch, imageHandler imagehandler.Handler) error { // set a timeout for fetching, but don't bind it to any context, as we want fetch operation to finish ctx, cancel := context.WithTimeout(context.Background(), FetchTimeout) defer cancel() @@ -105,7 +45,7 @@ func (m *Manager) fetchImageByTag(imageName, tag string, architecture Arch, imag } // fetchImageByDigest fetches an image by digest, verifies signatures, and exports it to the storage. -func (m *Manager) fetchImageByDigest(digestRef name.Digest, architecture Arch, imageHandler imageHandler) error { +func (m *Manager) fetchImageByDigest(digestRef name.Digest, architecture Arch, imageHandler imagehandler.Handler) error { var err error // set a timeout for fetching, but don't bind it to any context, as we want fetch operation to finish ctx, cancel := context.WithTimeout(context.Background(), FetchTimeout) @@ -149,7 +89,7 @@ func (m *Manager) fetchImageByDigest(digestRef name.Digest, architecture Arch, i func (m *Manager) fetchImager(tag string) error { destinationPath := filepath.Join(m.storagePath, tag) - if err := m.fetchImageByTag(m.options.ImagerImage, tag, ArchAmd64, imageExportHandler(func(logger *zap.Logger, r io.Reader) error { + if err := m.fetchImageByTag(m.options.ImagerImage, tag, ArchAmd64, imagehandler.Export(func(logger *zap.Logger, r io.Reader) error { return untarWithPrefix(logger, r, usrInstallPrefix, destinationPath+tmpSuffix) })); err != nil { return err @@ -164,7 +104,7 @@ func (m *Manager) extractOverlay(arch Arch, ref OverlayRef) error { destinationPath := filepath.Join(m.storagePath, string(arch)+"-"+ref.Digest+"-overlay") - if err := m.fetchImageByDigest(imageRef, arch, imageExportHandler(func(logger *zap.Logger, r io.Reader) error { + if err := m.fetchImageByDigest(imageRef, arch, imagehandler.Export(func(logger *zap.Logger, r io.Reader) error { return untarWithPrefix(logger, r, overlaysPrefix, destinationPath+tmpSuffix) })); err != nil { return err @@ -177,7 +117,7 @@ func (m *Manager) extractOverlay(arch Arch, ref OverlayRef) error { func (m *Manager) fetchExtensionImage(arch Arch, ref ExtensionRef, destPath string) error { imageRef := ref.TaggedReference.Digest(ref.Digest) - if err := m.fetchImageByDigest(imageRef, arch, imageOCIHandler(destPath+tmpSuffix)); err != nil { + if err := m.fetchImageByDigest(imageRef, arch, imagehandler.OCI(destPath+tmpSuffix)); err != nil { return err } @@ -188,7 +128,7 @@ func (m *Manager) fetchExtensionImage(arch Arch, ref ExtensionRef, destPath stri func (m *Manager) fetchOverlayImage(arch Arch, ref OverlayRef, destPath string) error { imageRef := m.imageRegistry.Repo(ref.TaggedReference.RepositoryStr()).Digest(ref.Digest) - if err := m.fetchImageByDigest(imageRef, arch, imageOCIHandler(destPath+tmpSuffix)); err != nil { + if err := m.fetchImageByDigest(imageRef, arch, imagehandler.OCI(destPath+tmpSuffix)); err != nil { return err } @@ -206,7 +146,7 @@ func (m *Manager) InstallerImageName(versionTag string) string { // fetchInstallerImage fetches a Talos installer image and exports it to the storage. func (m *Manager) fetchInstallerImage(arch Arch, versionTag string, destPath string) error { - if err := m.fetchImageByTag(m.InstallerImageName(versionTag), versionTag, arch, imageOCIHandler(destPath+tmpSuffix)); err != nil { + if err := m.fetchImageByTag(m.InstallerImageName(versionTag), versionTag, arch, imagehandler.OCI(destPath+tmpSuffix)); err != nil { return err } @@ -215,7 +155,7 @@ func (m *Manager) fetchInstallerImage(arch Arch, versionTag string, destPath str // fetchTalosctlImage fetches a Talosctl image and exports it to the storage. func (m *Manager) fetchTalosctlImage(versionTag string, destPath string) error { - if err := m.fetchImageByTag(m.options.TalosctlImage, versionTag, ArchAmd64, imageExportHandler(func(logger *zap.Logger, r io.Reader) error { + if err := m.fetchImageByTag(m.options.TalosctlImage, versionTag, ArchAmd64, imagehandler.Export(func(logger *zap.Logger, r io.Reader) error { return untarWithPrefix(logger, r, "", destPath+tmpSuffix) })); err != nil { return err diff --git a/internal/artifacts/imagehandler/imagehandler.go b/internal/artifacts/imagehandler/imagehandler.go new file mode 100644 index 0000000..3627ab0 --- /dev/null +++ b/internal/artifacts/imagehandler/imagehandler.go @@ -0,0 +1,89 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package imagehandler + +import ( + "context" + "fmt" + "io" + "os" + + "github.com/google/go-containerregistry/pkg/crane" + v1 "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/empty" + "github.com/google/go-containerregistry/pkg/v1/layout" + "go.uber.org/zap" + "golang.org/x/sync/errgroup" +) + +// Handler processes a fetched image, e.g. exporting it to local storage. +type Handler func(ctx context.Context, logger *zap.Logger, img v1.Image) error + +// Export streams the image's flattened filesystem as a tar to exportHandler. +func Export(exportHandler func(logger *zap.Logger, r io.Reader) error) Handler { + return func(_ context.Context, logger *zap.Logger, img v1.Image) error { + logger.Info("extracting the image") + + r, w := io.Pipe() + + var eg errgroup.Group + + eg.Go(func() error { + defer w.Close() //nolint:errcheck + + return crane.Export(img, w) + }) + + eg.Go(func() error { + err := exportHandler(logger, r) + if err != nil { + r.CloseWithError(err) // signal the exporter to stop + } + + return err + }) + + if err := eg.Wait(); err != nil { + return fmt.Errorf("error extracting the image: %w", err) + } + + return nil + } +} + +// OCI exports the image to an OCI layout at path. +func OCI(path string) Handler { + return func(_ context.Context, logger *zap.Logger, img v1.Image) error { + if err := os.RemoveAll(path); err != nil { + return fmt.Errorf("error removing the directory %q: %w", path, err) + } + + l, err := layout.Write(path, empty.Index) + if err != nil { + return fmt.Errorf("error creating layout: %w", err) + } + + logger.Info("exporting the image", zap.String("destination", path)) + + var opts []layout.Option + + // go-containerregistry's partial.Descriptor only copies Platform onto the index entry + // when the image was resolved from a multi-arch index. Derive platform from the image's + // config file if present so single arch images keep working. + if cfg, cfgErr := img.ConfigFile(); cfgErr == nil && cfg.Architecture != "" { + opts = append(opts, layout.WithPlatform(v1.Platform{ + OS: cfg.OS, + Architecture: cfg.Architecture, + Variant: cfg.Variant, + })) + } + + if err = l.AppendImage(img, opts...); err != nil { + return fmt.Errorf("error exporting the image: %w", err) + } + + return nil + } +} diff --git a/internal/artifacts/imagehandler/imagehandler_test.go b/internal/artifacts/imagehandler/imagehandler_test.go new file mode 100644 index 0000000..e3e72a2 --- /dev/null +++ b/internal/artifacts/imagehandler/imagehandler_test.go @@ -0,0 +1,46 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package imagehandler_test + +import ( + "testing" + + v1 "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/layout" + "github.com/google/go-containerregistry/pkg/v1/mutate" + "github.com/google/go-containerregistry/pkg/v1/random" + "github.com/stretchr/testify/require" + "go.uber.org/zap/zaptest" + + "github.com/siderolabs/image-factory/internal/artifacts/imagehandler" +) + +// TestOCIRecordsPlatform asserts that a single-arch image is written to the OCI layout with a +// platform descriptor, so the Talos imager can match it to a target architecture. +func TestOCIRecordsPlatform(t *testing.T) { + t.Parallel() + + img, err := random.Image(1024, 1) + require.NoError(t, err) + + // Single-arch images (e.g. custom-built extensions) carry their arch only in the config. + img, err = mutate.ConfigFile(img, &v1.ConfigFile{OS: "linux", Architecture: "arm64"}) + require.NoError(t, err) + + path := t.TempDir() + "/oci" + + require.NoError(t, imagehandler.OCI(path)(t.Context(), zaptest.NewLogger(t), img)) + + idx, err := layout.ImageIndexFromPath(path) + require.NoError(t, err) + + manifest, err := idx.IndexManifest() + require.NoError(t, err) + + require.Len(t, manifest.Manifests, 1) + require.NotNil(t, manifest.Manifests[0].Platform, "platform descriptor must be recorded") + require.Equal(t, "linux", manifest.Manifests[0].Platform.OS) + require.Equal(t, "arm64", manifest.Manifests[0].Platform.Architecture) +} diff --git a/internal/artifacts/imagehandler/spdx.go b/internal/artifacts/imagehandler/spdx.go new file mode 100644 index 0000000..21d15b4 --- /dev/null +++ b/internal/artifacts/imagehandler/spdx.go @@ -0,0 +1,116 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package imagehandler + +import ( + "archive/tar" + "context" + "errors" + "fmt" + "io" + "path/filepath" + "strings" + + "github.com/google/go-containerregistry/pkg/crane" + v1 "github.com/google/go-containerregistry/pkg/v1" + "go.uber.org/zap" + "golang.org/x/sync/errgroup" +) + +const ( + // SPDXFileSuffix is the file suffix for SPDX files. + SPDXFileSuffix = ".spdx.json" +) + +// SPDXFile represents an extracted SPDX file. +type SPDXFile struct { + // Filename is the original filename (e.g., "extension-name.spdx.json"). + Filename string + + // Source is the source identifier (extension name or "talos"). + Source string + + // Content is the raw JSON content. + Content []byte +} + +// SPDX creates an image handler that extracts SPDX files. +func SPDX(files *[]SPDXFile, source string) Handler { + return func(_ context.Context, logger *zap.Logger, img v1.Image) error { + logger.Info("extracting SPDX files from image") + + r, w := io.Pipe() + + var eg errgroup.Group + + eg.Go(func() error { + defer w.Close() //nolint:errcheck + + return crane.Export(img, w) + }) + + eg.Go(func() error { + extracted, err := extractSPDXFromTar(r, source) + if err != nil { + r.CloseWithError(err) + + return err + } + + *files = extracted + + return nil + }) + + if err := eg.Wait(); err != nil { + return fmt.Errorf("error extracting SPDX files: %w", err) + } + + logger.Info("extracted SPDX files", zap.Int("count", len(*files))) + + return nil + } +} + +// extractSPDXFromTar extracts SPDX files from a tar stream. +func extractSPDXFromTar(r io.Reader, source string) ([]SPDXFile, error) { + tr := tar.NewReader(r) + + var files []SPDXFile + + for { + hdr, err := tr.Next() + if err != nil { + if errors.Is(err, io.EOF) { + break + } + + return nil, fmt.Errorf("error reading tar header: %w", err) + } + + if hdr.Typeflag != tar.TypeReg { + continue + } + + // Check if the file is an SPDX file + if !strings.HasSuffix(hdr.Name, SPDXFileSuffix) { + continue + } + + // Read the file content + content, err := io.ReadAll(tr) + if err != nil { + return nil, fmt.Errorf("error reading SPDX file %q: %w", hdr.Name, err) + } + + files = append(files, SPDXFile{ + Filename: filepath.Base(hdr.Name), + Source: source, + Content: content, + }) + } + + return files, nil +} diff --git a/internal/artifacts/spdx.go b/internal/artifacts/spdx.go index 7a88768..789609f 100644 --- a/internal/artifacts/spdx.go +++ b/internal/artifacts/spdx.go @@ -5,44 +5,19 @@ package artifacts import ( - "archive/tar" "context" - "errors" "fmt" - "io" - "path/filepath" - "strings" - "github.com/google/go-containerregistry/pkg/crane" - v1 "github.com/google/go-containerregistry/pkg/v1" - "go.uber.org/zap" - "golang.org/x/sync/errgroup" + "github.com/siderolabs/image-factory/internal/artifacts/imagehandler" ) -const ( - // SPDXFileSuffix is the file suffix for SPDX files. - SPDXFileSuffix = ".spdx.json" -) - -// SPDXFile represents an extracted SPDX file. -type SPDXFile struct { - // Filename is the original filename (e.g., "extension-name.spdx.json"). - Filename string - - // Source is the source identifier (extension name or "talos"). - Source string - - // Content is the raw JSON content. - Content []byte -} - // ExtractExtensionSPDX extracts SPDX files from an extension image. -func (m *Manager) ExtractExtensionSPDX(ctx context.Context, arch Arch, ref ExtensionRef) ([]SPDXFile, error) { +func (m *Manager) ExtractExtensionSPDX(ctx context.Context, arch Arch, ref ExtensionRef) ([]imagehandler.SPDXFile, error) { imageRef := ref.TaggedReference.Digest(ref.Digest) - var files []SPDXFile + var files []imagehandler.SPDXFile - handler := spdxExportHandler(&files, ref.TaggedReference.RepositoryStr()) + handler := imagehandler.SPDX(&files, ref.TaggedReference.RepositoryStr()) if err := m.fetchImageByDigest(imageRef, arch, handler); err != nil { //nolint:contextcheck return nil, fmt.Errorf("failed to extract SPDX from extension %s: %w", ref.TaggedReference.RepositoryStr(), err) @@ -50,82 +25,3 @@ func (m *Manager) ExtractExtensionSPDX(ctx context.Context, arch Arch, ref Exten return files, nil } - -// spdxExportHandler creates an image handler that extracts SPDX files. -func spdxExportHandler(files *[]SPDXFile, source string) imageHandler { - return func(_ context.Context, logger *zap.Logger, img v1.Image) error { - logger.Info("extracting SPDX files from image") - - r, w := io.Pipe() - - var eg errgroup.Group - - eg.Go(func() error { - defer w.Close() //nolint:errcheck - - return crane.Export(img, w) - }) - - eg.Go(func() error { - extracted, err := extractSPDXFromTar(r, source) - if err != nil { - r.CloseWithError(err) - - return err - } - - *files = extracted - - return nil - }) - - if err := eg.Wait(); err != nil { - return fmt.Errorf("error extracting SPDX files: %w", err) - } - - logger.Info("extracted SPDX files", zap.Int("count", len(*files))) - - return nil - } -} - -// extractSPDXFromTar extracts SPDX files from a tar stream. -func extractSPDXFromTar(r io.Reader, source string) ([]SPDXFile, error) { - tr := tar.NewReader(r) - - var files []SPDXFile - - for { - hdr, err := tr.Next() - if err != nil { - if errors.Is(err, io.EOF) { - break - } - - return nil, fmt.Errorf("error reading tar header: %w", err) - } - - if hdr.Typeflag != tar.TypeReg { - continue - } - - // Check if the file is an SPDX file - if !strings.HasSuffix(hdr.Name, SPDXFileSuffix) { - continue - } - - // Read the file content - content, err := io.ReadAll(tr) - if err != nil { - return nil, fmt.Errorf("error reading SPDX file %q: %w", hdr.Name, err) - } - - files = append(files, SPDXFile{ - Filename: filepath.Base(hdr.Name), - Source: source, - Content: content, - }) - } - - return files, nil -} diff --git a/internal/artifacts/versions.go b/internal/artifacts/versions.go index 864be83..41bde22 100644 --- a/internal/artifacts/versions.go +++ b/internal/artifacts/versions.go @@ -20,6 +20,8 @@ import ( "github.com/siderolabs/gen/xslices" "go.uber.org/zap" "go.yaml.in/yaml/v4" + + "github.com/siderolabs/image-factory/internal/artifacts/imagehandler" ) func (m *Manager) fetchTalosVersions() (any, error) { @@ -134,7 +136,7 @@ type overlaysDescription struct { func (m *Manager) fetchExtensionList(image, tag string) ([]ExtensionRef, error) { var extensions []ExtensionRef - err := m.fetchImageByTag(image, tag, ArchAmd64, imageExportHandler(func(_ *zap.Logger, r io.Reader) error { + err := m.fetchImageByTag(image, tag, ArchAmd64, imagehandler.Export(func(_ *zap.Logger, r io.Reader) error { var extractErr error extensions, extractErr = extractExtensionList(r, m.imageRegistry) @@ -151,7 +153,7 @@ func (m *Manager) fetchExtensionList(image, tag string) ([]ExtensionRef, error) func (m *Manager) fetchOverlayList(tag string) ([]OverlayRef, error) { var overlays []OverlayRef - err := m.fetchImageByTag(m.options.OverlayManifestImage, tag, ArchAmd64, imageExportHandler(func(_ *zap.Logger, r io.Reader) error { + err := m.fetchImageByTag(m.options.OverlayManifestImage, tag, ArchAmd64, imagehandler.Export(func(_ *zap.Logger, r io.Reader) error { var extractErr error overlays, extractErr = extractOverlayList(r) @@ -168,7 +170,7 @@ func (m *Manager) fetchOverlayList(tag string) ([]OverlayRef, error) { func (m *Manager) fetchTalosctlTuples(tag string) ([]TalosctlTuple, error) { var talosctlTuples []TalosctlTuple - err := m.fetchImageByTag(m.options.TalosctlImage, tag, ArchAmd64, imageExportHandler(func(_ *zap.Logger, r io.Reader) error { + err := m.fetchImageByTag(m.options.TalosctlImage, tag, ArchAmd64, imagehandler.Export(func(_ *zap.Logger, r io.Reader) error { var extractErr error talosctlTuples, extractErr = extractTalosctlTuples(r)