reworked package recording so that it takes the parent image filesystem into account

Author: djcass44

cmd/build.go

@@ -6,6 +6,7 @@ import (
 	"os"
 	"path/filepath"
 	"strings"
+	"time"
 
 	"chainguard.dev/apko/pkg/apk/fs"
 	"github.com/Snakdy/container-build-engine/pkg/builder"
@@ -48,7 +49,8 @@ const (
 	flagCacheDir = "cache-dir"
 	flagPlatform = "platform"
 
-	flagSkipCACerts = "skip-ca-certificates"
+	flagSkipCACerts          = "skip-ca-certificates"
+	flagSkipPackageRecording = "skip-package-recording"
 )
 
 const (
@@ -70,6 +72,7 @@ func init() {
 	buildCmd.Flags().String(flagPlatform, "linux/amd64", "build platform")
 
 	buildCmd.Flags().Bool(flagSkipCACerts, false, "skip running update-ca-certificates")
+	buildCmd.Flags().Bool(flagSkipPackageRecording, true, "skip package recording")
 
 	_ = buildCmd.MarkFlagRequired(flagConfig)
 	_ = buildCmd.MarkFlagFilename(flagConfig, ".yaml", ".yml")
@@ -142,19 +145,56 @@ func build(cmd *cobra.Command, _ []string) error {
 	_ = os.Chdir(wd)
 	log.Info("updating working directory", "dir", wd)
 
-	rootfs := fs.NewMemFS()
+	// figure out what the uid should be
+	uid := cfg.Spec.User.Uid
+	if uid <= 0 && forceUid > 0 && forceUid != defaultUid {
+		uid = forceUid
+	} else if uid <= 0 {
+		uid = defaultUid
+	}
+
+	log.Info("preparing to build image", "username", username, "uid", uid, "dirfs", cfg.Spec.DirFS)
+	var filesystem fs.FullFS
+	if cfg.Spec.DirFS {
+		tmpFs, err := os.MkdirTemp("", "container-build-engine-fs-*")
+		if err != nil {
+			log.Error(err, "failed to setup tmpfs")
+			return err
+		}
+		filesystem = vfs.NewVFS(tmpFs)
+	} else {
+		filesystem = fs.NewMemFS()
+	}
 	log.V(3).Info("prepared root filesystem")
 
+	baseImage := airutil.ExpandEnv(lockFile.Packages[""].Resolved)
+	switch baseImage {
+	case containers.MagicImageScratch:
+	case "":
+		log.Info("using scratch base as nothing was provided")
+		baseImage = containers.MagicImageScratch
+	default:
+		baseImage = airutil.ExpandEnv(cfg.Spec.From)
+	}
+
+	// pull the base image
+	pullStart := time.Now()
+	baseImg, err := containers.Get(cmd.Context(), baseImage)
+	if err != nil {
+		return err
+	}
+	log.Info("pulled base image", "duration", time.Since(pullStart))
+
 	dl, err := downloader.NewDownloader(cacheDir)
 	if err != nil {
 		return err
 	}
 
-	alpineKeeper, err := alpine.NewPackageKeeper(cmd.Context(), repoURLs(cfg.Spec.Repositories[strings.ToLower(string(aybv1.PackageAlpine))]), rootfs)
+	alpineKeeper, err := alpine.NewPackageKeeper(cmd.Context(), repoURLs(cfg.Spec.Repositories[strings.ToLower(string(aybv1.PackageAlpine))]), filesystem, baseImg)
 	if err != nil {
 		return err
 	}
-	debianKeeper, err := debian.NewPackageKeeper(cmd.Context(), repoURLs(cfg.Spec.Repositories[strings.ToLower(string(aybv1.PackageDebian))]))
+	debianKeeper, err := debian.NewPackageKeeper(cmd.Context(), repoURLs(cfg.Spec.Repositories[strings.ToLower(string(aybv1.PackageDebian))]), filesystem, baseImg)
 	if err != nil {
 		return err
 	}
@@ -197,21 +237,6 @@ func build(cmd *cobra.Command, _ []string) error {
 		pkgDeps = append(pkgDeps, id)
 	}
 
-	baseImage := airutil.ExpandEnv(lockFile.Packages[""].Resolved)
-	switch baseImage {
-	case containers.MagicImageScratch:
-	case "":
-		log.Info("using scratch base as nothing was provided")
-		baseImage = containers.MagicImageScratch
-	default:
-		baseImage = airutil.ExpandEnv(cfg.Spec.From)
-	}
-
-	// pull the base image
-	baseImg, err := containers.Get(cmd.Context(), baseImage)
-	if err != nil {
-		return err
-	}
 	imgCfg, err := baseImg.ConfigFile()
 	if err != nil {
 		return err
@@ -300,7 +325,7 @@ func build(cmd *cobra.Command, _ []string) error {
 
 	// update ca certificates
 	if !skipCaCerts {
-		if err := cacertificates.UpdateCertificates(cmd.Context(), rootfs); err != nil {
+		if err := cacertificates.UpdateCertificates(cmd.Context(), filesystem); err != nil {
 			return err
 		}
 	}
@@ -310,28 +335,7 @@ func build(cmd *cobra.Command, _ []string) error {
 		entrypoint = []string{"/bin/sh"}
 	}
 
-	// figure out what the uid should be
-	uid := cfg.Spec.User.Uid
-	if uid <= 0 && forceUid > 0 && forceUid != defaultUid {
-		uid = forceUid
-	} else if uid <= 0 {
-		uid = defaultUid
-	}
-
 	// package everything up as our final container image
-	log.Info("preparing to build image", "username", username, "uid", uid, "dirfs", cfg.Spec.DirFS)
-	var filesystem fs.FullFS
-	if cfg.Spec.DirFS {
-		tmpFs, err := os.MkdirTemp("", "container-build-engine-fs-*")
-		if err != nil {
-			log.Error(err, "failed to setup tmpfs")
-			return err
-		}
-		filesystem = vfs.NewVFS(tmpFs)
-	} else {
-		filesystem = fs.NewMemFS()
-	}
-
 	imageBuilder, err := builder.NewBuilder(cmd.Context(), baseImage, pipelineStatements, builder.Options{
 		Username:        username,
 		Uid:             uid,

cmd/lock.go

@@ -109,11 +109,11 @@ func lock(cmd *cobra.Command, _ []string) error {
 		}
 	}
 
-	alpineKeeper, err := alpine.NewPackageKeeper(cmd.Context(), repoURLs(cfg.Spec.Repositories[strings.ToLower(string(aybv1.PackageAlpine))]), fs.NewMemFS())
+	alpineKeeper, err := alpine.NewPackageKeeper(cmd.Context(), repoURLs(cfg.Spec.Repositories[strings.ToLower(string(aybv1.PackageAlpine))]), fs.NewMemFS(), nil)
 	if err != nil {
 		return err
 	}
-	debianKeeper, err := debian.NewPackageKeeper(cmd.Context(), repoURLs(cfg.Spec.Repositories[strings.ToLower(string(aybv1.PackageDebian))]))
+	debianKeeper, err := debian.NewPackageKeeper(cmd.Context(), repoURLs(cfg.Spec.Repositories[strings.ToLower(string(aybv1.PackageDebian))]), fs.NewMemFS(), nil)
 	if err != nil {
 		return err
 	}

internal/containerutil/push.go

@@ -5,6 +5,7 @@ import (
 	"crypto/x509"
 	"fmt"
 	"net/http"
+	"time"
 
 	"github.com/Snakdy/container-build-engine/pkg/oci/auth"
 	"github.com/go-logr/logr"
@@ -20,6 +21,8 @@ func Push(ctx context.Context, img v1.Image, dst string, certPool *x509.CertPool
 	log := logr.FromContextOrDiscard(ctx).WithValues("ref", dst)
 	log.Info("pushing image")
 
+	start := time.Now()
+
 	// tweak the default transport so that we
 	// can provide a custom certPool
 	transport := remote.DefaultTransport.(*http.Transport).Clone()
@@ -43,5 +46,7 @@ func Push(ctx context.Context, img v1.Image, dst string, certPool *x509.CertPool
 		return err
 	}
 	fmt.Println(ref.String() + "@" + d.String())
+
+	log.Info("pushed image", "duration", time.Since(start))
 	return nil
 }

internal/statements/package.go

@@ -92,6 +92,11 @@ func (s *PackageStatement) Run(ctx *pipelines.BuildContext, _ ...cbev1.Options)
 		}
 	}
 
+	if _, err := keeper.Resolve(ctx.Context, name); err != nil {
+		log.Error(err, "failed to resolve package", "name", name, "version", version)
+		return cbev1.Options{}, fmt.Errorf("resolving package details: %w", err)
+	}
+
 	// unpack the package into the root
 	// filesystem
 	if err := keeper.Unpack(ctx.Context, pkgPath, ctx.FS); err != nil {

pkg/archiveutil/tar.go

@@ -45,6 +45,48 @@ func Zuntar(ctx context.Context, r io.Reader, rootfs fs.FullFS) error {
 	return Untar(ctx, zp, rootfs)
 }
 
+// UntarFile extracts a single file from a given archive.
+func UntarFile(ctx context.Context, r io.Reader, filename string, rootfs fs.FullFS) error {
+	log := logr.FromContextOrDiscard(ctx).WithValues("filename", filename)
+	log.V(3).Info("searching tarball for file")
+	tr := tar.NewReader(r)
+
+	for {
+		header, err := tr.Next()
+		switch {
+		case err == io.EOF:
+			return nil
+		case err != nil:
+			log.Error(err, "failed to read file from archive")
+			return err
+		case header == nil:
+			continue
+		}
+
+		target := filepath.Clean("/" + header.Name)
+
+		if header.Typeflag != tar.TypeReg || header.Name != filename {
+			return nil
+		}
+		log.V(5).Info("creating file", "target", target, "mode", header.Mode)
+		f, err := rootfs.OpenFile(target, os.O_CREATE|os.O_RDWR, os.FileMode(header.Mode))
+		if err != nil {
+			log.Error(err, "failed to open file", "target", target)
+			return err
+		}
+
+		if _, err := io.Copy(f, tr); err != nil {
+			log.Error(err, "failed to extract file", "target", target)
+			_ = f.Close()
+			return err
+		}
+		_ = f.Close()
+
+		// now that we've found the file can bail out
+		return nil
+	}
+}
+
 // Untar expands a tar archive into the given path.
 func Untar(ctx context.Context, r io.Reader, rootfs fs.FullFS) error {
 	log := logr.FromContextOrDiscard(ctx)

pkg/archiveutil/tar_test.go

@@ -1,14 +1,15 @@
 package archiveutil
 
 import (
-	"chainguard.dev/apko/pkg/apk/fs"
 	"context"
+	"os"
+	"testing"
+
+	"chainguard.dev/apko/pkg/apk/fs"
 	"github.com/go-logr/logr"
 	"github.com/go-logr/logr/testr"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
-	"os"
-	"testing"
 )
 
 func TestUntar(t *testing.T) {
@@ -29,3 +30,18 @@ func TestUntar(t *testing.T) {
 	_, err = rootfs.Stat("test-symbolic.txt")
 	assert.NotErrorIs(t, err, os.ErrNotExist)
 }
+
+func TestUntarFile(t *testing.T) {
+	ctx := logr.NewContext(context.TODO(), testr.NewWithOptions(t, testr.Options{Verbosity: 10}))
+
+	rootfs := fs.NewMemFS()
+
+	f, err := os.Open("./testdata/test.tar")
+	require.NoError(t, err)
+	defer f.Close()
+
+	assert.NoError(t, UntarFile(ctx, f, "test.txt", rootfs))
+
+	_, err = rootfs.Stat("test.txt")
+	assert.NotErrorIs(t, err, os.ErrNotExist)
+}

pkg/packages/alpine/alpine.go

@@ -4,28 +4,18 @@ import (
 	"context"
 	"net/http"
 	"os"
-	"path/filepath"
 
 	"chainguard.dev/apko/pkg/apk/apk"
 	"chainguard.dev/apko/pkg/apk/fs"
+	ociv1 "github.com/google/go-containerregistry/pkg/v1"
 
 	v1 "github.com/djcass44/all-your-base/pkg/api/v1"
 	"github.com/djcass44/all-your-base/pkg/archiveutil"
 	"github.com/djcass44/all-your-base/pkg/lockfile"
 	"github.com/go-logr/logr"
 )
 
-var installedFiles = []string{
-	filepath.Join("/lib", "apk", "db", "installed"),
-	filepath.Join("/usr", "lib", "apk", "db", "installed"),
-}
-
-type PackageKeeper struct {
-	rootfs  fs.FullFS
-	indices []apk.NamedIndex
-}
-
-func NewPackageKeeper(ctx context.Context, repositories []string, rootfs fs.FullFS) (*PackageKeeper, error) {
+func NewPackageKeeper(ctx context.Context, repositories []string, rootfs fs.FullFS, base ociv1.Image) (*PackageKeeper, error) {
 	log := logr.FromContextOrDiscard(ctx)
 	indices, err := apk.GetRepositoryIndexes(ctx, repositories, map[string][]byte{}, "x86_64", apk.WithIgnoreSignatures(true), apk.WithHTTPClient(http.DefaultClient))
 	if err != nil {
@@ -40,6 +30,7 @@ func NewPackageKeeper(ctx context.Context, repositories []string, rootfs fs.Full
 	return &PackageKeeper{
 		indices: indices,
 		rootfs:  rootfs,
+		base:    base,
 	}, nil
 }
 
@@ -60,19 +51,6 @@ func (*PackageKeeper) Unpack(ctx context.Context, pkg string, rootfs fs.FullFS)
 	return nil
 }
 
-func (p *PackageKeeper) Record(ctx context.Context, pkg *apk.RepositoryPackage, rootfs fs.FullFS) error {
-	log := logr.FromContextOrDiscard(ctx).WithValues("pkg", pkg.Name)
-	log.V(5).Info("recording package")
-
-	for _, i := range installedFiles {
-		if err := p.writeInstalled(ctx, i, pkg, rootfs); err != nil {
-			return err
-		}
-	}
-
-	return nil
-}
-
 func (p *PackageKeeper) Resolve(ctx context.Context, pkg string) ([]lockfile.Package, error) {
 	resolver := apk.NewPkgResolver(ctx, p.indices)
 
@@ -83,7 +61,7 @@ func (p *PackageKeeper) Resolve(ctx context.Context, pkg string) ([]lockfile.Pac
 		return nil, err
 	}
 
-	if err := p.Record(ctx, repoPkg, p.rootfs); err != nil {
+	if err := p.writeInstalled(ctx, append(repoPkgDeps, repoPkg), p.rootfs); err != nil {
 		return nil, err
 	}
 
@@ -98,9 +76,6 @@ func (p *PackageKeeper) Resolve(ctx context.Context, pkg string) ([]lockfile.Pac
 		Direct:    true,
 	}
 	for i := range repoPkgDeps {
-		if err := p.Record(ctx, repoPkgDeps[i], p.rootfs); err != nil {
-			return nil, err
-		}
 		names[i+1] = lockfile.Package{
 			Name:      repoPkgDeps[i].Name,
 			Resolved:  repoPkgDeps[i].URL(),

pkg/packages/alpine/alpine_test.go

@@ -6,6 +6,7 @@ import (
 	"testing"
 
 	"chainguard.dev/apko/pkg/apk/fs"
+	"github.com/Snakdy/container-build-engine/pkg/containers"
 	"github.com/djcass44/all-your-base/pkg/packages"
 	"github.com/go-logr/logr"
 	"github.com/go-logr/logr/testr"
@@ -31,10 +32,13 @@ func TestPackageKeeper_Unpack(t *testing.T) {
 }
 
 func TestPackageKeeper_Resolve(t *testing.T) {
+	ctx := logr.NewContext(context.TODO(), testr.NewWithOptions(t, testr.Options{Verbosity: 10}))
 	testfs := fs.NewMemFS()
 
-	ctx := logr.NewContext(context.TODO(), testr.NewWithOptions(t, testr.Options{Verbosity: 10}))
-	pkg, err := NewPackageKeeper(ctx, []string{"https://mirror.aarnet.edu.au/pub/alpine/v3.18/main"}, testfs)
+	baseImage, err := containers.Get(ctx, "harbor.dcas.dev/docker.io/library/alpine:3.23")
+	require.NoError(t, err)
+
+	pkg, err := NewPackageKeeper(ctx, []string{"https://mirror.aarnet.edu.au/pub/alpine/v3.23/main"}, testfs, baseImage)
 	require.NoError(t, err)
 
 	packageNames, err := pkg.Resolve(ctx, "git")