Make estargz compression-algorithm-agnostic and support zstd (a.k.a. zstd:chunked)

This is the subset of containerd#281.

Initially, eStargz is based on gzip compression. But, through zstd:chunked work,
it turned out that eStargz is not limited to gzip compression and the same
chunking & verifying & prefetching method can be applied to other compression
algorithms as well (e.g. zstd).

This commit makes `estargz` pkg configurable and agnostic about compression
algorithms. For supporting non-gzip compression, the user must implement
`estargz.Decompressor` and `estargz.Compressor` interfaces and must plug them to
`estargz` tools (e.g. `estargz.Open` and
`estargz.NewWriterWithCompression`). `estargz` also provides test suite that is
usable for testing these non-gzip eStargz implementations.

This commit comes with `zstdchunked` pkg that support zstd compression for
eStargz (a.k.a. zstd:chunked), based on the above extensibility. `zstdchunked`
pkg contains `zstdchunked.Decompressor` and `zstdchunked.Compressor` that allows
`estargz` pkg to use zstd compression (i.e. zstd:chunked) instead of gzip.

Layer converter and filesystem now support zstd:chunked leveraging `zstdchunked`
pkg. `ctr-remote image optimize` and `ctr-remote image convert` support
`--zstdchunked` option that omits zstd-based eStargz and filesystem supports
zstd-based eStargz layers by default.

Signed-off-by: Kohei Tokunaga <[email protected]>

Author: ktock

cmd/ctr-remote/commands/convert.go

@@ -27,7 +27,9 @@ import (
 	"github.com/containerd/containerd/images/converter/uncompress"
 	"github.com/containerd/containerd/platforms"
 	"github.com/containerd/stargz-snapshotter/estargz"
+	"github.com/containerd/stargz-snapshotter/nativeconverter"
 	estargzconvert "github.com/containerd/stargz-snapshotter/nativeconverter/estargz"
+	zstdchunkedconvert "github.com/containerd/stargz-snapshotter/nativeconverter/zstdchunked"
 	"github.com/containerd/stargz-snapshotter/recorder"
 	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
 	"github.com/pkg/errors"
@@ -66,6 +68,11 @@ When '--all-platforms' is given all images in a manifest list must be available.
 			Usage: "eStargz chunk size",
 			Value: 0,
 		},
+		// zstd:chunked flags
+		cli.BoolFlag{
+			Name:  "zstdchunked",
+			Usage: "use zstd compression instead of gzip (a.k.a zstd:chunked). Must be used in conjunction with '--oci'.",
+		},
 		// generic flags
 		cli.BoolFlag{
 			Name:  "uncompress",
@@ -96,7 +103,10 @@ When '--all-platforms' is given all images in a manifest list must be available.
 			return errors.New("src and target image need to be specified")
 		}
 
-		if !context.Bool("all-platforms") {
+		var platformMC platforms.MatchComparer
+		if context.Bool("all-platforms") {
+			platformMC = platforms.All
+		} else {
 			if pss := context.StringSlice("platform"); len(pss) > 0 {
 				var all []ocispec.Platform
 				for _, ps := range pss {
@@ -106,31 +116,57 @@ When '--all-platforms' is given all images in a manifest list must be available.
 					}
 					all = append(all, p)
 				}
-				convertOpts = append(convertOpts, converter.WithPlatform(platforms.Ordered(all...)))
+				platformMC = platforms.Ordered(all...)
 			} else {
-				convertOpts = append(convertOpts, converter.WithPlatform(platforms.DefaultStrict()))
+				platformMC = platforms.DefaultStrict()
 			}
 		}
+		convertOpts = append(convertOpts, converter.WithPlatform(platformMC))
 
+		var layerConvertFunc converter.ConvertFunc
 		if context.Bool("estargz") {
 			esgzOpts, err := getESGZConvertOpts(context)
 			if err != nil {
 				return err
 			}
-			convertOpts = append(convertOpts, converter.WithLayerConvertFunc(estargzconvert.LayerConvertFunc(esgzOpts...)))
+			layerConvertFunc = estargzconvert.LayerConvertFunc(esgzOpts...)
 			if !context.Bool("oci") {
 				logrus.Warn("option --estargz should be used in conjunction with --oci")
 			}
 			if context.Bool("uncompress") {
 				return errors.New("option --estargz conflicts with --uncompress")
 			}
+			if context.Bool("zstdchunked") {
+				return errors.New("option --estargz conflicts with --zstdchunked")
+			}
+		}
+
+		if context.Bool("zstdchunked") {
+			esgzOpts, err := getESGZConvertOpts(context)
+			if err != nil {
+				return err
+			}
+			layerConvertFunc = zstdchunkedconvert.LayerConvertFunc(esgzOpts...)
+			if !context.Bool("oci") {
+				return errors.New("option --zstdchunked must be used in conjunction with --oci")
+			}
+			if context.Bool("uncompress") {
+				return errors.New("option --zstdchunked conflicts with --uncompress")
+			}
 		}
 
 		if context.Bool("uncompress") {
-			convertOpts = append(convertOpts, converter.WithLayerConvertFunc(uncompress.LayerConvertFunc))
+			layerConvertFunc = uncompress.LayerConvertFunc
+		}
+
+		if layerConvertFunc == nil {
+			return errors.New("specify layer converter")
 		}
+		convertOpts = append(convertOpts, converter.WithLayerConvertFunc(layerConvertFunc))
 
+		var docker2oci bool
 		if context.Bool("oci") {
+			docker2oci = true
 			convertOpts = append(convertOpts, converter.WithDockerToOCI(true))
 		}
 
@@ -140,6 +176,10 @@ When '--all-platforms' is given all images in a manifest list must be available.
 		}
 		defer cancel()
 
+		convertOpts = append(convertOpts, converter.WithIndexConvertFunc(
+			// index converter patched for zstd compression
+			// TODO: upstream this to containerd/containerd
+			nativeconverter.IndexConvertFunc(layerConvertFunc, docker2oci, platformMC)))
 		newImg, err := converter.Convert(ctx, client, targetRef, srcRef, convertOpts...)
 		if err != nil {
 			return err

cmd/ctr-remote/commands/gettocdigest.go

@@ -0,0 +1,109 @@
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package commands
+
+import (
+	"encoding/json"
+	"fmt"
+	"io"
+
+	"github.com/containerd/containerd/cmd/ctr/commands"
+	"github.com/containerd/stargz-snapshotter/estargz"
+	"github.com/containerd/stargz-snapshotter/zstdchunked"
+	digest "github.com/opencontainers/go-digest"
+	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
+	"github.com/pkg/errors"
+	"github.com/urfave/cli"
+)
+
+var GetTOCDigestCommand = cli.Command{
+	Name:      "gettocdigest",
+	Usage:     "get the digest of TOC of a layer",
+	ArgsUsage: "<layer digest>",
+	Flags: []cli.Flag{
+		// zstd:chunked flags
+		cli.BoolFlag{
+			Name:  "zstdchunked",
+			Usage: "parse layer as zstd:chunked",
+		},
+		// other flags for debugging
+		cli.BoolFlag{
+			Name:  "dump-toc",
+			Usage: "dump TOC instead of digest. Note that the dumped TOC might be formatted with indents so may have different digest against the original in the layer",
+		},
+	},
+	Action: func(clicontext *cli.Context) error {
+		layerDgstStr := clicontext.Args().Get(0)
+		if layerDgstStr == "" {
+			return errors.New("layer digest need to be specified")
+		}
+
+		client, ctx, cancel, err := commands.NewClient(clicontext)
+		if err != nil {
+			return err
+		}
+		defer cancel()
+
+		layerDgst, err := digest.Parse(layerDgstStr)
+		if err != nil {
+			return err
+		}
+		ra, err := client.ContentStore().ReaderAt(ctx, ocispec.Descriptor{Digest: layerDgst})
+		if err != nil {
+			return err
+		}
+		defer ra.Close()
+
+		footerSize := estargz.FooterSize
+		if clicontext.Bool("zstdchunked") {
+			footerSize = zstdchunked.FooterSize
+		}
+		footer := make([]byte, footerSize)
+		if _, err := ra.ReadAt(footer, ra.Size()-int64(footerSize)); err != nil {
+			return errors.Wrapf(err, "error reading footer")
+		}
+
+		var decompressor estargz.Decompressor
+		decompressor = new(estargz.GzipDecompressor)
+		if clicontext.Bool("zstdchunked") {
+			decompressor = new(zstdchunked.Decompressor)
+		}
+
+		tocOff, tocSize, err := decompressor.ParseFooter(footer)
+		if err != nil {
+			return errors.Wrapf(err, "error parsing footer")
+		}
+		if tocSize <= 0 {
+			tocSize = ra.Size() - tocOff - int64(footerSize)
+		}
+		toc, tocDgst, err := decompressor.ParseTOC(io.NewSectionReader(ra, tocOff, tocSize))
+		if err != nil {
+			return errors.Wrapf(err, "error parsing TOC")
+		}
+
+		if clicontext.Bool("dump-toc") {
+			tocJSON, err := json.MarshalIndent(toc, "", "\t")
+			if err != nil {
+				return errors.Wrapf(err, "failed to marshal toc")
+			}
+			fmt.Println(string(tocJSON))
+			return nil
+		}
+		fmt.Println(tocDgst.String())
+		return nil
+	},
+}

cmd/ctr-remote/commands/optimize.go

@@ -31,7 +31,9 @@ import (
 	"github.com/containerd/containerd/platforms"
 	"github.com/containerd/stargz-snapshotter/analyzer"
 	"github.com/containerd/stargz-snapshotter/estargz"
+	"github.com/containerd/stargz-snapshotter/nativeconverter"
 	estargzconvert "github.com/containerd/stargz-snapshotter/nativeconverter/estargz"
+	zstdchunkedconvert "github.com/containerd/stargz-snapshotter/nativeconverter/zstdchunked"
 	"github.com/containerd/stargz-snapshotter/recorder"
 	"github.com/containerd/stargz-snapshotter/util/containerdutil"
 	"github.com/opencontainers/go-digest"
@@ -77,6 +79,10 @@ var OptimizeCommand = cli.Command{
 			Name:  "oci",
 			Usage: "convert Docker media types to OCI media types",
 		},
+		cli.BoolFlag{
+			Name:  "zstdchunked",
+			Usage: "use zstd compression instead of gzip (a.k.a zstd:chunked)",
+		},
 	}, samplerFlags...),
 	Action: func(clicontext *cli.Context) error {
 		convertOpts := []converter.Opt{}
@@ -86,7 +92,10 @@ var OptimizeCommand = cli.Command{
 			return errors.New("src and target image need to be specified")
 		}
 
-		if !clicontext.Bool("all-platforms") {
+		var platformMC platforms.MatchComparer
+		if clicontext.Bool("all-platforms") {
+			platformMC = platforms.All
+		} else {
 			if pss := clicontext.StringSlice("platform"); len(pss) > 0 {
 				var all []ocispec.Platform
 				for _, ps := range pss {
@@ -96,15 +105,21 @@ var OptimizeCommand = cli.Command{
 					}
 					all = append(all, p)
 				}
-				convertOpts = append(convertOpts, converter.WithPlatform(platforms.Ordered(all...)))
+				platformMC = platforms.Ordered(all...)
 			} else {
-				convertOpts = append(convertOpts, converter.WithPlatform(platforms.DefaultStrict()))
+				platformMC = platforms.DefaultStrict()
 			}
 		}
+		convertOpts = append(convertOpts, converter.WithPlatform(platformMC))
 
+		var docker2oci bool
 		if clicontext.Bool("oci") {
+			docker2oci = true
 			convertOpts = append(convertOpts, converter.WithDockerToOCI(true))
 		} else {
+			if clicontext.Bool("zstdchunked") {
+				return errors.New("option --zstdchunked must be used in conjunction with --oci")
+			}
 			logrus.Warn("option --oci should be used as well")
 		}
 
@@ -129,12 +144,21 @@ var OptimizeCommand = cli.Command{
 				return errors.Wrapf(err, "failed output record file")
 			}
 		}
-		f := estargzconvert.LayerConvertWithLayerOptsFunc(esgzOptsPerLayer)
+		var f converter.ConvertFunc
+		if clicontext.Bool("zstdchunked") {
+			f = zstdchunkedconvert.LayerConvertWithLayerOptsFunc(esgzOptsPerLayer)
+		} else {
+			f = estargzconvert.LayerConvertWithLayerOptsFunc(esgzOptsPerLayer)
+		}
 		if wrapper != nil {
 			f = wrapper(f)
 		}
-		convertOpts = append(convertOpts, converter.WithLayerConvertFunc(logWrapper(f)))
-
+		layerConvertFunc := logWrapper(f)
+		convertOpts = append(convertOpts, converter.WithLayerConvertFunc(layerConvertFunc))
+		convertOpts = append(convertOpts, converter.WithIndexConvertFunc(
+			// index converter patched for zstd compression
+			// TODO: upstream this to containerd/containerd
+			nativeconverter.IndexConvertFunc(layerConvertFunc, docker2oci, platformMC)))
 		newImg, err := converter.Convert(ctx, client, targetRef, srcRef, convertOpts...)
 		if err != nil {
 			return err

cmd/ctr-remote/main.go

@@ -31,7 +31,12 @@ func init() {
 }
 
 func main() {
-	customCommands := []cli.Command{commands.RpullCommand, commands.OptimizeCommand, commands.ConvertCommand}
+	customCommands := []cli.Command{
+		commands.RpullCommand,
+		commands.OptimizeCommand,
+		commands.ConvertCommand,
+		commands.GetTOCDigestCommand,
+	}
 	app := app.New()
 	for i := range app.Commands {
 		if app.Commands[i].Name == "images" {