Skip to content

Support concatenated gzip (mgzip/pigz) layers in ztoc #1871

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
prafgup wants to merge 1 commit into
base: main
Choose a base branch
from
+389 −7
Open

Support concatenated gzip (mgzip/pigz) layers in ztoc #1871

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
125 changes: 125 additions & 0 deletions integration/util_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,7 @@
package integration

import (
"archive/tar"
"bytes"
"crypto/rand"
"crypto/rsa"
Expand All @@ -43,8 +44,10 @@ import (
"encoding/pem"
"errors"
"fmt"
"io"
"math/big"
"os"
"os/exec"
"path/filepath"
"runtime"
"sort"
Expand Down Expand Up @@ -1040,3 +1043,125 @@ func withContentStoreConfig(opts ...store.Option) snapshotterConfigOpt {
c.ServiceConfig.FSConfig.ContentStoreConfig = store.NewStoreConfig(opts...).ContentStoreConfig
}
}

type pigzImageInfo struct {
ref string
files map[string]string
layerCount int
}

// buildPigzImage constructs a minimal OCI image with a single pigz-compressed layer
// and imports it into containerd via "ctr images import"
func buildPigzImage(t *testing.T, sh *shell.Shell, imageName string) pigzImageInfo {
t.Helper()

pigzPath, err := exec.LookPath("pigz")
if err != nil {
t.Fatal("pigz is required but not installed")
}

r := testutil.NewTestRand(t)
testFiles := map[string]string{
"testfile1.txt": "pigz-test-content-alpha-" + string(r.RandomByteData(1<<20)), // ~1 MB
"testfile2.txt": "pigz-test-content-beta-" + string(r.RandomByteData(1<<20)), // ~1 MB
}

entries := []testutil.TarEntry{
testutil.File("testfile1.txt", testFiles["testfile1.txt"]),
testutil.File("padding1.bin", string(r.RandomByteData(1<<23))), // 8 MB
testutil.File("testfile2.txt", testFiles["testfile2.txt"]),
testutil.File("padding2.bin", string(r.RandomByteData(1<<23))), // 8 MB
}
tarData, err := io.ReadAll(testutil.BuildTar(entries))
if err != nil {
t.Fatalf("failed to build tar: %v", err)
}

// Compress with pigz using 128KB block size (produces concatenated gzip members).
pigzCmd := exec.Command(pigzPath, "-b", "128", "-c")
pigzCmd.Stdin = bytes.NewReader(tarData)
compressedData, err := pigzCmd.Output()
if err != nil {
t.Fatalf("pigz compression failed: %v", err)
}
layerDigest := digest.FromBytes(compressedData)

platform := spec.Platform{Architecture: runtime.GOARCH, OS: "linux"}

// Build OCI image config.
configBytes, err := json.Marshal(spec.Image{
Platform: platform,
RootFS: spec.RootFS{Type: "layers", DiffIDs: []digest.Digest{digest.FromBytes(tarData)}},
})
if err != nil {
t.Fatalf("failed to marshal config: %v", err)
}

configDigest := digest.FromBytes(configBytes)
manifest := spec.Manifest{
MediaType: spec.MediaTypeImageManifest,
Config: spec.Descriptor{MediaType: spec.MediaTypeImageConfig, Digest: configDigest, Size: int64(len(configBytes))},
Layers: []spec.Descriptor{{MediaType: spec.MediaTypeImageLayerGzip, Digest: layerDigest, Size: int64(len(compressedData))}},
}
manifest.SchemaVersion = 2
manifestBytes, err := json.Marshal(manifest)
if err != nil {
t.Fatalf("failed to marshal manifest: %v", err)
}
manifestDigest := digest.FromBytes(manifestBytes)

// Build OCI index.
index := spec.Index{
MediaType: spec.MediaTypeImageIndex,
Manifests: []spec.Descriptor{
{
MediaType: spec.MediaTypeImageManifest,
Digest: manifestDigest,
Size: int64(len(manifestBytes)),
Platform: &platform,
Annotations: map[string]string{"io.containerd.image.name": imageName},
},
},
}
index.SchemaVersion = 2
indexBytes, err := json.Marshal(index)
if err != nil {
t.Fatalf("failed to marshal index: %v", err)
}

// Assemble OCI image layout as a tar archive.
var buf bytes.Buffer
tw := tar.NewWriter(&buf)
for name, data := range map[string][]byte{
"oci-layout": []byte(`{"imageLayoutVersion":"1.0.0"}`),
"index.json": indexBytes,
"blobs/sha256/" + layerDigest.Encoded(): compressedData,
"blobs/sha256/" + configDigest.Encoded(): configBytes,
"blobs/sha256/" + manifestDigest.Encoded(): manifestBytes,
} {
if err := tw.WriteHeader(&tar.Header{Name: name, Mode: 0644, Size: int64(len(data))}); err != nil {
t.Fatalf("failed to write tar header for %s: %v", name, err)
}
if _, err := tw.Write(data); err != nil {
t.Fatalf("failed to write tar data for %s: %v", name, err)
}
}
if err := tw.Close(); err != nil {
t.Fatalf("failed to close tar writer: %v", err)
}
ociTar := buf.Bytes()

// Write OCI tar to test container and import into containerd.
tmpPath := "/tmp/pigz-image-" + xid.New().String() + ".tar"
if err := testutil.WriteFileContents(sh, tmpPath, ociTar, 0644); err != nil {
t.Fatalf("failed to write OCI tar to container: %v", err)
}
sh.X("ctr", "images", "import", tmpPath)
sh.X("rm", "-f", tmpPath)

return pigzImageInfo{
ref: imageName,
files: testFiles,
layerCount: 1,
}
}
46 changes: 46 additions & 0 deletions integration/ztoc_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -474,6 +474,52 @@ func dedupeZtocBlobs(ztocBlobs []*v1.Descriptor) []*v1.Descriptor {
return dedupedZtocBlobs
}

func TestSociZtocWithPigzLayers(t *testing.T) {
t.Parallel()

regConfig := newRegistryConfig()
sh, done := newShellWithRegistry(t, regConfig)
defer done()

rebootContainerd(t, sh, getContainerdConfigToml(t, false), getSnapshotterConfigToml(t))

pigzImg := buildPigzImage(t, sh, "pigz-test:latest")
mirrorRef := regConfig.mirror("pigz-test:latest")
sh.X("ctr", "i", "tag", pigzImg.ref, mirrorRef.ref)
sh.X(append([]string{"nerdctl", "push", "-q"}, encodeImageInfoNerdctl(mirrorRef)[0]...)...)

indexDigest := buildIndex(sh, mirrorRef, withMinLayerSize(0))
if indexDigest == "" {
t.Fatal("failed to create SOCI index for pigz-compressed image")
}

// Verify a ztoc was created for the pigz layer and extract files to check content
sociIndex, err := sociIndexFromDigest(sh, indexDigest)
if err != nil {
t.Fatalf("failed to read SOCI index: %v", err)
}
for _, blob := range sociIndex.Blobs {
if blob.MediaType != soci.SociLayerMediaType {
continue
}
for fileName, expectedContent := range pigzImg.files {
output, err := sh.OLog("soci", "ztoc", "get-file", blob.Digest.String(), fileName)
if err != nil {
t.Fatalf("soci ztoc get-file failed for %s: %v", fileName, err)
}
actual := strings.TrimRight(string(output), "\n")
if actual != expectedContent {
t.Fatalf("file %s content mismatch: expected %q, got %q", fileName, expectedContent, actual)
}
}
}

sh.X("soci", "push", "--user", regConfig.creds(), mirrorRef.ref)
sh.X("ctr", "i", "rm", mirrorRef.ref)
sh.X(append(imagePullCmd, "--soci-index-digest", indexDigest, mirrorRef.ref)...)
checkFuseMounts(t, sh, pigzImg.layerCount)
}

func verifyInfoOutput(zinfo Info, ztoc *ztoc.Ztoc) error {
if zinfo.Version != string(ztoc.Version) {
return fmt.Errorf("different versions: expected %s got %s", ztoc.Version, zinfo.Version)
Expand Down
94 changes: 87 additions & 7 deletions ztoc/compression/gzip_zinfo.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -48,7 +48,8 @@
#include <stdlib.h>
#include <string.h>

#define CHUNK (1 << 14) // file input buffer size
#define CHUNK (1 << 14) // file input buffer size
#define GZIP_TRAILER_SIZE 8 // gzip trailer: 4-byte CRC32 + 4-byte ISIZE


// zinfo - internal helpers start.
Expand Down Expand Up @@ -253,8 +254,19 @@ int generate_zinfo_from_fp(FILE* in, offset_t span, struct gzip_zinfo** idx) {
ret = Z_DATA_ERROR;
if (ret == Z_MEM_ERROR || ret == Z_DATA_ERROR)
goto build_index_error;
if (ret == Z_STREAM_END)
if (ret == Z_STREAM_END) {
/* Handle concatenated gzip streams (e.g., mgzip/pigz).
If there's more data, reset inflate for the next member. */
if (strm.avail_in > 0 ||
ungetc(getc(in), in) != EOF) {
ret = inflateReset2(&strm, 47);
if (ret != Z_OK)
goto build_index_error;
if (strm.avail_in > 0)
continue;
}
break;
}

/* if at end of block, consider adding an index entry (note that if
data_type indicates an end-of-block, then all of the
Expand Down Expand Up @@ -308,10 +320,11 @@ int generate_zinfo_from_file(const char *filepath, offset_t span, struct gzip_zi

int extract_data_from_fp(FILE *in, struct gzip_zinfo *index, offset_t offset, void *buffer, int len) {
int ret, skip;
int raw_mode = 1; // track if we're in initial raw inflate mode
z_stream strm;
struct gzip_checkpoint *here;
unsigned char input[CHUNK], discard[WINSIZE];
uchar* buf = buffer;
uchar* buf = buffer;

/* proceed only if something reasonable to do */
if (len < 0)
Expand Down Expand Up @@ -379,8 +392,43 @@ int extract_data_from_fp(FILE *in, struct gzip_zinfo *index, offset_t offset, vo
ret = Z_DATA_ERROR;
if (ret == Z_MEM_ERROR || ret == Z_DATA_ERROR)
goto extract_ret;
if (ret == Z_STREAM_END)
if (ret == Z_STREAM_END) {
/* Handle concatenated gzip member boundary */
if (skip || strm.avail_out > 0) {
if (raw_mode) {
/* Skip the 8-byte gzip trailer (CRC32 + ISIZE).
In raw inflate mode (-15), the trailer is NOT
consumed by inflate. After resetting to gzip mode
(47), subsequent trailers are consumed automatically,
so this is only needed once. */
unsigned drop = GZIP_TRAILER_SIZE;
if (strm.avail_in >= drop) {
strm.avail_in -= drop;
strm.next_in += drop;
} else {
drop -= strm.avail_in;
strm.avail_in = 0;
do {
if (getc(in) == EOF) {
ret = ferror(in) ? Z_ERRNO : Z_BUF_ERROR;
goto extract_ret;
}
} while (--drop);
}
raw_mode = 0;
}
if (strm.avail_in > 0 ||
ungetc(getc(in), in) != EOF) {
ret = inflateReset2(&strm, 47);
if (ret != Z_OK)
goto extract_ret;
if (strm.avail_out > 0)
continue;
break;
}
}
break;
}
} while (strm.avail_out != 0);

/* if reach end of stream, then don't keep trying to get more */
Expand Down Expand Up @@ -414,6 +462,7 @@ int extract_data_from_buffer(void *d, offset_t datalen,
struct gzip_zinfo *index, offset_t offset,
void *buffer, offset_t len, int first_checkpoint) {
int ret, skip;
int raw_mode = 1; // track if we're in initial raw inflate mode
z_stream strm;
unsigned char input[CHUNK], discard[WINSIZE];
uchar *buf = buffer;
Expand All @@ -430,8 +479,8 @@ int extract_data_from_buffer(void *d, offset_t datalen,
return ret;

if (bits) {
int ret = data[0];
inflatePrime(&strm, bits, ret >> (8 - bits));
int byte_val = data[0];
inflatePrime(&strm, bits, byte_val >> (8 - bits));
data++;
}
(void)inflateSetDictionary(&strm, index->list[first_checkpoint].window,
Expand Down Expand Up @@ -471,8 +520,39 @@ int extract_data_from_buffer(void *d, offset_t datalen,
ret = Z_DATA_ERROR;
if (ret == Z_MEM_ERROR || ret == Z_DATA_ERROR)
goto extract_ret;
if (ret == Z_STREAM_END)
if (ret == Z_STREAM_END) {
Comment thread
Shubhranshu153 marked this conversation as resolved.
/* Handle concatenated gzip member boundary */
if (skip || strm.avail_out > 0) {
if (raw_mode) {
/* Skip the 8-byte gzip trailer (CRC32 + ISIZE).
See comment in extract_data_from_fp. */
unsigned drop = GZIP_TRAILER_SIZE;
if (strm.avail_in >= drop) {
strm.avail_in -= drop;
strm.next_in += drop;
} else {
drop -= strm.avail_in;
strm.avail_in = 0;
if (remaining < (int)drop) {
ret = Z_BUF_ERROR;
goto extract_ret;
}
data += drop;
remaining -= drop;
}
raw_mode = 0;
}
if (strm.avail_in > 0 || remaining > 0) {
ret = inflateReset2(&strm, 47);
if (ret != Z_OK)
goto extract_ret;
if (strm.avail_out > 0)
continue;
break; // let outer loop set up next buffer
}
}
break;
}
} while (strm.avail_out != 0);

/* if reach end of stream, then don't keep trying to get more */
Expand Down
Loading