diff --git a/README.md b/README.md index 79229ab..3bc50ff 100644 --- a/README.md +++ b/README.md @@ -73,7 +73,18 @@ BuildKit ──► PACKET_REQ (for each regular file it needs) BuildKit ◄── PACKET_DATA (shim reads from local cache) ``` -There is also a fallback path (`Info` → `Read`) used by `FS.Open` when the local cache is unpopulated — a narrow race window at the start of a build. The host enforces the same boundary rules on this path. +There is also a fallback path (`Info` → `Read`) used by `FS.Open` when the local cache is unpopulated — a narrow race window at the start of a build in `tar` mode (see below). The host enforces the same boundary rules on this path. + +### Transfer modes + +The Walk phase supports two modes, selected by the `transfer-mode` build option (`tar`, the default, or `json`) and fixed for the lifetime of the build's `FSSyncProxy`: + +| Mode | Walk response | File content | +|---|---|---| +| **`tar`** | Host packs matching paths into a tar archive, which the shim unpacks into a content-addressed local cache. | Served from the local cache via `DiffCopy`. `FS.Open`'s `Info`/`Read` fallback is only hit in the narrow window before the cache is populated. | +| **`json`** | Host returns a single JSON array of file metadata (name, size, mode, modtime, uid/gid, symlink target) — no file bytes. | Not transferred during Walk. `FS.Open` fetches each file's content from the host on demand, per file, as BuildKit requests it — the `Info`/`Read` round-trip is the normal path in this mode, not a fallback. | + +In both modes, staged `Dockerfile`/`.dockerignore` entries (see below) are synthesized locally by the shim and injected into the file list before `.dockerignore` exclusion is applied. ### Context boundary rules @@ -91,7 +102,7 @@ These rules govern how files are selected, transferred, and presented. They refl ### `.dockerignore` -`.dockerignore` filtering is the shim's responsibility, not the host's. After unpacking the tar, the shim walks the cache directory and applies the `exclude-patterns` received from BuildKit before emitting `PACKET_STAT` entries. The host has no knowledge of `.dockerignore`. +`.dockerignore` filtering is the shim's responsibility, not the host's. The shim applies the `exclude-patterns` received from BuildKit to each file entry — walked from the unpacked tar cache in `tar` mode, or from the host's JSON metadata list in `json` mode — before emitting `PACKET_STAT` entries. The host has no knowledge of `.dockerignore`. ### `followpaths` diff --git a/pkg/build/buildopts.go b/pkg/build/buildopts.go index fb77643..e7fd0e6 100644 --- a/pkg/build/buildopts.go +++ b/pkg/build/buildopts.go @@ -65,6 +65,8 @@ const ( KeySecrets = "secrets" // SSH agent to forward during the build process. KeySSH = "ssh" + // Build context transfer mode: "tar" (default) or "json" (metadata-only, content fetched on demand). + KeyTransferMode = "transfer-mode" // Cache import sources. KeyCacheIn = "cache-in" // Cache export destinations. @@ -277,6 +279,7 @@ func NewBuildOpts(ctx context.Context, basePath string, contextMap map[string][] return nil, err } ssh := sshExtract(KeySSH) + transferMode, _ := first(KeyTransferMode) cacheIn := contextMap[KeyCacheIn] cacheOut := contextMap[KeyCacheOut] outputs := contextMap[KeyOutput] @@ -350,7 +353,7 @@ func NewBuildOpts(ctx context.Context, basePath string, contextMap map[string][] } } - fssyncProxy, err := fssync.NewFSSyncProxy(".", basePath, addedGlobs, dockerfileBytes, dockerignoreBytes) + fssyncProxy, err := fssync.NewFSSyncProxy(".", basePath, addedGlobs, dockerfileBytes, dockerignoreBytes, transferMode) if err != nil { return nil, err } diff --git a/pkg/fssync/fssync.go b/pkg/fssync/fssync.go index 820f5ce..d96060a 100644 --- a/pkg/fssync/fssync.go +++ b/pkg/fssync/fssync.go @@ -58,10 +58,12 @@ type FSSyncProxy struct { dockerfile []byte dockerignore []byte + + mode TransferMode } func NewFSSyncProxy(contextDir string, basePath string, addedGlobs []string, - dockerfile []byte, dockerignore []byte) (*FSSyncProxy, error) { + dockerfile []byte, dockerignore []byte, mode string) (*FSSyncProxy, error) { f := new(FSSyncProxy) f.contextDir = contextDir @@ -70,6 +72,13 @@ func NewFSSyncProxy(contextDir string, basePath string, addedGlobs []string, f.dockerfile = dockerfile f.dockerignore = dockerignore + + switch TransferMode(mode) { + case ModeJSON: + f.mode = ModeJSON + default: + f.mode = ModeTAR + } return f, nil } diff --git a/pkg/fssync/walk.go b/pkg/fssync/walk.go index 52290e6..a708348 100644 --- a/pkg/fssync/walk.go +++ b/pkg/fssync/walk.go @@ -18,9 +18,11 @@ package fssync import ( "context" + "encoding/json" "fmt" "io/fs" "strings" + "time" "github.com/google/uuid" "github.com/moby/patternmatcher" @@ -34,14 +36,19 @@ import ( /* Walk requests build-context files from the macOS host and presents them to BuildKit. -The host is asked for a tar archive containing the paths identified by -followpaths (glob patterns BuildKit sends in the request metadata). The shim -unpacks the tar to a content-addressed local cache and then walks the unpacked -tree, filtering each entry through the exclude-patterns (from .dockerignore) -before passing it to fn. +The mode is fixed for the lifetime of the FSSyncProxy (via the transfer-mode +build option; see pkg/build/buildopts.go) and applies to every Walk call: -Only TAR mode is supported. The JSON mode wire format is defined in -RawFileInfo below but is not exercised by the current shim. + - TAR: the host packs the paths identified by followpaths into a tar + archive. The shim unpacks it to a content-addressed local cache and walks + the unpacked tree, filtering each entry through the exclude-patterns + (from .dockerignore) before passing it to fn. File content is served from + the cache; see FS.Open. + - JSON: the host returns a single BuildTransfer whose Data is a JSON array + of RawFileInfo — metadata only, no file content. The shim filters each + entry through the same exclude-patterns and passes it to fn directly (see + receiveJSON). File content is fetched on demand later via FS.Open's + Info/Read round-trip to the host. If BuildKit does not supply followpaths, the shim falls back to addedGlobs — source paths pre-computed from the Dockerfile AST (see pkg/build/buildopts.go). @@ -60,28 +67,24 @@ Request Format: } } -Depending on the specified mode, the server may respond with file info in JSON format, -or send a tar archive for remote file data. +Response Format ('tar' mode): a tar archive streamed as one or more +BuildTransfer packets, unpacked by fileutils.TarReceiver. -Response Format ('json' mode): +Response Format ('json' mode): a single BuildTransfer whose Data is a JSON +array of RawFileInfo, e.g. - BuildTransfer { - ID: $uuid, - Direction: INTO, - Source: $path, - Metadata: { - "os": "linux", - "stage": "fssync", - "method": "Walk", - "size": "$size", - "mode": $file_mode, // uint32 value - "modified_at": "$modified_timestamp", - "uid": $uid, - "gid": $gid, - }, - "is_directory": $is_directory, - "complete": "true" - } + [ + { + "name": "some/path", + "size": 1234, + "mode": 420, + "isDir": false, + "modTime": "2026-07-31T00:00:00Z", + "uid": 0, + "gid": 0, + "target": "" + } + ] In TAR mode, the server sends a tar archive; we unpack it locally and then walk the resulting directory paths. @@ -90,10 +93,11 @@ func (f *FS) Walk(ctx context.Context, target string, fn fs.WalkDirFunc) error { cancellableCtx, cancel := context.WithCancel(ctx) defer cancel() - walkMeta, err := unmarshalWalkMetadata(cancellableCtx) + walkMeta, err := unmarshalWalkMetadata(cancellableCtx, f.proxy.mode) if err != nil { return err } + excludeMatcher, err := patternmatcher.New(strings.Split(walkMeta.ExcludedPatterns, ",")) if err != nil { return err @@ -150,11 +154,110 @@ func (f *FS) Walk(ctx context.Context, target string, fn fs.WalkDirFunc) error { defer f._checksumMutex.Unlock() f._checksum = checksum return nil + case ModeJSON: + return receiveJSON(demux, excludeMatcher, f.proxy.dockerfile, f.proxy.dockerignore, fn) default: return fmt.Errorf("unsupported walk mode: %q", walkMeta.Mode) } } +// receiveJSON handles ModeJSON walk responses. The proxy sends a single +// BuildTransfer whose Data field is a JSON array of RawFileInfo. File contents +// are not transferred here; they are fetched on-demand via FS.Open. +func receiveJSON(demux *stream.Demultiplexer, excludeMatcher *patternmatcher.PatternMatcher, dockerfile, dockerignore []byte, fn fs.WalkDirFunc) error { + resp, err := demux.Recv() + if err != nil { + return fmt.Errorf("json walk: failed receiving response: %w", err) + } + bt := resp.GetBuildTransfer() + if bt == nil { + return fmt.Errorf("json walk: expected BuildTransfer, got nil") + } + if errMsg, ok := bt.Metadata["error"]; ok { + return fmt.Errorf("json walk: server error: %s", errMsg) + } + + var files []RawFileInfo + if err := json.Unmarshal(bt.Data, &files); err != nil { + return fmt.Errorf("json walk: failed to unmarshal file list: %w", err) + } + + // Staged Dockerfile/dockerignore live under DockerfileStaging (".com.apple.container"). + // That prefix starts with '.' which sorts before any regular path component, + // so these entries must be emitted BEFORE the regular file list. + // Skip the staging dir entirely when it is covered by the exclude patterns + // (e.g. a docker-specific .dockerignore appends ".com.apple.container"); this + // mirrors the TAR-mode path where filepath.Walk hits the same exclude filter. + if len(dockerignore) > 0 { + stagingDir := DockerfileStaging + stagingExcluded, err := excludeMatcher.MatchesOrParentMatches(stagingDir) + if err != nil { + return err + } + if !stagingExcluded { + dirEntry := &fileutils.FileInfo{ + NameVal: stagingDir, + ModeVal: fs.ModeDir | 0755, + IsDirVal: true, + } + if err := fn(stagingDir, fs.FileInfoToDirEntry(dirEntry), nil); err != nil { + return err + } + for _, staged := range []struct { + name string + data []byte + }{ + {"Dockerfile", dockerfile}, + {"Dockerfile.dockerignore", dockerignore}, + } { + path := stagingDir + "/" + staged.name + fi := &fileutils.FileInfo{ + NameVal: path, + SizeVal: int64(len(staged.data)), + ModeVal: 0644, + } + if err := fn(path, fs.FileInfoToDirEntry(fi), nil); err != nil { + return err + } + } + } + } + + for _, f := range files { + excluded, err := excludeMatcher.MatchesOrParentMatches(f.Name) + if err != nil { + return err + } + if excluded { + continue + } + modTime, err := time.Parse(time.RFC3339, f.ModTime) + if err != nil { + modTime = time.Time{} + } + modeVal := fs.FileMode(f.Mode) + if f.IsDir { + modeVal |= fs.ModeDir + } else if f.Target != "" { + modeVal |= fs.ModeSymlink + } + fi := &fileutils.FileInfo{ + NameVal: f.Name, + SizeVal: int64(f.Size), + ModeVal: modeVal, + ModTimeVal: modTime, + IsDirVal: f.IsDir, + Uid: f.UID, + Gid: f.GID, + LinkName: f.Target, + } + if err := fn(f.Name, fs.FileInfoToDirEntry(fi), nil); err != nil { + return err + } + } + return nil +} + // RawFileInfo is the wire‑format for Walk (json mode). type RawFileInfo struct { Name string `json:"name"` @@ -175,23 +278,13 @@ type WalkMetadata struct { Mode TransferMode } -func unmarshalWalkMetadata(ctx context.Context) (*WalkMetadata, error) { - md := &WalkMetadata{} +func unmarshalWalkMetadata(ctx context.Context, mode TransferMode) (*WalkMetadata, error) { + md := &WalkMetadata{Mode: mode} if m, ok := metadata.FromIncomingContext(ctx); ok { md.IncludePatterns = strings.Join(m["include-patterns"], ",") md.ExcludedPatterns = strings.Join(m["exclude-patterns"], ",") md.FollowPaths = strings.Join(m["followpaths"], ",") md.DirName = strings.Join(m["dir-name"], ",") - modeStr := strings.Join(m["mode"], ",") - switch modeStr { - case "", "tar": - modeStr = string(ModeTAR) - default: - return nil, fmt.Errorf("invalid walk mode: %s", modeStr) - } - md.Mode = TransferMode(modeStr) - } else { - md.Mode = ModeTAR } return md, nil } diff --git a/pkg/fssync/walk_test.go b/pkg/fssync/walk_test.go index 8d65883..7654a18 100644 --- a/pkg/fssync/walk_test.go +++ b/pkg/fssync/walk_test.go @@ -22,13 +22,13 @@ import ( "context" "crypto/sha256" "encoding/hex" + "encoding/json" gofs "io/fs" "testing" "time" "github.com/apple/container-builder-shim/pkg/api" "github.com/apple/container-builder-shim/pkg/stream" - "google.golang.org/grpc/metadata" ) func (p *FSSyncProxy) RegisterDemux(id string, d *stream.Demultiplexer) { @@ -74,6 +74,29 @@ func makeNestedTarHeaderAndBody() (checksum string, full []byte) { func (p *FSSyncProxy) Send(s *api.ServerStream) error { id := s.BuildId d := demuxes[id] + + bt := s.GetBuildTransfer() + if bt != nil && bt.Metadata["mode"] == string(ModeJSON) { + files := []RawFileInfo{ + {Name: "dir", Mode: 0o755, IsDir: true, ModTime: time.Now().UTC().Format(time.RFC3339)}, + {Name: "dir/file.txt", Size: 42, Mode: 0o644, IsDir: false, ModTime: time.Now().UTC().Format(time.RFC3339)}, + } + data, _ := json.Marshal(files) + go func() { + _ = d.Accept(&api.ClientStream{ + BuildId: id, + PacketType: &api.ClientStream_BuildTransfer{ + BuildTransfer: &api.BuildTransfer{ + Id: id, + Complete: true, + Data: data, + }, + }, + }) + }() + return nil + } + checksum, full := makeNestedTarHeaderAndBody() go func() { _ = d.Accept(&api.ClientStream{ @@ -113,7 +136,7 @@ func (p *FSSyncProxy) Send(s *api.ServerStream) error { } func TestUnmarshalWalkMetadata_Defaults(t *testing.T) { - md, err := unmarshalWalkMetadata(context.Background()) + md, err := unmarshalWalkMetadata(context.Background(), ModeTAR) if err != nil { t.Fatalf("unexpected err: %v", err) } @@ -122,19 +145,11 @@ func TestUnmarshalWalkMetadata_Defaults(t *testing.T) { } } -func TestUnmarshalWalkMetadata_InvalidMode(t *testing.T) { - ctx := metadata.NewIncomingContext(context.Background(), metadata.Pairs("mode", "json")) - _, err := unmarshalWalkMetadata(ctx) - if err == nil { - t.Fatal("expected error for unsupported mode 'json', got nil") - } -} - func TestWalk_UnsupportedMode(t *testing.T) { - ctx := metadata.NewIncomingContext(context.Background(), metadata.Pairs("mode", "json")) - fs := NewFS(ctx, &FSSyncProxy{}, "/", t.TempDir()) // proxy never used + // A zero-value FSSyncProxy has mode="" which is not a recognised TransferMode. + fs := NewFS(context.Background(), &FSSyncProxy{}, "/", t.TempDir()) var fn gofs.WalkDirFunc = func(string, gofs.DirEntry, error) error { return nil } - err := fs.Walk(ctx, "", fn) + err := fs.Walk(context.Background(), "", fn) if err == nil { t.Fatal("Walk returned nil error, want unsupported-mode error") } @@ -145,7 +160,7 @@ func TestWalk_TarModeSuccess(t *testing.T) { _, full := makeNestedTarHeaderAndBody() - fs := NewFS(context.Background(), &FSSyncProxy{}, "/", tmp) + fs := NewFS(context.Background(), &FSSyncProxy{mode: ModeTAR}, "/", tmp) var walked []string err := fs.Walk(context.Background(), "", func(path string, _ gofs.DirEntry, _ error) error { @@ -164,3 +179,33 @@ func TestWalk_TarModeSuccess(t *testing.T) { t.Errorf("walk callback not invoked") } } + +func TestWalk_JSONModeSuccess(t *testing.T) { + fs := NewFS(context.Background(), &FSSyncProxy{mode: ModeJSON}, "/", t.TempDir()) + + type result struct { + path string + isDir bool + mode gofs.FileMode + } + var walked []result + err := fs.Walk(context.Background(), "", func(path string, d gofs.DirEntry, _ error) error { + info, _ := d.Info() + walked = append(walked, result{path: path, isDir: d.IsDir(), mode: info.Mode()}) + return nil + }) + if err != nil { + t.Fatalf("Walk returned err=%v", err) + } + if len(walked) != 2 { + t.Fatalf("got %d entries, want 2", len(walked)) + } + if walked[0].path != "dir" || !walked[0].isDir || walked[0].mode&gofs.ModeDir == 0 { + t.Errorf("entry[0]: got path=%q isDir=%v mode=%v, want dir entry with ModeDir set", + walked[0].path, walked[0].isDir, walked[0].mode) + } + if walked[1].path != "dir/file.txt" || walked[1].isDir { + t.Errorf("entry[1]: got path=%q isDir=%v, want dir/file.txt regular file", + walked[1].path, walked[1].isDir) + } +}