Support json mode build context transfer

README.md

@@ -73,7 +73,18 @@ BuildKit ──► PACKET_REQ             (for each regular file it needs)
 BuildKit ◄── PACKET_DATA            (shim reads from local cache)
 ```
 
-There is also a fallback path (`Info` → `Read`) used by `FS.Open` when the local cache is unpopulated — a narrow race window at the start of a build. The host enforces the same boundary rules on this path.
+There is also a fallback path (`Info` → `Read`) used by `FS.Open` when the local cache is unpopulated — a narrow race window at the start of a build in `tar` mode (see below). The host enforces the same boundary rules on this path.
+
+### Transfer modes
+
+The Walk phase supports two modes, selected by the `transfer-mode` build option (`tar`, the default, or `json`) and fixed for the lifetime of the build's `FSSyncProxy`:
+
+| Mode | Walk response | File content |
+|---|---|---|
+| **`tar`** | Host packs matching paths into a tar archive, which the shim unpacks into a content-addressed local cache. | Served from the local cache via `DiffCopy`. `FS.Open`'s `Info`/`Read` fallback is only hit in the narrow window before the cache is populated. |
+| **`json`** | Host returns a single JSON array of file metadata (name, size, mode, modtime, uid/gid, symlink target) — no file bytes. | Not transferred during Walk. `FS.Open` fetches each file's content from the host on demand, per file, as BuildKit requests it — the `Info`/`Read` round-trip is the normal path in this mode, not a fallback. |
+
+In both modes, staged `Dockerfile`/`.dockerignore` entries (see below) are synthesized locally by the shim and injected into the file list before `.dockerignore` exclusion is applied.
 
 ### Context boundary rules
 
@@ -91,7 +102,7 @@ These rules govern how files are selected, transferred, and presented. They refl
 
 ### `.dockerignore`
 
-`.dockerignore` filtering is the shim's responsibility, not the host's. After unpacking the tar, the shim walks the cache directory and applies the `exclude-patterns` received from BuildKit before emitting `PACKET_STAT` entries. The host has no knowledge of `.dockerignore`.
+`.dockerignore` filtering is the shim's responsibility, not the host's. The shim applies the `exclude-patterns` received from BuildKit to each file entry — walked from the unpacked tar cache in `tar` mode, or from the host's JSON metadata list in `json` mode — before emitting `PACKET_STAT` entries. The host has no knowledge of `.dockerignore`.
 
 ### `followpaths`
 

pkg/build/buildopts.go

@@ -65,6 +65,8 @@ const (
 	KeySecrets = "secrets"
 	// SSH agent to forward during the build process.
 	KeySSH = "ssh"
+	// Build context transfer mode: "tar" (default) or "json" (metadata-only, content fetched on demand).
+	KeyTransferMode = "transfer-mode"
 	// Cache import sources.
 	KeyCacheIn = "cache-in"
 	// Cache export destinations.
@@ -277,6 +279,7 @@ func NewBuildOpts(ctx context.Context, basePath string, contextMap map[string][]
 		return nil, err
 	}
 	ssh := sshExtract(KeySSH)
+	transferMode, _ := first(KeyTransferMode)
 	cacheIn := contextMap[KeyCacheIn]
 	cacheOut := contextMap[KeyCacheOut]
 	outputs := contextMap[KeyOutput]
@@ -350,7 +353,7 @@ func NewBuildOpts(ctx context.Context, basePath string, contextMap map[string][]
 		}
 	}
 
-	fssyncProxy, err := fssync.NewFSSyncProxy(".", basePath, addedGlobs, dockerfileBytes, dockerignoreBytes)
+	fssyncProxy, err := fssync.NewFSSyncProxy(".", basePath, addedGlobs, dockerfileBytes, dockerignoreBytes, transferMode)
 	if err != nil {
 		return nil, err
 	}

pkg/fssync/fssync.go

@@ -58,10 +58,12 @@ type FSSyncProxy struct {
 
 	dockerfile   []byte
 	dockerignore []byte
+
+	mode TransferMode
 }
 
 func NewFSSyncProxy(contextDir string, basePath string, addedGlobs []string,
-	dockerfile []byte, dockerignore []byte) (*FSSyncProxy, error) {
+	dockerfile []byte, dockerignore []byte, mode string) (*FSSyncProxy, error) {
 
 	f := new(FSSyncProxy)
 	f.contextDir = contextDir
@@ -70,6 +72,13 @@ func NewFSSyncProxy(contextDir string, basePath string, addedGlobs []string,
 
 	f.dockerfile = dockerfile
 	f.dockerignore = dockerignore
+
+	switch TransferMode(mode) {
+	case ModeJSON:
+		f.mode = ModeJSON
+	default:
+		f.mode = ModeTAR
+	}
 	return f, nil
 }
 

pkg/fssync/walk.go

@@ -18,9 +18,11 @@ package fssync
 
 import (
 	"context"
+	"encoding/json"
 	"fmt"
 	"io/fs"
 	"strings"
+	"time"
 
 	"github.com/google/uuid"
 	"github.com/moby/patternmatcher"
@@ -34,14 +36,19 @@ import (
 /*
 Walk requests build-context files from the macOS host and presents them to BuildKit.
 
-The host is asked for a tar archive containing the paths identified by
-followpaths (glob patterns BuildKit sends in the request metadata). The shim
-unpacks the tar to a content-addressed local cache and then walks the unpacked
-tree, filtering each entry through the exclude-patterns (from .dockerignore)
-before passing it to fn.
+The mode is fixed for the lifetime of the FSSyncProxy (via the transfer-mode
+build option; see pkg/build/buildopts.go) and applies to every Walk call:
 
-Only TAR mode is supported. The JSON mode wire format is defined in
-RawFileInfo below but is not exercised by the current shim.
+  - TAR: the host packs the paths identified by followpaths into a tar
+    archive. The shim unpacks it to a content-addressed local cache and walks
+    the unpacked tree, filtering each entry through the exclude-patterns
+    (from .dockerignore) before passing it to fn. File content is served from
+    the cache; see FS.Open.
+  - JSON: the host returns a single BuildTransfer whose Data is a JSON array
+    of RawFileInfo — metadata only, no file content. The shim filters each
+    entry through the same exclude-patterns and passes it to fn directly (see
+    receiveJSON). File content is fetched on demand later via FS.Open's
+    Info/Read round-trip to the host.
 
 If BuildKit does not supply followpaths, the shim falls back to addedGlobs —
 source paths pre-computed from the Dockerfile AST (see pkg/build/buildopts.go).
@@ -60,28 +67,24 @@ Request Format:
 	    }
 	}
 
-Depending on the specified mode, the server may respond with file info in JSON format,
-or send a tar archive for remote file data.
+Response Format ('tar' mode): a tar archive streamed as one or more
+BuildTransfer packets, unpacked by fileutils.TarReceiver.
 
-Response Format ('json' mode):
+Response Format ('json' mode): a single BuildTransfer whose Data is a JSON
+array of RawFileInfo, e.g.
 
-	BuildTransfer {
-	    ID: $uuid,
-	    Direction: INTO,
-	    Source: $path,
-	    Metadata: {
-	        "os":          "linux",
-	        "stage":       "fssync",
-	        "method":      "Walk",
-	        "size":        "$size",
-	        "mode":        $file_mode, // uint32 value
-	        "modified_at": "$modified_timestamp",
-	        "uid":         $uid,
-	        "gid":         $gid,
-	    },
-	    "is_directory": $is_directory,
-	    "complete":     "true"
-	}
+	[
+	    {
+	        "name":    "some/path",
+	        "size":    1234,
+	        "mode":    420,
+	        "isDir":   false,
+	        "modTime": "2026-07-31T00:00:00Z",
+	        "uid":     0,
+	        "gid":     0,
+	        "target":  ""
+	    }
+	]
 
 In TAR mode, the server sends a tar archive; we unpack it locally and then walk
 the resulting directory paths.
@@ -90,10 +93,11 @@ func (f *FS) Walk(ctx context.Context, target string, fn fs.WalkDirFunc) error {
 	cancellableCtx, cancel := context.WithCancel(ctx)
 	defer cancel()
 
-	walkMeta, err := unmarshalWalkMetadata(cancellableCtx)
+	walkMeta, err := unmarshalWalkMetadata(cancellableCtx, f.proxy.mode)
 	if err != nil {
 		return err
 	}
+
 	excludeMatcher, err := patternmatcher.New(strings.Split(walkMeta.ExcludedPatterns, ","))
 	if err != nil {
 		return err
@@ -150,11 +154,110 @@ func (f *FS) Walk(ctx context.Context, target string, fn fs.WalkDirFunc) error {
 		defer f._checksumMutex.Unlock()
 		f._checksum = checksum
 		return nil
+	case ModeJSON:
+		return receiveJSON(demux, excludeMatcher, f.proxy.dockerfile, f.proxy.dockerignore, fn)
 	default:
 		return fmt.Errorf("unsupported walk mode: %q", walkMeta.Mode)
 	}
 }
 
+// receiveJSON handles ModeJSON walk responses. The proxy sends a single
+// BuildTransfer whose Data field is a JSON array of RawFileInfo. File contents
+// are not transferred here; they are fetched on-demand via FS.Open.
+func receiveJSON(demux *stream.Demultiplexer, excludeMatcher *patternmatcher.PatternMatcher, dockerfile, dockerignore []byte, fn fs.WalkDirFunc) error {
+	resp, err := demux.Recv()
+	if err != nil {
+		return fmt.Errorf("json walk: failed receiving response: %w", err)
+	}
+	bt := resp.GetBuildTransfer()
+	if bt == nil {
+		return fmt.Errorf("json walk: expected BuildTransfer, got nil")
+	}
+	if errMsg, ok := bt.Metadata["error"]; ok {
+		return fmt.Errorf("json walk: server error: %s", errMsg)
+	}
+
+	var files []RawFileInfo
+	if err := json.Unmarshal(bt.Data, &files); err != nil {
+		return fmt.Errorf("json walk: failed to unmarshal file list: %w", err)
+	}
+
+	// Staged Dockerfile/dockerignore live under DockerfileStaging (".com.apple.container").
+	// That prefix starts with '.' which sorts before any regular path component,
+	// so these entries must be emitted BEFORE the regular file list.
+	// Skip the staging dir entirely when it is covered by the exclude patterns
+	// (e.g. a docker-specific .dockerignore appends ".com.apple.container"); this
+	// mirrors the TAR-mode path where filepath.Walk hits the same exclude filter.
+	if len(dockerignore) > 0 {
+		stagingDir := DockerfileStaging
+		stagingExcluded, err := excludeMatcher.MatchesOrParentMatches(stagingDir)
+		if err != nil {
+			return err
+		}
+		if !stagingExcluded {
+			dirEntry := &fileutils.FileInfo{
+				NameVal:  stagingDir,
+				ModeVal:  fs.ModeDir | 0755,
+				IsDirVal: true,
+			}
+			if err := fn(stagingDir, fs.FileInfoToDirEntry(dirEntry), nil); err != nil {
+				return err
+			}
+			for _, staged := range []struct {
+				name string
+				data []byte
+			}{
+				{"Dockerfile", dockerfile},
+				{"Dockerfile.dockerignore", dockerignore},
+			} {
+				path := stagingDir + "/" + staged.name
+				fi := &fileutils.FileInfo{
+					NameVal: path,
+					SizeVal: int64(len(staged.data)),
+					ModeVal: 0644,
+				}
+				if err := fn(path, fs.FileInfoToDirEntry(fi), nil); err != nil {
+					return err
+				}
+			}
+		}
+	}
+
+	for _, f := range files {
+		excluded, err := excludeMatcher.MatchesOrParentMatches(f.Name)
+		if err != nil {
+			return err
+		}
+		if excluded {
+			continue
+		}
+		modTime, err := time.Parse(time.RFC3339, f.ModTime)
+		if err != nil {
+			modTime = time.Time{}
+		}
+		modeVal := fs.FileMode(f.Mode)
+		if f.IsDir {
+			modeVal |= fs.ModeDir
+		} else if f.Target != "" {
+			modeVal |= fs.ModeSymlink
+		}
+		fi := &fileutils.FileInfo{
+			NameVal:    f.Name,
+			SizeVal:    int64(f.Size),
+			ModeVal:    modeVal,
+			ModTimeVal: modTime,
+			IsDirVal:   f.IsDir,
+			Uid:        f.UID,
+			Gid:        f.GID,
+			LinkName:   f.Target,
+		}
+		if err := fn(f.Name, fs.FileInfoToDirEntry(fi), nil); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
 // RawFileInfo is the wire‑format for Walk (json mode).
 type RawFileInfo struct {
 	Name    string `json:"name"`
@@ -175,23 +278,13 @@ type WalkMetadata struct {
 	Mode             TransferMode
 }
 
-func unmarshalWalkMetadata(ctx context.Context) (*WalkMetadata, error) {
-	md := &WalkMetadata{}
+func unmarshalWalkMetadata(ctx context.Context, mode TransferMode) (*WalkMetadata, error) {
+	md := &WalkMetadata{Mode: mode}
 	if m, ok := metadata.FromIncomingContext(ctx); ok {
 		md.IncludePatterns = strings.Join(m["include-patterns"], ",")
 		md.ExcludedPatterns = strings.Join(m["exclude-patterns"], ",")
 		md.FollowPaths = strings.Join(m["followpaths"], ",")
 		md.DirName = strings.Join(m["dir-name"], ",")
-		modeStr := strings.Join(m["mode"], ",")
-		switch modeStr {
-		case "", "tar":
-			modeStr = string(ModeTAR)
-		default:
-			return nil, fmt.Errorf("invalid walk mode: %s", modeStr)
-		}
-		md.Mode = TransferMode(modeStr)
-	} else {
-		md.Mode = ModeTAR
 	}
 	return md, nil
 }