Merge upstream main into container clean command

Author: recrack

Package.resolved

@@ -22,8 +22,8 @@ import PackageDescription
 
 let releaseVersion = ProcessInfo.processInfo.environment["RELEASE_VERSION"] ?? "0.0.0"
 let gitCommit = ProcessInfo.processInfo.environment["GIT_COMMIT"] ?? "unspecified"
-let builderShimVersion = "0.13.0"
-let scVersion = "0.38.0"
+let builderShimVersion = "0.13.1"
+let scVersion = "0.40.1"
 
 let package = Package(
     name: "container",
@@ -143,6 +143,7 @@ let package = Package(
                 .product(name: "Containerization", package: "containerization"),
                 .product(name: "ContainerizationArchive", package: "containerization"),
                 .product(name: "ContainerizationOCI", package: "containerization"),
+                .product(name: "ContainerizationOS", package: "containerization"),
                 .product(name: "ArgumentParser", package: "swift-argument-parser"),
                 .product(name: "GRPCCore", package: "grpc-swift-2"),
                 .product(name: "GRPCNIOTransportHTTP2", package: "grpc-swift-nio-transport"),
@@ -574,12 +575,18 @@ let package = Package(
             name: "ContainerTestSupport",
             dependencies: [
                 .product(name: "AsyncHTTPClient", package: "async-http-client"),
+                .product(name: "Containerization", package: "containerization"),
+                .product(name: "ContainerizationArchive", package: "containerization"),
                 .product(name: "ContainerizationExtras", package: "containerization"),
                 .product(name: "Logging", package: "swift-log"),
+                .product(name: "NIOCore", package: "swift-nio"),
+                .product(name: "NIOHTTP1", package: "swift-nio"),
+                .product(name: "NIOPosix", package: "swift-nio"),
                 .product(name: "SystemPackage", package: "swift-system"),
                 .product(name: "TOML", package: "swift-toml"),
                 "ContainerLog",
                 "ContainerPersistence",
+                "ContainerPlugin",
                 "ContainerResource",
             ]
         ),

Package.swift

@@ -22,18 +22,49 @@ import CryptoKit
 import Foundation
 import GRPCCore
 
+/// Handles the `fssync` stage of the build protocol.
+///
+/// When BuildKit needs build-context files it sends `Walk`, `Read`, and `Info`
+/// requests to the shim, which proxies them over the gRPC stream to this actor.
+///
+/// ## Primary path: Walk (tar mode)
+///
+/// `Walk` is the primary data path. The host packs all requested context paths
+/// into a tar archive and streams it to the shim. The shim unpacks the tar to a
+/// local cache and presents the files to BuildKit via `DiffCopy`. BuildKit then
+/// issues `PACKET_REQ` for regular files it needs; the shim serves those from
+/// the local cache without any further calls to the host.
+///
+/// When a context path is a symlink whose target lies within the context root,
+/// ``walk(_:_:_:)`` adds the target to the archive alongside the symlink so
+/// BuildKit can dereference it during `COPY`/`ADD` processing.
+///
+/// ## Fallback path: Info + Read
+///
+/// `FS.Open()` in the shim falls back to `Info` followed by `Read` calls when
+/// its local checksum cache is unpopulated (a narrow race window at the start of
+/// a build). These paths are not exercised during a normal build.
+///
+/// ## Symlink safety
+///
+/// The host enforces that no file served to the builder resolves to a path
+/// outside the context root. If any component of a requested path is a symlink
+/// whose target lies outside the context root the request is rejected.
+/// Dockerignore filtering is **not** applied here; the shim applies it after
+/// unpacking the tar.
 actor BuildFSSync: BuildPipelineHandler {
     let contextDir: URL
 
     init(_ contextDir: URL) throws {
+        let resolved = contextDir.resolvingSymlinksInPath()
         guard FileManager.default.fileExists(atPath: contextDir.cleanPath) else {
             throw Error.contextNotFound(contextDir.cleanPath)
         }
-        guard try contextDir.isDir() else {
+        guard resolved.isDirectory else {
             throw Error.contextIsNotDirectory(contextDir.cleanPath)
         }
 
-        self.contextDir = contextDir
+        self.contextDir = resolved
     }
 
     nonisolated func accept(_ packet: ServerStream) throws -> Bool {
@@ -63,6 +94,11 @@ actor BuildFSSync: BuildPipelineHandler {
         }
     }
 
+    /// Serves the content of a single context file to the shim.
+    ///
+    /// Called only via the shim's `FS.Open()` fallback path, not during a
+    /// normal `Walk`-based build. Rejects any path whose symlink chain resolves
+    /// outside the context root.
     func read(_ sender: AsyncStream<ClientStream>.Continuation, _ packet: BuildTransfer, _ buildID: String) async throws {
         let offset: UInt64 = packet.offset() ?? 0
         let size: Int = packet.len() ?? 0
@@ -79,6 +115,10 @@ actor BuildFSSync: BuildPipelineHandler {
             path = URL(filePath: self.contextDir.cleanPath)
             path.append(components: packet.source.cleanPathComponent)
         }
+        let resolved = path.resolvingSymlinksInPath()
+        guard self.contextDir.parentOf(resolved) else {
+            throw Error.pathIsNotChild(resolved.cleanPath, self.contextDir.cleanPath)
+        }
         let data = try {
             if try path.isDir() {
                 return Data()
@@ -95,6 +135,12 @@ actor BuildFSSync: BuildPipelineHandler {
         sender.yield(response)
     }
 
+    /// Returns metadata (mode, size, modification time, uid/gid) for a single
+    /// context path.
+    ///
+    /// Called only via the shim's `FS.Open()` fallback path, not during a
+    /// normal `Walk`-based build. Must reject paths that escape the context root
+    /// via symlinks for the same reasons as ``read(_:_:_:)``.
     func info(_ sender: AsyncStream<ClientStream>.Continuation, _ packet: BuildTransfer, _ buildID: String) async throws {
         let path: URL
         if packet.source.hasPrefix("/") {
@@ -105,6 +151,10 @@ actor BuildFSSync: BuildPipelineHandler {
                 .appendingPathComponent(packet.source)
                 .standardizedFileURL
         }
+        let resolved = path.resolvingSymlinksInPath()
+        guard self.contextDir.parentOf(resolved) else {
+            throw Error.pathIsNotChild(resolved.cleanPath, self.contextDir.cleanPath)
+        }
         let transfer = try path.buildTransfer(id: packet.id, contextDir: self.contextDir, complete: true)
         var response = ClientStream()
         response.buildID = buildID
@@ -127,6 +177,23 @@ actor BuildFSSync: BuildPipelineHandler {
         }
     }
 
+    /// Packs requested context paths into a tar archive and streams it to the shim.
+    ///
+    /// This is the primary data path for build-context transfer. BuildKit sends
+    /// a `Walk` request whose `followpaths` field names the context paths needed
+    /// for the current build step (e.g. the source of a `COPY` instruction).
+    /// The host resolves those globs, builds an entry set, and passes it to
+    /// `Archiver.compress` to produce the tar.
+    ///
+    /// For any symlink in the entry set whose target lies within the context
+    /// root, the target is added to the entry set so BuildKit can dereference
+    /// the symlink during `COPY`/`ADD` processing without a separate request.
+    /// Symlinks whose targets lie outside the context root are included as
+    /// symlink entries but their targets are not; BuildKit will resolve them
+    /// against the shim's local filesystem on Linux, not the macOS host.
+    ///
+    /// Dockerignore filtering is the shim's responsibility and is applied after
+    /// the tar is unpacked; this method has no knowledge of `.dockerignore`.
     func walk(
         _ sender: AsyncStream<ClientStream>.Continuation,
         _ packet: BuildTransfer,
@@ -334,16 +401,10 @@ actor BuildFSSync: BuildPipelineHandler {
         let target: String
 
         init(path: URL, contextDir: URL) throws {
-            if path.isSymlink {
-                let target: URL = path.resolvingSymlinksInPath()
-                if contextDir.parentOf(target) {
-                    self.target = target.relativePathFrom(from: path)
-                } else {
-                    self.target = target.cleanPath
-                }
-            } else {
-                self.target = ""
-            }
+            // Always report the literal, unresolved on-disk symlink target —
+            // the same value tar mode provides via Archiver's use of
+            // destinationOfSymbolicLink — rather than a host-resolved path.
+            self.target = path.isSymlink ? try FileManager.default.destinationOfSymbolicLink(atPath: path.cleanPath) : ""
 
             self.name = try path.relativeChildPath(to: contextDir)
             self.modTime = try path.modTime()

Sources/ContainerBuild/BuildFSSync.swift

@@ -23,6 +23,13 @@ import GRPCCore
 import Logging
 import TerminalProgress
 
+/// Handles the `resolver` stage of the build protocol.
+///
+/// Resolves image references on behalf of BuildKit: authenticates with
+/// registries, pulls missing base-image manifests and layers, and stores
+/// them in the local content store. BuildKit delegates these operations to
+/// the host because registry credentials and network access live on the
+/// macOS side, not inside the builder VM.
 struct BuildImageResolver: BuildPipelineHandler {
     let contentStore: ContentStore
     let quiet: Bool

Sources/ContainerBuild/BuildImageResolver.swift

@@ -18,11 +18,61 @@ import Foundation
 import GRPCCore
 import NIO
 
+/// A handler for one stage of the build protocol.
+///
+/// The build pipeline multiplexes a single bidirectional gRPC stream between
+/// the macOS host and the builder shim. Each packet carries a stage tag;
+/// a handler claims packets for its stage via ``accept(_:)`` and processes
+/// them via ``handle(_:_:)``.
 protocol BuildPipelineHandler: Sendable {
     func accept(_ packet: ServerStream) throws -> Bool
     func handle(_ sender: AsyncStream<ClientStream>.Continuation, _ packet: ServerStream) async throws
 }
 
+/// Drives a build session by routing packets from the builder shim to the
+/// appropriate handler.
+///
+/// ## Three-tier architecture
+///
+/// Builds involve three components with distinct responsibilities:
+///
+/// **macOS host (`BuildPipeline` / its handlers)**
+/// Serves resources to the builder shim over a bidirectional gRPC stream.
+/// Responsibilities include:
+/// - Packing requested build-context files into a tar archive (``BuildFSSync``).
+/// - Proxying image-layer blobs from the local content store (``BuildRemoteContentProxy``).
+/// - Resolving and pulling base images (``BuildImageResolver``).
+/// - Relaying builder stdout/stderr to the terminal (``BuildStdio``).
+/// - Enforcing the context root boundary: directory traversal uses `openat(O_NOFOLLOW)`
+///   at every descent step, and every individual file request resolves symlinks to their
+///   canonical path before verifying containment within the context root.
+///
+/// **Builder shim (`container-builder-shim`)**
+/// A Go process running inside a Linux VM that bridges the host gRPC stream
+/// and BuildKit's `filesync` gRPC interface. Responsibilities include:
+/// - Receiving the context tar from the host, unpacking it to a local cache,
+///   and presenting the result to BuildKit via `DiffCopy`.
+/// - Applying dockerignore exclusions (received from BuildKit as
+///   `exclude-patterns` metadata) when walking the unpacked cache.
+/// - Passing `followpaths` from BuildKit to the host so the host knows which
+///   context paths to include in the tar.
+///
+/// **BuildKit**
+/// Parses and executes the Dockerfile. Responsibilities include:
+/// - Sending `Walk` requests with `followpaths` derived from each `COPY`/`ADD`
+///   source and `exclude-patterns` derived from `.dockerignore`.
+/// - Dereferencing symlinks, recursing into directories, and applying all
+///   other COPY/ADD transfer semantics on the unpacked context the shim provides.
+///
+/// ## Packet flow
+///
+/// ```
+/// BuildKit ──► shim DiffCopy ──► host Walk  (tar of context files)
+///                            ◄── tar archive
+///          ◄── PACKET_STAT per file (after shim unpacks + filters)
+///          ──► PACKET_REQ for each regular file
+///          ◄── PACKET_DATA (shim reads from local unpacked cache)
+/// ```
 public actor BuildPipeline {
     let handlers: [BuildPipelineHandler]
     public init(_ config: Builder.BuildConfig) async throws {

Sources/ContainerBuild/BuildPipelineHandler.swift

@@ -21,6 +21,11 @@ import ContainerizationOCI
 import Foundation
 import GRPCCore
 
+/// Handles the `content-store` stage of the build protocol.
+///
+/// Proxies image-layer blob requests from BuildKit to the host's local
+/// containerd content store. BuildKit issues these requests when it needs
+/// base-image layers that are not already present in the builder VM.
 struct BuildRemoteContentProxy: BuildPipelineHandler {
     let local: ContentStore
 

Sources/ContainerBuild/BuildRemoteContentProxy.swift

@@ -19,6 +19,11 @@ import Foundation
 import GRPCCore
 import NIO
 
+/// Handles the stdio stage of the build protocol.
+///
+/// Relays builder stdout/stderr from the shim to the client terminal.
+/// Build output (layer download progress, `RUN` command output, etc.) flows
+/// through this handler and is written directly to the configured file handle.
 actor BuildStdio: BuildPipelineHandler {
     public let quiet: Bool
     public let handle: FileHandle