Add stream-based copyIn/copyOut for tar stream cp support

yash-mandaviya · claude · yash-mandaviya · commit 570a56f74bf7 · 2026-07-23T23:14:47.000-04:00
Adds file-handle/stream-based copy entrypoints so callers can transfer a tar stream directly to/from a container's filesystem without staging the contents on the host, enabling true `docker cp -` / `podman cp -` parity in downstream CLIs (apple/container#1908). - LinuxContainer.copyIn(archive:to:) splices a tar stream from a file handle straight to the guest over vsock; the guest extracts it in place, honoring the ownership/mode/symlink metadata in the tar headers. - LinuxContainer.copyOut(from:to:) streams the guest-produced tar archive to a file handle; it always archives, even a single regular file, to match `docker cp CONTAINER:/path -`. - ArchiveReader.init(fileHandle:) auto-detects archive format and compression filter, so an externally supplied tar stream (uncompressed or gzip/bzip2/xz) is accepted, not only the internal pax+gzip form. - vminitd copy handlers use the auto-detecting reader for copyIn and honor CopyRequest.is_archive to force single-file archiving on copyOut. No host-side temp staging, so host permission and path-length limits no longer constrain the transfer. Existing path-based copyIn/copyOut behavior is unchanged (the new is_archive-on-copyOut path is opt-in). Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
diff --git a/Sources/Containerization/LinuxContainer.swift b/Sources/Containerization/LinuxContainer.swift
@@ -1439,6 +1439,179 @@ extension LinuxContainer {
             }
         }
     }
+
+    /// Copy the contents of a tar stream into the container.
+    ///
+    /// The `archive` file handle is expected to yield a tar stream, either
+    /// uncompressed or compressed (gzip/bzip2/xz), matching `docker cp -` /
+    /// `podman cp -` input semantics. The bytes are streamed directly to the
+    /// guest, which extracts them in place honoring the ownership, mode and
+    /// symlink metadata recorded in the tar headers. No intermediate files are
+    /// created on the host, so host-side permission and path-length constraints
+    /// do not apply.
+    ///
+    /// - Parameters:
+    ///   - archive: A file handle yielding the tar stream to extract.
+    ///   - destination: The guest directory to extract the archive into.
+    ///   - createParents: Create parent directories of `destination` if missing.
+    ///   - chunkSize: The transfer chunk size in bytes.
+    public func copyIn(
+        archive: FileHandle,
+        to destination: URL,
+        createParents: Bool = true,
+        chunkSize: Int = defaultCopyChunkSize
+    ) async throws {
+        try await self.state.withLock {
+            let state = try $0.startedState("copyIn")
+
+            let guestPath = URL(filePath: self.root).appending(path: destination.path)
+            let port = self.hostVsockPorts.wrappingAdd(1, ordering: .relaxed).oldValue
+            let listener = try state.vm.listen(port)
+
+            try await withThrowingTaskGroup(of: Void.self) { group in
+                group.addTask {
+                    try await state.vm.withAgent { agent in
+                        guard let vminitd = agent as? Vminitd else {
+                            throw ContainerizationError(.unsupported, message: "copyIn requires Vminitd agent")
+                        }
+                        try await vminitd.copy(
+                            direction: .copyIn,
+                            guestPath: guestPath,
+                            vsockPort: port,
+                            createParents: createParents,
+                            isArchive: true
+                        )
+                    }
+                }
+
+                group.addTask {
+                    guard let conn = await listener.first(where: { _ in true }) else {
+                        throw ContainerizationError(.internalError, message: "copyIn: vsock connection not established")
+                    }
+                    try listener.finish()
+
+                    try await withCheckedThrowingContinuation { (continuation: CheckedContinuation<Void, any Error>) in
+                        self.copyQueue.async {
+                            do {
+                                defer { conn.closeFile() }
+                                try Self.spliceFd(
+                                    from: archive.fileDescriptor,
+                                    to: conn.fileDescriptor,
+                                    chunkSize: chunkSize,
+                                    label: "copyIn"
+                                )
+                                continuation.resume()
+                            } catch {
+                                continuation.resume(throwing: error)
+                            }
+                        }
+                    }
+                }
+
+                try await group.waitForAll()
+            }
+        }
+    }
+
+    /// Stream the contents of a container path as a tar archive to a file handle.
+    ///
+    /// Matches `docker cp CONTAINER:/path -` / `podman cp` output semantics: the
+    /// output is always a tar archive, even for a single regular file, and
+    /// preserves the ownership and mode recorded in the guest filesystem. The
+    /// guest performs the archiving natively and streams the result directly;
+    /// no intermediate files are created on the host.
+    ///
+    /// - Parameters:
+    ///   - source: The guest path (file or directory) to archive.
+    ///   - archive: A file handle the tar stream is written to.
+    ///   - chunkSize: The transfer chunk size in bytes.
+    public func copyOut(
+        from source: URL,
+        to archive: FileHandle,
+        chunkSize: Int = defaultCopyChunkSize
+    ) async throws {
+        try await self.state.withLock {
+            let state = try $0.startedState("copyOut")
+
+            let guestPath = URL(filePath: self.root).appending(path: source.path)
+            let port = self.hostVsockPorts.wrappingAdd(1, ordering: .relaxed).oldValue
+            let listener = try state.vm.listen(port)
+
+            let (metadataStream, metadataCont) = AsyncStream.makeStream(of: Vminitd.CopyMetadata.self)
+
+            try await withThrowingTaskGroup(of: Void.self) { group in
+                group.addTask {
+                    try await state.vm.withAgent { agent in
+                        guard let vminitd = agent as? Vminitd else {
+                            throw ContainerizationError(.unsupported, message: "copyOut requires Vminitd agent")
+                        }
+                        try await vminitd.copy(
+                            direction: .copyOut,
+                            guestPath: guestPath,
+                            vsockPort: port,
+                            isArchive: true,
+                            onMetadata: { meta in
+                                metadataCont.yield(meta)
+                                metadataCont.finish()
+                            }
+                        )
+                    }
+                }
+
+                group.addTask {
+                    // Wait for the guest to report metadata (and thus that the
+                    // source exists) before accepting the data connection.
+                    _ = await metadataStream.first(where: { _ in true })
+
+                    guard let conn = await listener.first(where: { _ in true }) else {
+                        throw ContainerizationError(.internalError, message: "copyOut: vsock connection not established")
+                    }
+                    try listener.finish()
+
+                    try await withCheckedThrowingContinuation { (continuation: CheckedContinuation<Void, any Error>) in
+                        self.copyQueue.async {
+                            do {
+                                defer { conn.closeFile() }
+                                try Self.spliceFd(
+                                    from: conn.fileDescriptor,
+                                    to: archive.fileDescriptor,
+                                    chunkSize: chunkSize,
+                                    label: "copyOut"
+                                )
+                                continuation.resume()
+                            } catch {
+                                continuation.resume(throwing: error)
+                            }
+                        }
+                    }
+                }
+
+                try await group.waitForAll()
+            }
+        }
+    }
+
+    /// Copy raw bytes from one file descriptor to another until EOF.
+    private static func spliceFd(from srcFd: Int32, to dstFd: Int32, chunkSize: Int, label: String) throws {
+        var buf = [UInt8](repeating: 0, count: chunkSize)
+        while true {
+            let n = read(srcFd, &buf, buf.count)
+            if n == 0 { break }
+            guard n > 0 else {
+                throw ContainerizationError(.internalError, message: "\(label): read error: \(String(cString: strerror(errno)))")
+            }
+            var written = 0
+            while written < n {
+                let w = buf.withUnsafeBytes { ptr in
+                    write(dstFd, ptr.baseAddress! + written, n - written)
+                }
+                guard w > 0 else {
+                    throw ContainerizationError(.internalError, message: "\(label): vsock write error: \(String(cString: strerror(errno)))")
+                }
+                written += w
+            }
+        }
+    }
 }
 
 extension VirtualMachineInstance {
diff --git a/Sources/ContainerizationArchive/ArchiveReader.swift b/Sources/ContainerizationArchive/ArchiveReader.swift
@@ -106,6 +106,29 @@ public final class ArchiveReader {
             .checkOk(elseThrow: { .unableToOpenArchive($0) })
     }
 
+    /// Initializes an `ArchiveReader` to read from the provided file descriptor,
+    /// auto-detecting the archive `Format` and compression `Filter`.
+    ///
+    /// Use this when the incoming stream's format and compression are not known
+    /// ahead of time, e.g. a tar stream supplied on stdin that may be
+    /// uncompressed or compressed (gzip, bzip2, xz, ...), matching
+    /// `docker cp -` / `podman cp -` input semantics. zstd is not auto-detected
+    /// from a stream because it requires seeking; use
+    /// ``init(format:filter:fileHandle:)`` with ``Filter/zstd`` for that case.
+    public init(fileHandle: FileHandle) throws {
+        self.underlying = archive_read_new()
+        self.fileHandle = fileHandle
+
+        try archive_read_support_filter_all(underlying)
+            .checkOk(elseThrow: .failedToDetectFilter)
+        try archive_read_support_format_all(underlying)
+            .checkOk(elseThrow: .failedToDetectFormat)
+
+        let fd = fileHandle.fileDescriptor
+        try archive_read_open_fd(underlying, fd, 4096)
+            .checkOk(elseThrow: { .unableToOpenArchive($0) })
+    }
+
     /// Initialize the `ArchiveReader` to read from a specified file URL
     /// by trying to auto determine the archives `Format` and `Filter`.
     public init(file: URL) throws {
diff --git a/Tests/ContainerizationArchiveTests/ArchiveReaderTests.swift b/Tests/ContainerizationArchiveTests/ArchiveReaderTests.swift
@@ -658,6 +658,66 @@ struct ArchiveReaderTests {
         }
     }
 
+    // MARK: - FileHandle Stream Auto-Detect Tests
+
+    /// The stream-based auto-detecting reader must accept an uncompressed tar
+    /// stream, matching `docker cp -` / `podman cp -` input semantics.
+    @Test func readUncompressedTarFromFileHandle() throws {
+        let archiveURL = try createTestArchive(
+            name: "stream-plain",
+            entries: [
+                ("dir/", .directory, nil),
+                ("dir/file.txt", .regular("streamed plain"), nil),
+            ])
+        defer { try? FileManager.default.removeItem(at: archiveURL.deletingLastPathComponent()) }
+
+        let extractDir = try createExtractionDirectory(name: "stream-plain")
+        defer { try? FileManager.default.removeItem(at: extractDir.deletingLastPathComponent()) }
+
+        let fh = try FileHandle(forReadingFrom: archiveURL)
+        defer { try? fh.close() }
+
+        let reader = try ArchiveReader(fileHandle: fh)
+        let rejected = try reader.extractContents(to: extractDir)
+
+        #expect(rejected.isEmpty)
+        let content = try String(contentsOf: extractDir.appendingPathComponent("dir/file.txt"), encoding: .utf8)
+        #expect(content == "streamed plain")
+    }
+
+    /// The stream-based auto-detecting reader must also accept a gzip-compressed
+    /// tar stream (the format the guest emits internally for directory copies),
+    /// without being told the filter up front.
+    @Test func readGzipTarFromFileHandle() throws {
+        let testDirectory = createTemporaryDirectory(baseName: "ArchiveReaderTests")!
+        defer { try? FileManager.default.removeItem(at: testDirectory) }
+        let archiveURL = testDirectory.appendingPathComponent("stream-gzip.tar.gz")
+
+        let writer = try ArchiveWriter(configuration: .init(format: .pax, filter: .gzip))
+        try writer.open(file: archiveURL)
+        let entry = WriteEntry()
+        entry.path = "hello.txt"
+        entry.fileType = .regular
+        entry.permissions = 0o644
+        let data = "streamed gzip".data(using: .utf8)!
+        entry.size = numericCast(data.count)
+        try writer.writeEntry(entry: entry, data: data)
+        try writer.finishEncoding()
+
+        let extractDir = try createExtractionDirectory(name: "stream-gzip")
+        defer { try? FileManager.default.removeItem(at: extractDir.deletingLastPathComponent()) }
+
+        let fh = try FileHandle(forReadingFrom: archiveURL)
+        defer { try? fh.close() }
+
+        let reader = try ArchiveReader(fileHandle: fh)
+        let rejected = try reader.extractContents(to: extractDir)
+
+        #expect(rejected.isEmpty)
+        let content = try String(contentsOf: extractDir.appendingPathComponent("hello.txt"), encoding: .utf8)
+        #expect(content == "streamed gzip")
+    }
+
     // MARK: - Zstd Compression Tests
 
     @Test func readZstdCompressedArchive() throws {
diff --git a/vminitd/Sources/VminitdCore/Server+GRPC.swift b/vminitd/Sources/VminitdCore/Server+GRPC.swift
@@ -535,7 +535,11 @@ extension Initd: Com_Apple_Containerization_Sandbox_V3_SandboxContext.SimpleServ
             try FileManager.default.createDirectory(at: destURL, withIntermediateDirectories: true)
 
             let fileHandle = FileHandle(fileDescriptor: sockFd, closeOnDealloc: false)
-            let reader = try ArchiveReader(format: .pax, filter: .gzip, fileHandle: fileHandle)
+            // Auto-detect format and compression filter so both the internal
+            // pax+gzip archives (directory copyIn) and externally supplied tar
+            // streams (uncompressed or compressed, e.g. `docker cp -`) are
+            // accepted.
+            let reader = try ArchiveReader(fileHandle: fileHandle)
             return try reader.extractContents(to: destURL)
         }
 
@@ -562,7 +566,10 @@ extension Initd: Com_Apple_Containerization_Sandbox_V3_SandboxContext.SimpleServ
         guard FileManager.default.fileExists(atPath: path, isDirectory: &isDirectory) else {
             throw RPCError(code: .notFound, message: "copy: path not found '\(path)'")
         }
-        let isArchive = isDirectory.boolValue
+        // `request.isArchive` forces archive output even for a single regular
+        // file, matching `docker cp CONTAINER:/path -` which always emits a tar
+        // stream. Directories are always archived.
+        let isArchive = isDirectory.boolValue || request.isArchive
 
         // Determine total size for single files.
         var totalSize: UInt64 = 0
@@ -593,7 +600,15 @@ extension Initd: Com_Apple_Containerization_Sandbox_V3_SandboxContext.SimpleServ
                 let fileURL = URL(fileURLWithPath: path)
                 let writer = try ArchiveWriter(configuration: .init(format: .pax, filter: .gzip))
                 try writer.open(fileDescriptor: sock.fileDescriptor)
-                try writer.archiveDirectory(fileURL)
+                if isDirectory.boolValue {
+                    try writer.archiveDirectory(fileURL)
+                } else {
+                    // Forced single-file archive: emit one entry named after the
+                    // file's basename, relative to its parent directory.
+                    let filePath = FilePath(path)
+                    let base = filePath.removingLastComponent()
+                    try writer.archive([filePath], base: base)
+                }
                 try writer.finishEncoding()
             } else {
                 let srcFd = open(path, O_RDONLY)
