update filesystem operation to run in new namespace

Co-authored-by: John Logan <john_logan@apple.com>

Author: saehejkang

Sources/Containerization/LinuxContainer.swift

@@ -1134,8 +1134,7 @@ extension LinuxContainer {
                 guard let vminitd = agent as? Vminitd else {
                     throw ContainerizationError(.unsupported, message: "filesystemOperation requires Vminitd agent")
                 }
-                let guestPath = URL(filePath: Self.guestRootfsPath(self.id)).appending(path: path).path
-                try await vminitd.filesystemOperation(operation: operation, path: guestPath)
+                try await vminitd.filesystemOperation(operation: operation, path: path, containerID: self.id)
             }
         }
     }

Sources/Containerization/LinuxPod.swift

@@ -1285,8 +1285,7 @@ extension LinuxPod {
                 guard let vminitd = agent as? Vminitd else {
                     throw ContainerizationError(.unsupported, message: "filesystemOperation requires Vminitd agent")
                 }
-                let guestPath = URL(filePath: Self.guestRootfsPath(containerID)).appending(path: path).path
-                try await vminitd.filesystemOperation(operation: operation, path: guestPath)
+                try await vminitd.filesystemOperation(operation: operation, path: path, containerID: containerID)
             }
         }
     }

Sources/Containerization/SandboxContext/SandboxContext.pb.swift

@@ -1154,6 +1154,15 @@ public nonisolated struct Com_Apple_Containerization_Sandbox_V3_FilesystemOperat
     set {operation = .thaw(newValue)}
   }
 
+  public var containerID: String {
+    get {_containerID ?? String()}
+    set {_containerID = newValue}
+  }
+  /// Returns true if `containerID` has been explicitly set.
+  public var hasContainerID: Bool {self._containerID != nil}
+  /// Clears the value of `containerID`. Subsequent reads from it will return its default value.
+  public mutating func clearContainerID() {self._containerID = nil}
+
   public var unknownFields = SwiftProtobuf.UnknownStorage()
 
   public nonisolated enum OneOf_Operation: Equatable, Sendable {
@@ -1164,6 +1173,8 @@ public nonisolated struct Com_Apple_Containerization_Sandbox_V3_FilesystemOperat
   }
 
   public init() {}
+
+  fileprivate var _containerID: String? = nil
 }
 
 public nonisolated struct Com_Apple_Containerization_Sandbox_V3_FilesystemOperationResponse: Sendable {
@@ -3453,7 +3464,7 @@ nonisolated extension Com_Apple_Containerization_Sandbox_V3_FiTrimResult: SwiftP
 
 nonisolated extension Com_Apple_Containerization_Sandbox_V3_FilesystemOperationRequest: SwiftProtobuf.Message, SwiftProtobuf._MessageImplementationBase, SwiftProtobuf._ProtoNameProviding {
   public static let protoMessageName: String = _protobuf_package + ".FilesystemOperationRequest"
-  public static let _protobuf_nameMap = SwiftProtobuf._NameMap(bytecode: "\0\u{1}path\0\u{1}trim\0\u{1}freeze\0\u{1}thaw\0")
+  public static let _protobuf_nameMap = SwiftProtobuf._NameMap(bytecode: "\0\u{1}path\0\u{1}trim\0\u{1}freeze\0\u{1}thaw\0\u{1}containerID\0")
 
   public mutating func decodeMessage<D: SwiftProtobuf.Decoder>(decoder: inout D) throws {
     while let fieldNumber = try decoder.nextFieldNumber() {
@@ -3501,6 +3512,7 @@ nonisolated extension Com_Apple_Containerization_Sandbox_V3_FilesystemOperationR
           self.operation = .thaw(v)
         }
       }()
+      case 5: try { try decoder.decodeSingularStringField(value: &self._containerID) }()
       default: break
       }
     }
@@ -3529,12 +3541,16 @@ nonisolated extension Com_Apple_Containerization_Sandbox_V3_FilesystemOperationR
     }()
     case nil: break
     }
+    try { if let v = self._containerID {
+      try visitor.visitSingularStringField(value: v, fieldNumber: 5)
+    } }()
     try unknownFields.traverse(visitor: &visitor)
   }
 
   public static func ==(lhs: Com_Apple_Containerization_Sandbox_V3_FilesystemOperationRequest, rhs: Com_Apple_Containerization_Sandbox_V3_FilesystemOperationRequest) -> Bool {
     if lhs.path != rhs.path {return false}
     if lhs.operation != rhs.operation {return false}
+    if lhs._containerID != rhs._containerID {return false}
     if lhs.unknownFields != rhs.unknownFields {return false}
     return true
   }

Sources/Containerization/SandboxContext/SandboxContext.proto

@@ -315,6 +315,7 @@ message FilesystemOperationRequest {
     FiFreezeParams freeze = 3;
     FiThawParams thaw = 4;
   }
+  optional string containerID = 5;
 }
 
 message FilesystemOperationResponse {

Sources/Containerization/VirtualMachineAgent.swift

@@ -41,7 +41,7 @@ public protocol VirtualMachineAgent: Sendable {
     /// Close any resources held by the agent.
     func close() async throws
     // Perform a filesystem operation on the given path.
-    func filesystemOperation(operation: FilesystemOperation, path: String) async throws
+    func filesystemOperation(operation: FilesystemOperation, path: String, containerID: String?) async throws
 
     // POSIX-y
     func getenv(key: String) async throws -> String

Sources/Containerization/Vminitd.swift

@@ -213,11 +213,14 @@ extension Vminitd: VirtualMachineAgent {
     }
 
     /// Perform a filesystem operation on a path inside the sandbox's environment.
-    public func filesystemOperation(operation: FilesystemOperation, path: String) async throws {
+    public func filesystemOperation(operation: FilesystemOperation, path: String, containerID: String? = nil) async throws {
         _ = try await client.filesystemOperation(
             .with {
                 $0.operation = operation.toProtoOperation()
                 $0.path = path
+                if let containerID {
+                    $0.containerID = containerID
+                }
             })
     }
 

vminitd/Sources/LCShim/include/syscall.h

@@ -28,6 +28,9 @@
 
 // CLONE_* flags
 #ifndef CLONE_NEWNS
+#define CLONE_FS        0x00000200
+#endif
+#ifndef CLONE_NEWNS
 #define CLONE_NEWNS     0x00020000
 #endif
 #ifndef CLONE_NEWCGROUP

vminitd/Sources/VminitdCore/Server+GRPC.swift

@@ -27,6 +27,7 @@ import ContainerizationOS
 import Foundation
 import GRPCCore
 import GRPCProtobuf
+import LCShim
 import Logging
 import NIOCore
 import NIOPosix
@@ -722,18 +723,95 @@ extension Initd: Com_Apple_Containerization_Sandbox_V3_SandboxContext.SimpleServ
         async throws -> Com_Apple_Containerization_Sandbox_V3_FilesystemOperationResponse
     {
         let path = FilePath(request.path)
+        if !request.hasContainerID {
+            throw ContainerizationError(
+                .invalidArgument,
+                message: "containerID is required"
+            )
+        }
+
+        guard let operation = request.operation else {
+            throw ContainerizationError(
+                .invalidArgument,
+                message: "operation is required"
+            )
+        }
+
+        let container = try await state.get(container: request.containerID)
+        guard let containerPid = await container.pid else {
+            throw ContainerizationError(
+                .invalidArgument,
+                message: "container PID is not present"
+            )
+        }
 
         log.debug(
             "filesystemOperation",
             metadata: [
-                "operation": "\(String(describing: request.operation))",
+                "containerID": "\(request.containerID)",
+                "containerPid": "\(containerPid)",
+                "operation": "\(operation)",
                 "path": "\(path)",
             ])
 
         if !path.isAbsolute {
             throw RPCError(code: .invalidArgument, message: "path must be absolute")
         }
 
+        let selfMountFd = open("/proc/self/ns/mnt", O_RDONLY | O_CLOEXEC)
+        if selfMountFd < 0 {
+            let error = swiftErrno("open")
+            throw RPCError(code: .internalError, message: "failed to open self mount namespace", cause: error)
+        }
+
+        defer { close(selfMountFd) }
+
+        let containerMountFd = open("/proc/\(containerPid)/ns/mnt", O_RDONLY | O_CLOEXEC)
+        if containerMountFd < 0 {
+            let error = swiftErrno("open")
+            throw RPCError(code: .internalError, message: "failed to open container mount namespace", cause: error)
+        }
+
+        defer { close(containerMountFd) }
+
+        var finfo = _stat_struct()
+        let selfMountStat = fstat(selfMountFd, &finfo)
+        if selfMountStat != 0 {
+            let error = swiftErrno("fstat")
+            throw RPCError(code: .internalError, message: "failed to stat self mount namespace", cause: error)
+        }
+        let selfInode = finfo.st_ino
+
+        let containerMountStat = fstat(containerMountFd, &finfo)
+        if containerMountStat != 0 {
+            let error = swiftErrno("fstat")
+            throw RPCError(code: .internalError, message: "failed to stat container mount namespace", cause: error)
+        }
+        let containerInode = finfo.st_ino
+
+        if selfInode == containerInode {
+            try doFilesystemOperation(path: path, operation: operation)
+        } else {
+            try await self.runOnDedicatedThread {
+                if unshare(CLONE_FS) != 0 {
+                    let error = self.swiftErrno("unshare(CLONE_FS)")
+                    throw RPCError(code: .internalError, message: "failed to unshare filesystem namespace", cause: error)
+                }
+                if setns(containerMountFd, CLONE_NEWNS) != 0 {
+                    let error = self.swiftErrno("setns(CLONE_NEWNS)")
+                    throw RPCError(code: .internalError, message: "failed to enter container mount namespace", cause: error)
+                }
+                try self.doFilesystemOperation(path: path, operation: operation)
+            }
+        }
+
+        return .init()
+    }
+
+    private func doFilesystemOperation(
+        path: FilePath,
+        operation: Com_Apple_Containerization_Sandbox_V3_FilesystemOperationRequest.OneOf_Operation
+    ) throws {
         var finfo = _stat_struct()
         let rc = _stat(path.string, &finfo)
         if rc != 0 {
@@ -753,20 +831,18 @@ extension Initd: Com_Apple_Containerization_Sandbox_V3_SandboxContext.SimpleServ
         defer { close(fd) }
 
         do {
-            switch request.operation {
-            case .freeze:
+            switch operation {
+            case .freeze(_):
                 try freezeFilesystem(fd: fd)
-            case .thaw:
+            case .thaw(_):
                 try thawFilesystem(fd: fd)
             case .trim(let params):
                 switch params.schedule {
-                case .oneShot:
+                case .oneShot(_):
                     try trimFilesystem(fd: fd)
                 case .none:
                     throw RPCError(code: .invalidArgument, message: "trim schedule must be specified")
                 }
-            case .none:
-                throw RPCError(code: .invalidArgument, message: "invalid operation")
             }
         } catch {
             log.error(
@@ -776,8 +852,6 @@ extension Initd: Com_Apple_Containerization_Sandbox_V3_SandboxContext.SimpleServ
                 ])
             throw RPCError(code: .internalError, message: "filesystemOperation", cause: error)
         }
-
-        return .init()
     }
 
     private func freezeFilesystem(fd: Int32) throws {
@@ -1650,6 +1724,22 @@ extension Initd: Com_Apple_Containerization_Sandbox_V3_SandboxContext.SimpleServ
         return error
     }
 
+    private func runOnDedicatedThread<T: Sendable>(
+        _ work: @escaping @Sendable () throws -> T
+    ) async throws -> T {
+        try await withCheckedThrowingContinuation { continuation in
+            let thread = Thread {
+                do {
+                    let result = try work()
+                    continuation.resume(returning: result)
+                } catch {
+                    continuation.resume(throwing: error)
+                }
+            }
+            thread.start()
+        }
+    }
+
     // NOTE: This is just crummy. It works because today the assumption is
     // every NIC in the root net namespace is for the container(s), but if we
     // ever supported individual containers having their own NICs/IPs then this