Retry SSH connect with configurable max attempts; update dependencies (#70)

Author: HeEAaD

Cilicon.xcodeproj/project.xcworkspace/xcshareddata/swiftpm/Package.resolved

@@ -12,6 +12,7 @@ struct Config: Codable {
         autoTransferImageVolume: String? = nil,
         retryDelay: Int,
         sshCredentials: SSHCredentials,
+        sshConnectMaxRetries: Int,
         preRun: String? = nil,
         postRun: String? = nil,
         consoleDevices: [String] = []
@@ -25,6 +26,7 @@ struct Config: Codable {
         self.runnerName = runnerName
         self.retryDelay = retryDelay
         self.sshCredentials = sshCredentials
+        self.sshConnectMaxRetries = sshConnectMaxRetries
         self.preRun = preRun
         self.postRun = postRun
         self.consoleDevices = consoleDevices
@@ -50,6 +52,8 @@ struct Config: Codable {
     let retryDelay: Int
     /// Credentials to be used when connecting via SSH.
     let sshCredentials: SSHCredentials
+    /// Maximum number of retries for SSH connection attempts.
+    let sshConnectMaxRetries: Int
     /// A command to run before the provisioning commands are run.
     let preRun: String?
     /// A command to run after the provisioning commands are run.
@@ -67,6 +71,7 @@ struct Config: Codable {
         case runnerName
         case retryDelay
         case sshCredentials
+        case sshConnectMaxRetries
         case preRun
         case postRun
         case consoleDevices
@@ -85,6 +90,7 @@ struct Config: Codable {
         self.runnerName = try container.decodeIfPresent(String.self, forKey: .runnerName)
         self.retryDelay = try container.decodeIfPresent(Int.self, forKey: .retryDelay) ?? 5
         self.sshCredentials = try container.decodeIfPresent(SSHCredentials.self, forKey: .sshCredentials) ?? .default
+        self.sshConnectMaxRetries = try container.decodeIfPresent(Int.self, forKey: .sshConnectMaxRetries) ?? 10
         self.preRun = try container.decodeIfPresent(String.self, forKey: .preRun)
         self.postRun = try container.decodeIfPresent(String.self, forKey: .postRun)
         self.consoleDevices = try container.decodeIfPresent([String].self, forKey: .consoleDevices) ?? []

Cilicon/Config/Config.swift

@@ -115,12 +115,8 @@ class VMManager: NSObject, ObservableObject {
         vmState = .running(virtualMachine)
         self.ip = try await fetchIP(macAddress: clonedBundle.configuration.macAddress.string)
 
-        let client = try await SSHClient.connect(
-            host: ip,
-            authenticationMethod: .passwordBased(username: config.sshCredentials.username, password: config.sshCredentials.password),
-            hostKeyValidator: .acceptAnything(),
-            reconnect: .always
-        )
+        // Wait for VM to fully boot and can execute SSH commands before proceeding
+        let client = try await createAndConnectSSHClient(ip: ip)
 
         if let preRun = config.preRun {
             let streamOutput = try await client.executeCommandStream(preRun, inShell: true)
@@ -161,7 +157,59 @@ class VMManager: NSObject, ObservableObject {
         }
     }
 
-    func provisionVM() async throws { }
+    /// Creates and connects an SSH client to the given IP address, retrying until successful or a timeout occurs.
+    @MainActor
+    private func createAndConnectSSHClient(ip: String) async throws -> SSHClient {
+        SSHLogger.shared.log(string: "Waiting for VM to boot and SSH to be available...\n")
+        let maxRetries = config.sshConnectMaxRetries
+        var tries = 0
+
+        while tries < maxRetries {
+            do {
+                let client = try await SSHClient.connect(
+                    host: ip,
+                    authenticationMethod: .passwordBased(
+                        username: config.sshCredentials.username,
+                        password: config.sshCredentials.password
+                    ),
+                    hostKeyValidator: .acceptAnything(),
+                    reconnect: .never,
+                    connectTimeout: .seconds(5)
+                )
+
+                // Test if we can execute a simple command
+                let token = "ssh-connected"
+                let streamOutput = try await client.executeCommandStream("echo \(token)", inShell: true)
+                var commandSuccessful = false
+
+                for try await blob in streamOutput {
+                    switch blob {
+                    case let .stdout(stdout):
+                        let output = String(buffer: stdout)
+                        if output.contains(token) {
+                            commandSuccessful = true
+                        }
+                    case .stderr:
+                        break
+                    }
+                }
+
+                if commandSuccessful {
+                    SSHLogger.shared.log(string: "VM fully booted and SSH available\n")
+                    return client
+                }
+
+                try await client.close()
+            } catch {
+                // SSH not ready yet, continue waiting
+                tries += 1
+                SSHLogger.shared.log(string: "SSH connect \(tries)/\(maxRetries): SSH not ready, waiting 5s...\n")
+                try await Task.sleep(for: .seconds(5))
+            }
+        }
+
+        throw VMManagerError.sshConnectTimeout
+    }
 
     func isBundleComplete() throws -> Bool {
         let filesExist = [
@@ -313,6 +361,7 @@ extension VMManager {
 enum VMManagerError: Error {
     case masterBundleNotFound(path: String)
     case failedToCreateDiskFile
+    case sshConnectTimeout
 }
 
 extension VMManagerError: LocalizedError {
@@ -322,6 +371,8 @@ extension VMManagerError: LocalizedError {
             return "Could not found bundle at \(path)"
         case .failedToCreateDiskFile:
             return "Failed to create Disk File"
+        case .sshConnectTimeout:
+            return "SSH Connect timeout"
         }
     }
 }

Cilicon/VMManager.swift

@@ -105,7 +105,8 @@ provisioner:
     downloadURL: <DOWNLOAD_URL> # defaults to GitLab official S3 bucket
     tomlPath: <PATH_TO_TOML> # defaults to `nil`. If set, it ignores the other runner related variables and passes the specified path to the runner executable
 consoleDevices:
-  - tart-version-cilicon
+  - tart-version-2
+sshConnectMaxRetries: <SSH_CONNECT_MAX_RETRIES> # defaults to 10
 ```
 
 #### Buildkite Agent
@@ -142,7 +143,16 @@ To add console devices, use the `consoleDevices` field in your configuration:
 
 ```yml
 consoleDevices:
-  - tart-version-cilicon
+  - tart-version-2
+```
+
+#### SSH Connect Retries
+After the VM starts, Cilicon connects to the guest over SSH to run any preRun/postRun commands and the provisioner. Some images may take a while before the SSH service is ready. Use `sshConnectMaxRetries` to control how many connection attempts Cilicon will make before giving up (default: 10).
+
+Example:
+
+```yml
+sshConnectMaxRetries: 20
 ```
 
 ### 🔨 Setting Up the Host OS