Merge pull request #2 from timzhong1024/codex/permission-onboarding

[codex] Add permission onboarding flow

Author: timzhong1024

README.md

@@ -17,11 +17,14 @@ It is designed around three defaults:
 
 Use `macos-cua doctor` to inspect current readiness.
 
+Use `macos-cua onboard` to trigger the native prompts, open the relevant System Settings panes, and guide a human through granting both permissions. In a tty session it waits by default; in non-tty mode it triggers the flow and returns immediately unless you pass `--wait`. When Screen Recording appears to have been granted but the process has not yet been restarted, `onboard` surfaces a targeted restart hint rather than a generic enable instruction. Add `--json` for structured output including per-permission `granted`, `waited`, and `likelyNeedsRestart` fields.
+
 ## Commands
 
 ```text
 macos-cua [--json] <command> [args...]
 
+onboard [--wait|--no-wait] [--timeout <seconds>] [--no-request] [--no-open]
 doctor
 state
 record enable|disable|status
@@ -42,6 +45,9 @@ window frontmost|list|activate|minimize|maximize|close
 
 ```bash
 swift run macos-cua doctor
+swift run macos-cua onboard
+swift run macos-cua onboard --wait --timeout 180
+swift run macos-cua --json onboard --no-wait
 swift run macos-cua record enable
 swift run macos-cua --json state
 swift run macos-cua screenshot /tmp/frontmost.png
@@ -66,6 +72,7 @@ swift run macos-cua window list
 - If no usable frontmost window is available, default coordinate-taking commands fall back to screen coordinates and report that fallback in output.
 - `window list` is AX-first when Accessibility is available, then falls back to CoreGraphics window discovery.
 - `window list`, `window frontmost`, and `state.frontmostWindow.bounds` remain screen-global diagnostics; they are not window-local action coordinates.
+- Missing permission errors point back to `macos-cua onboard` so agent and human flows land on the same recovery path.
 - Browser DOM/ref actions are intentionally out of scope for this repo.
 - `record enable` starts a persistent session under `~/Library/Application Support/macos-cua/records/`; each subsequent command appends an action log entry, a full-screen timeline screenshot, failure-only snapshots, and a replayable `replay.sh` trace until `record disable`.
 - A shareable VS Code debug example lives at `.vscode/launch.example.json`; local `.vscode/launch.json` stays ignored.

Sources/macos-cua/Commands/CLI.swift

@@ -6,6 +6,7 @@ enum CLI {
       macos-cua [--json] <command> [args...]
 
     Commands:
+      onboard [--wait|--no-wait] [--timeout <seconds>] [--no-request] [--no-open]
       doctor
       state
       record enable|disable|status
@@ -47,6 +48,8 @@ enum CLI {
         let output = CLIOutput(json: json)
         try Recorder.executeInvocation(arguments: arguments, command: command, output: output) {
             switch command {
+            case "onboard", "onboarding":
+                try onboard(args: Array(args.dropFirst()), output: output)
             case "doctor":
                 try doctor(output: output)
             case "state":
@@ -81,6 +84,60 @@ enum CLI {
         }
     }
 
+    static func onboard(args: [String], output: CLIOutput) throws {
+        var waitForReady = PermissionSupport.isInteractiveSession()
+        var timeoutSeconds = waitForReady ? 120 : 0
+        var requestPrompt = true
+        var openSettings = true
+        var index = 0
+
+        while index < args.count {
+            switch args[index] {
+            case "--wait":
+                waitForReady = true
+                if timeoutSeconds == 0 {
+                    timeoutSeconds = 120
+                }
+                index += 1
+            case "--no-wait":
+                waitForReady = false
+                timeoutSeconds = 0
+                index += 1
+            case "--timeout":
+                guard index + 1 < args.count else {
+                    throw CUAError(message: "usage: macos-cua onboard [--wait|--no-wait] [--timeout <seconds>] [--no-request] [--no-open]")
+                }
+                timeoutSeconds = try parseInt(args[index + 1], name: "timeout")
+                if timeoutSeconds < 0 {
+                    throw CUAError(message: "timeout must be >= 0")
+                }
+                waitForReady = timeoutSeconds > 0
+                index += 2
+            case "--no-request":
+                requestPrompt = false
+                index += 1
+            case "--no-open":
+                openSettings = false
+                index += 1
+            default:
+                throw CUAError(message: "usage: macos-cua onboard [--wait|--no-wait] [--timeout <seconds>] [--no-request] [--no-open]")
+            }
+        }
+
+        let progress: ((String) -> Void)? = output.json ? nil : { line in
+            print(line)
+        }
+        let shouldLogProgress = PermissionSupport.isInteractiveSession() && (waitForReady || requestPrompt || openSettings)
+        let result = PermissionSupport.onboarding(
+            requestPrompts: requestPrompt,
+            openSettingsPane: openSettings,
+            waitForReady: waitForReady,
+            timeoutSeconds: timeoutSeconds,
+            log: shouldLogProgress ? progress : nil
+        )
+        try output.emit(result.payload, lines: result.lines)
+    }
+
     static func doctor(output: CLIOutput) throws {
         let accessibility = WindowSupport.isAccessibilityTrusted()
         let screenRecording = ScreenshotSupport.screenCaptureAccess()
@@ -113,20 +170,26 @@ enum CLI {
             "screenRecording": screenRecording,
             "syntheticInputReady": accessibility,
             "screenshotReady": screenshotCheck,
+            "allReady": accessibility && screenRecording,
+            "onboardCommand": "macos-cua onboard",
             "frontmostApp": frontmostApp as Any,
             "frontmostWindow": frontmostWindow as Any,
             "actionSpace": actionSpace,
         ]
+        var lines = [
+            "Accessibility: \(accessibility ? "ready" : "missing")",
+            "Screen Recording: \(screenRecording ? "ready" : "missing")",
+            "Synthetic input: \(accessibility ? "ready" : "missing")",
+            "Screenshot check: \((screenshotCheck["ok"] as? Bool) == true ? "ok" : "failed")",
+            "Frontmost app: \((frontmostApp?["name"] as? String) ?? "n/a")",
+            "Frontmost window: \((frontmostWindow?["title"] as? String).flatMap { $0.isEmpty ? nil : $0 } ?? "<untitled>")",
+        ]
+        if !accessibility || !screenRecording {
+            lines.append("Next: run `macos-cua onboard` to request missing permissions.")
+        }
         try output.emit(
             payload,
-            lines: [
-                "Accessibility: \(accessibility ? "ready" : "missing")",
-                "Screen Recording: \(screenRecording ? "ready" : "missing")",
-                "Synthetic input: \(accessibility ? "ready" : "missing")",
-                "Screenshot check: \((screenshotCheck["ok"] as? Bool) == true ? "ok" : "failed")",
-                "Frontmost app: \((frontmostApp?["name"] as? String) ?? "n/a")",
-                "Frontmost window: \((frontmostWindow?["title"] as? String).flatMap { $0.isEmpty ? nil : $0 } ?? "<untitled>")",
-            ]
+            lines: lines
         )
     }
 

Sources/macos-cua/Core/Recorder.swift

@@ -60,7 +60,13 @@ struct RecorderEnvironment {
         return payload
     }
     var captureScreenshot: (_ target: ScreenshotTarget, _ path: String) throws -> [String: Any] = { target, path in
-        try ScreenshotSupport.capture(target: target, path: path)
+        try ScreenshotSupport.capture(
+            target: target,
+            path: path,
+            coordinateSpace: .screen,
+            coordinateFallback: false,
+            reportedBounds: ScreenshotSupport.bounds(for: target)
+        )
     }
 }
 

Sources/macos-cua/Desktop/InputSupport.swift

@@ -72,6 +72,7 @@ enum InputSupport {
     }
 
     static func post(_ event: CGEvent?) throws {
+        try PermissionSupport.require(.accessibility, for: "synthetic input")
         guard let event else {
             throw CUAError(message: "failed to create CGEvent")
         }