diff --git a/apps/desktop-gpui/src/devices.rs b/apps/desktop-gpui/src/devices.rs index 8b7be80e82..ad8c703f25 100644 --- a/apps/desktop-gpui/src/devices.rs +++ b/apps/desktop-gpui/src/devices.rs @@ -158,7 +158,7 @@ impl DeviceSnapshot { pub fn enumerate() -> Self { Self { cameras: list_cameras(), - microphones: list_microphones(), + microphones: list_microphone_names(), displays: list_displays(), windows: list_windows(), } @@ -553,23 +553,24 @@ pub fn camera_formats(device_id: &str) -> Result, String> { .formats) } -/// Mirrors `MicrophoneFeed::list_with_settings`: the default input device is -/// inserted first so it heads the list, then every other input device is -/// appended, deduped by name. +fn list_microphone_names() -> Vec { + cap_recording::feeds::microphone::MicrophoneFeed::list_names() + .into_iter() + .map(|name| MicrophoneOption { + name, + sample_rate: None, + channels: None, + }) + .collect() +} + fn list_microphones() -> Vec { // CPAL's configuration lookup opens an input AudioUnit and can prompt for consent. #[cfg(target_os = "macos")] if !crate::permissions::check_raw().is_some_and(|permissions| { permissions.microphone == crate::permissions::MediaAuthorization::Authorized }) { - return cap_recording::feeds::microphone::MicrophoneFeed::list_names() - .into_iter() - .map(|name| MicrophoneOption { - name, - sample_rate: None, - channels: None, - }) - .collect(); + return list_microphone_names(); } let host = cpal::default_host(); diff --git a/apps/desktop-gpui/src/main.rs b/apps/desktop-gpui/src/main.rs index f67e3c69e2..a3b6abe961 100644 --- a/apps/desktop-gpui/src/main.rs +++ b/apps/desktop-gpui/src/main.rs @@ -45,6 +45,10 @@ mod onboarding_audio; mod onboarding_window; mod permissions; mod permissions_ui; +#[cfg(test)] +mod picker_benchmark; +#[cfg(debug_assertions)] +mod picker_ui_benchmark; mod platform; mod presets; mod recording; @@ -644,6 +648,8 @@ fn main() { // the primary display and record for N seconds (or capture once). The // end-to-end check drives the recorder this way because unprivileged // synthetic clicks are dropped. + #[cfg(debug_assertions)] + picker_ui_benchmark::run(window_handle, cx); if let Ok(auto) = std::env::var("CAP_GPUI_AUTO_RECORD") && let Some((mode, secs)) = parse_auto_record(&auto) { diff --git a/apps/desktop-gpui/src/picker_benchmark.rs b/apps/desktop-gpui/src/picker_benchmark.rs new file mode 100644 index 0000000000..4b4bea8259 --- /dev/null +++ b/apps/desktop-gpui/src/picker_benchmark.rs @@ -0,0 +1,126 @@ +use std::time::{Duration, Instant}; + +use cap_recording::feeds::{camera, microphone::MicrophoneFeed}; +use kameo::Actor; + +use crate::devices::{DeviceSnapshot, InputSnapshot, TargetSnapshot}; + +fn measure(stage: &str, sample: usize, run: impl FnOnce() -> T) -> T { + let started = Instant::now(); + let result = run(); + println!( + "{}", + serde_json::json!({ + "stage": stage, + "sample": sample, + "elapsedMs": started.elapsed().as_secs_f64() * 1000.0, + }) + ); + result +} + +#[tokio::test] +#[ignore = "opens local capture devices to measure native picker dependencies"] +async fn native_picker_latency() -> anyhow::Result<()> { + for sample in 0..10 { + let snapshot = measure("gpui_startup_discovery", sample, DeviceSnapshot::enumerate); + println!( + "{}", + serde_json::json!({ + "sample": sample, + "cameras": snapshot.cameras.len(), + "microphones": snapshot.microphones.len(), + "displays": snapshot.displays.len(), + "windows": snapshot.windows.len(), + }) + ); + measure("camera_picker_discovery", sample, || { + InputSnapshot::cameras(&[]) + }); + measure( + "microphone_picker_discovery", + sample, + InputSnapshot::microphones, + ); + measure("tauri_device_inventory", sample, || { + ( + cap_camera::list_cameras().collect::>(), + MicrophoneFeed::list_names(), + ) + }); + measure("target_discovery", sample, TargetSnapshot::enumerate); + let names = MicrophoneFeed::list_names(); + measure("microphone_metadata_all_devices", sample, || { + names + .iter() + .map(|name| MicrophoneFeed::list().swap_remove(name)) + .collect::>() + }); + measure("microphone_metadata_named_devices", sample, || { + names + .iter() + .map(|name| MicrophoneFeed::device_with_settings(name, None)) + .collect::>() + }); + } + + let device_id = std::env::var("CAP_PICKER_BENCH_CAMERA_ID")?; + let camera_info = cap_camera::list_cameras() + .find(|camera| camera.device_id() == device_id) + .ok_or_else(|| anyhow::anyhow!("Benchmark camera is unavailable"))?; + let id = camera::DeviceOrModelID::from_info(&camera_info); + for (stage, reuse) in [ + ("camera_repeated_selection", false), + ("camera_reused_selection", true), + ] { + measure_camera_selection(stage, reuse, &id).await?; + } + Ok(()) +} + +async fn measure_camera_selection( + stage: &str, + reuse: bool, + id: &camera::DeviceOrModelID, +) -> anyhow::Result<()> { + let feed = camera::CameraFeed::spawn(camera::CameraFeed::default()); + let (sender, receiver) = flume::bounded(4); + feed.ask(camera::AddSender(sender)).await?; + for sample in 0..10 { + while receiver.try_recv().is_ok() {} + if sample > 0 { + tokio::time::timeout(Duration::from_secs(5), receiver.recv_async()).await??; + } + let started = Instant::now(); + let reused = reuse + && feed + .ask(camera::CheckInput { + id: id.clone(), + settings: None, + }) + .await?; + if !reused { + feed.ask(camera::SetInput { + id: id.clone(), + settings: None, + }) + .await? + .await?; + } + let ready_ms = started.elapsed().as_secs_f64() * 1000.0; + tokio::time::timeout(Duration::from_secs(5), receiver.recv_async()).await??; + println!( + "{}", + serde_json::json!({ + "stage": stage, + "sample": sample, + "elapsedMs": ready_ms, + "frameMs": started.elapsed().as_secs_f64() * 1000.0, + "reused": reused, + }) + ); + } + feed.ask(camera::RemoveInput).await?; + feed.stop_gracefully().await?; + Ok(()) +} diff --git a/apps/desktop-gpui/src/picker_ui_benchmark.rs b/apps/desktop-gpui/src/picker_ui_benchmark.rs new file mode 100644 index 0000000000..5eeef2ea4b --- /dev/null +++ b/apps/desktop-gpui/src/picker_ui_benchmark.rs @@ -0,0 +1,106 @@ +use std::time::{Duration, Instant}; + +use gpui::{App, WindowHandle}; + +use crate::{ + app_windows, + feeds::Feeds, + main_window::{MainWindow, TargetType}, +}; + +pub fn run(main: WindowHandle, cx: &mut App) { + let Some(output) = std::env::var_os("CAP_PICKER_BENCHMARK_OUTPUT") else { + return; + }; + cx.spawn(async move |cx| { + let delay = std::env::var("CAP_PICKER_BENCHMARK_DELAY_MS") + .ok() + .and_then(|delay| delay.parse().ok()) + .unwrap_or(2000); + cx.background_executor() + .timer(Duration::from_millis(delay)) + .await; + let mut samples = Vec::new(); + for sample in 0..6 { + let started = Instant::now(); + let kind = if sample % 2 == 0 { + TargetType::Display + } else { + TargetType::Window + }; + loop { + let enumerating = main + .update(cx, |view, _, _| view.is_enumerating_devices()) + .unwrap_or(true); + if !enumerating || started.elapsed() > Duration::from_secs(15) { + break; + } + cx.background_executor() + .timer(Duration::from_millis(5)) + .await; + } + let enumeration_ms = started.elapsed().as_secs_f64() * 1000.0; + let readiness = main.update(cx, |view, _, cx| { + view.arm_overlay(kind, cx); + Feeds::global(cx).read(cx).input_readiness() + }); + let mut errors = Vec::new(); + if let Ok(readiness) = readiness { + for input in [readiness.camera, readiness.microphone] + .into_iter() + .flatten() + { + if let Err(error) = input.await { + errors.push(error); + } + } + } + let inputs_ms = started.elapsed().as_secs_f64() * 1000.0; + let (sender, receiver) = flume::bounded(1); + cx.update(|cx| { + let overlay = cx + .global::() + .overlays + .first() + .map(|(_, window)| *window); + if let Some(overlay) = overlay { + let _ = overlay.update(cx, |_, window, cx| { + cx.on_next_frame(window, move |_, window, cx| { + cx.on_next_frame(window, move |_, _, _| { + let _ = sender.send(()); + }); + window.refresh(); + }); + window.refresh(); + }); + } + }); + let ready = matches!( + futures_util::future::select( + Box::pin(receiver.recv_async()), + Box::pin(cx.background_executor().timer(Duration::from_secs(20))), + ) + .await, + futures_util::future::Either::Left((Ok(()), _)) + ); + samples.push(serde_json::json!({ + "sample": sample, + "mode": if kind == TargetType::Display { "display" } else { "window" }, + "elapsedMs": started.elapsed().as_secs_f64() * 1000.0, + "enumerationMs": enumeration_ms, + "inputsMs": inputs_ms, + "ready": ready && errors.is_empty(), + "errors": errors, + })); + cx.update(app_windows::dismiss_target_overlays); + cx.background_executor() + .timer(Duration::from_millis(300)) + .await; + } + if let Err(error) = std::fs::write(output, serde_json::json!(samples).to_string()) { + tracing::error!(%error, "Could not write picker benchmark results"); + } + cx.update(|cx| cx.quit()); + }) + .detach(); +} diff --git a/apps/desktop/scripts/desktop-picker-performance.md b/apps/desktop/scripts/desktop-picker-performance.md new file mode 100644 index 0000000000..d3e87c8a33 --- /dev/null +++ b/apps/desktop/scripts/desktop-picker-performance.md @@ -0,0 +1,58 @@ +# Desktop picker latency + +The display and window pickers should reuse healthy selected devices. Restoring saved inputs must not open error dialogs for disconnected devices. Recording still waits for input setup, and missing selections remain saved for reconnection. + +## Benchmarks + +The ignored native benchmark runs real device discovery, microphone configuration lookups, camera setup, and camera frame delivery. It compares repeated camera setup with reuse and compares microphone lookup strategies in the same process. It opens the specified camera without creating recordings. + +```sh +cd apps/desktop-gpui +CAP_PICKER_BENCH_CAMERA_ID= cargo test -p cap-desktop-gpui --bin cap-gpui picker_benchmark::native_picker_latency -- --ignored --nocapture +``` + +Both debug apps also support `CAP_PICKER_BENCHMARK_OUTPUT=` and `CAP_PICKER_BENCHMARK_DELAY_MS=0`. They open and dismiss six alternating display/window pickers, write timings, and exit. The default delay is 2000 ms. Run them sequentially after compilation has finished. + +- Tauri measures from the target-mode request through native window creation, frontend initialization, input restoration, and two animation frames with the recording button enabled. +- GPUI includes pending device discovery and selected-input readiness, then waits for two rendered overlay frames. +- Sample zero measures the first picker in a fresh process. Later samples measure reopening. These are picker timings, not total process launch times. They are not directly interchangeable across UI implementations. + +Use isolated settings for these app runs. GPUI accepts `CAP_GPUI_APP_DATA_DIR`. Build Tauri with a separate `TAURI_CONFIG` identifier and seed only that identifier's application-data `store`. Set the recording mode to `studio` and select the test inputs in that store. Never point a benchmark at customer recordings or an active recording session. + +For a Tauri build with bundled frontend assets and no development server: + +```sh +pnpm --filter @cap/desktop build +TAURI_CONFIG='{"identifier":"so.cap.desktop.picker-benchmark","productName":"Cap Picker Benchmark","build":{"devUrl":null}}' cargo build -p cap-desktop --features tauri/custom-protocol +``` + +## What changed + +- Startup restoration bypasses the global mutation handler that displays native error dialogs. Explicit device selections retain their normal error handling. +- Matching simultaneous Tauri requests share pending setup. A different device, configuration, or camera-window request supersedes it. +- Repeated Tauri selections reuse a feed only when its identity and settings match and frames or audio samples arrived within 250 ms. Stalled feeds continue through setup and recovery. +- Camera-only mode retains its normal camera setup path, which also handles its different preview routing. +- GPUI startup lists microphone names without opening every device's audio configuration. Detailed configuration remains available when opening the microphone menu. +- Tauri microphone details configure only the requested device, avoiding a complete configuration scan for each row. +- The picker keeps its action label while readiness checks run. + +## Local comparison, 2026-09-09 + +On macOS with five cameras, five microphones, one display, and twelve listed windows; the built-in camera and microphone were selected. The baseline used the original Rust selection/discovery paths from `0c403be8a57f7ba4de67140273708a392c8b4050`. Both variants used the same frontend, timing hooks, settings, and build profile. Runs were sequential with no compilation in progress. + +| UI measurement | Before | After | +| --- | ---: | ---: | +| Tauri first display picker | 1,921 ms | 955 ms | +| Tauri reopening, median of five alternating display/window openings | 815 ms | 65 ms | +| GPUI first display picker | 1,046 ms | 679 ms | +| GPUI reopening, median of five alternating display/window openings | 28 ms | 24 ms | + +All 24 openings reached readiness. GPUI reopening was already fast; its improvement is in discovery during startup. Across the optimized Tauri run, native logs showed one microphone setup and one camera setup for all six openings. + +The final native comparison measured median microphone metadata lookup time of 1,579 ms for repeated full scans versus 187 ms for named lookups. Repeated camera setup took 433 ms; checking an already-streaming matching camera took 0.02 ms, with subsequent frames received in every sample. That last number is the feed acknowledgment time, not UI latency or first-frame startup. + +An additional Tauri run restored the unavailable `Shure MV7+` and `046d:08e5` selections. All six pickers reached readiness without blocking dialogs, and both selections remained persisted. Cold samples are individual observations, not percentile estimates. + +## Validation boundaries + +The local measurements exercise macOS hardware and debug Rust builds with bundled production frontend assets. They do not establish Windows hardware latency or release-package behavior. Keep cold samples separate from reopening samples, and preserve device counts and first-frame checks when comparing runs. diff --git a/apps/desktop/src-tauri/src/lib.rs b/apps/desktop/src-tauri/src/lib.rs index 6e07ad10ba..751f027c4e 100644 --- a/apps/desktop/src-tauri/src/lib.rs +++ b/apps/desktop/src-tauri/src/lib.rs @@ -35,6 +35,8 @@ mod macos_save_panel; mod notifications; mod panel_manager; mod permissions; +#[cfg(debug_assertions)] +mod picker_benchmark; mod platform; mod power_observer; mod presets; @@ -1210,6 +1212,7 @@ pub(crate) struct RequestedInput { revision: u64, pending: bool, error: Option, + configuration: Option, } impl RequestedInput { @@ -1219,6 +1222,7 @@ impl RequestedInput { revision: 0, pending: false, error: None, + configuration: None, } } @@ -1227,9 +1231,25 @@ impl RequestedInput { self.revision = self.revision.wrapping_add(1); self.pending = true; self.error = None; + self.configuration = None; self.revision } + fn begin_or_join(&mut self, value: Option, configuration: serde_json::Value) -> (u64, bool) + where + T: PartialEq, + { + if self.pending + && self.value == value + && self.configuration.as_ref() == Some(&configuration) + { + return (self.revision, true); + } + let revision = self.begin(value); + self.configuration = Some(configuration); + (revision, false) + } + fn finish(&mut self, revision: u64, result: &Result<(), String>) { if self.revision == revision { self.pending = false; @@ -1261,6 +1281,29 @@ impl RequestedInput { } } +async fn wait_for_existing_input( + revision: u64, + kind: &str, + read: impl Fn() -> RequestedInput, +) -> Result<(), String> { + tokio::time::timeout(Duration::from_secs(30), async { + loop { + let input = read(); + if input.revision != revision { + return Err(format!( + "{kind} selection was superseded by a newer request" + )); + } + if !input.pending { + return input.error.map_or(Ok(()), Err); + } + tokio::time::sleep(Duration::from_millis(10)).await; + } + }) + .await + .map_err(|_| format!("Timed out waiting for the selected {kind}"))? +} + #[derive(Clone)] pub(crate) struct RequestedInputs { microphone: RequestedInput, @@ -1360,6 +1403,7 @@ impl RequestedInputsState { struct AppliedMicrophoneInput { valid: bool, generation: u64, + settings: Option, } impl AppliedMicrophoneInput { @@ -1890,12 +1934,21 @@ async fn set_mic_input( label: Option, ) -> Result<(), String> { let requested = app_handle.state::(); - let revision = requested + let settings = label.as_ref().and_then(|label| { + recording_settings::RecordingSettingsStore::microphone_settings_for(&app_handle, label) + }); + let (revision, joined) = requested .inner .lock() .unwrap() .microphone - .begin(label.clone()); + .begin_or_join(label.clone(), serde_json::json!(settings)); + if joined { + return wait_for_existing_input(revision, "Microphone", || { + requested.inner.lock().unwrap().microphone.clone() + }) + .await; + } let result = async { check_requested_microphone_permission( label.as_deref(), @@ -2060,7 +2113,35 @@ async fn apply_mic_input( permissions::check_microphone_access, )?; - let (mic_feed, studio_handle, app_handle, applied_generation) = { + let settings = desired_label.as_ref().and_then(|label| { + recording_settings::RecordingSettingsStore::microphone_settings_for(app_handle, label) + }); + let reusable = { + let app = state.read().await; + (!matches!(app.recording_state, RecordingState::Active(_)) + && app.applied_mic_input.valid + && app.applied_mic_input.settings == settings + && app.selected_mic_label == desired_label) + .then(|| (app.mic_feed.clone(), app.applied_mic_input.generation)) + }; + if let Some((feed, generation)) = reusable + && let Some(label) = &desired_label + && MicrophoneFeed::list_names().contains(label) + && feed + .ask(microphone::CheckInput(label.clone())) + .await + .unwrap_or(false) + { + let app = state.read().await; + if requested.mic_is_current(revision) + && app.applied_mic_input.valid + && app.applied_mic_input.generation == generation + { + return Ok(()); + } + } + + let (mic_feed, studio_handle, applied_generation) = { let mut app = state.write().await; if !requested.mic_is_current(revision) { return Err("Microphone selection was superseded by a newer request".into()); @@ -2096,7 +2177,6 @@ async fn apply_mic_input( ( app.mic_feed.clone(), handle, - app.handle.clone(), app.applied_mic_input.generation, ) }; @@ -2131,10 +2211,6 @@ async fn apply_mic_input( "The Studio recording stopped while changing microphone input.".into(), ); } - let settings = recording_settings::RecordingSettingsStore::microphone_settings_for( - &app_handle, - label, - ); wait_for_microphone_setup(async { mic_feed .ask(feeds::microphone::SetInput { @@ -2214,6 +2290,7 @@ async fn apply_mic_input( return; } confirmed = true; + app.applied_mic_input.settings = settings; app.selected_mic_label = desired_label; cleared = app .disconnected_inputs @@ -2261,7 +2338,19 @@ async fn set_camera_input( skip_camera_window: Option, ) -> Result<(), String> { let requested = app_handle.state::(); - let revision = requested.inner.lock().unwrap().camera.begin(id.clone()); + let settings = id.as_ref().and_then(|id| { + recording_settings::RecordingSettingsStore::camera_settings_for(&app_handle, id) + }); + let (revision, joined) = requested.inner.lock().unwrap().camera.begin_or_join( + id.clone(), + serde_json::json!((settings, skip_camera_window.unwrap_or(false))), + ); + if joined { + return wait_for_existing_input(revision, "Camera", || { + requested.inner.lock().unwrap().camera.clone() + }) + .await; + } let result = async { check_requested_camera_permission(id.as_ref(), permissions::check_camera_access)?; let _operation = requested.operation.lock().await; @@ -2358,6 +2447,31 @@ async fn apply_camera_input( )); } + let settings = id.as_ref().and_then(|id| { + recording_settings::RecordingSettingsStore::camera_settings_for(app_handle, id) + }); + if !recording_active + && !skip_camera_window + && camera_in_use + && id == current_id + && let Some(id) = &id + && camera_feed + .ask(feeds::camera::CheckInput { + id: id.clone(), + settings, + }) + .await + .unwrap_or(false) + { + if !requested.camera_is_current(revision) { + return Err("Camera selection was superseded by a newer request".into()); + } + if !camera_window_is_visible { + show_requested_camera_window(app_handle, revision).await?; + } + return Ok(()); + } + if let Some(handle) = &studio_handle { handle .set_camera_feed(None) @@ -2403,8 +2517,6 @@ async fn apply_camera_input( }) { return Err("Camera selection was superseded by a newer request".into()); } - let settings = - recording_settings::RecordingSettingsStore::camera_settings_for(app_handle, id); let (camera_ws_sender, camera_preview_sender, use_ws_preview) = { let app = &mut *state.write().await; let use_ws_preview = !(camera_window_is_visible @@ -7098,6 +7210,8 @@ pub async fn run(recording_logging_handle: LoggingHandle, logs_dir: PathBuf) { move |_| { app.state::().set_ready(true); tracing::info!("Main window frontend ready"); + #[cfg(debug_assertions)] + picker_benchmark::run(app.clone()); gpu_context::prewarm_gpu(); tokio::task::spawn_blocking(cap_rendering::prewarm_fonts); tokio::spawn(screenshot_editor::prewarm_screenshot_renderer()); @@ -9451,6 +9565,7 @@ mod applied_microphone_tests { applied: AppliedMicrophoneInput { valid: true, generation: 0, + settings: None, }, selected: Some("A".into()), actual: Some("A".into()), @@ -9771,7 +9886,54 @@ mod microphone_permission_tests { #[cfg(test)] mod requested_inputs_tests { - use super::{RequestedInput, RequestedInputsState}; + use super::{RequestedInput, RequestedInputsState, wait_for_existing_input}; + + #[test] + fn matching_pending_requests_share_setup_but_changed_configuration_supersedes_it() { + let mut input = RequestedInput::new(None::); + let settings = serde_json::json!({ "sampleRate": 48_000 }); + let (first, joined) = input.begin_or_join(Some("mic".into()), settings.clone()); + assert!(!joined); + assert_eq!( + input.begin_or_join(Some("mic".into()), settings.clone()), + (first, true) + ); + let (second, joined) = input.begin_or_join( + Some("mic".into()), + serde_json::json!({ "sampleRate": 44_100 }), + ); + assert!(!joined); + assert_ne!(first, second); + input.finish(first, &Ok(())); + assert!(input.pending); + input.finish(second, &Err("disconnected".into())); + let (retry, joined) = input.begin_or_join(Some("mic".into()), settings); + assert!(!joined); + assert_ne!(retry, second); + } + + #[tokio::test] + async fn joined_input_requests_report_completion_failure_and_supersession() { + for result in [Ok(()), Err("device unavailable".to_string())] { + let input = std::sync::Mutex::new(RequestedInput::new(None::)); + let revision = input.lock().unwrap().begin(Some("mic".into())); + let waiter = + wait_for_existing_input(revision, "Microphone", || input.lock().unwrap().clone()); + let finish = async { + tokio::task::yield_now().await; + input.lock().unwrap().finish(revision, &result); + }; + let (observed, ()) = tokio::join!(waiter, finish); + assert_eq!(observed, result); + input.lock().unwrap().begin(Some("different".into())); + assert!( + wait_for_existing_input(revision, "Microphone", || input.lock().unwrap().clone()) + .await + .unwrap_err() + .contains("superseded") + ); + } + } #[test] fn persisted_intent_is_available_before_preview_setup() { diff --git a/apps/desktop/src-tauri/src/picker-benchmark.js b/apps/desktop/src-tauri/src/picker-benchmark.js new file mode 100644 index 0000000000..d675f323c9 --- /dev/null +++ b/apps/desktop/src-tauri/src/picker-benchmark.js @@ -0,0 +1,28 @@ +(() => { + let armed = true; + let frames = 0; + const invoke = window.__TAURI_INTERNALS__.invoke; + globalThis.__capArmPickerBenchmark = () => { + armed = true; + frames = 0; + }; + const tick = () => { + const button = Array.from( + document.querySelectorAll('[data-disabled="false"]'), + ).find((element) => element.textContent.includes("Start Recording")); + if (armed && button?.getBoundingClientRect().width > 0) { + frames += 1; + if (frames === 2) { + armed = false; + void invoke("plugin:event|emit", { + event: "cap-picker-benchmark-ready", + payload: { navigationMs: performance.now() }, + }); + } + } else { + frames = 0; + } + requestAnimationFrame(tick); + }; + requestAnimationFrame(tick); +})(); diff --git a/apps/desktop/src-tauri/src/picker_benchmark.rs b/apps/desktop/src-tauri/src/picker_benchmark.rs new file mode 100644 index 0000000000..7ec14003c2 --- /dev/null +++ b/apps/desktop/src-tauri/src/picker_benchmark.rs @@ -0,0 +1,67 @@ +use std::time::{Duration, Instant}; + +use tauri::{Listener, Manager}; +use tauri_specta::Event; + +use crate::{RequestSetTargetMode, recording_settings::RecordingTargetMode}; + +pub const SCRIPT: &str = include_str!("picker-benchmark.js"); + +pub fn enabled() -> bool { + std::env::var_os("CAP_PICKER_BENCHMARK_OUTPUT").is_some() +} + +pub fn run(app: tauri::AppHandle) { + let Some(output) = std::env::var_os("CAP_PICKER_BENCHMARK_OUTPUT") else { + return; + }; + tokio::spawn(async move { + let delay = std::env::var("CAP_PICKER_BENCHMARK_DELAY_MS") + .ok() + .and_then(|delay| delay.parse().ok()) + .unwrap_or(2000); + tokio::time::sleep(Duration::from_millis(delay)).await; + let (sender, mut receiver) = tokio::sync::mpsc::unbounded_channel(); + let listener = app.listen_any("cap-picker-benchmark-ready", move |event| { + let _ = sender.send(event.payload().to_owned()); + }); + let mut samples = Vec::new(); + for sample in 0..6 { + let mode = if sample % 2 == 0 { + RecordingTargetMode::Display + } else { + RecordingTargetMode::Window + }; + for (label, window) in app.webview_windows() { + if label.starts_with("target-select-overlay") { + let _ = window.eval("globalThis.__capArmPickerBenchmark?.()"); + } + } + let started = Instant::now(); + let _ = RequestSetTargetMode { + target_mode: Some(mode), + display_id: None, + } + .emit(&app); + let result = tokio::time::timeout(Duration::from_secs(20), receiver.recv()).await; + samples.push(serde_json::json!({ + "sample": sample, + "mode": mode, + "elapsedMs": started.elapsed().as_secs_f64() * 1000.0, + "ready": matches!(result, Ok(Some(_))), + "frontend": result.ok().flatten().and_then(|payload| serde_json::from_str::(&payload).ok()), + })); + let _ = RequestSetTargetMode { + target_mode: None, + display_id: None, + } + .emit(&app); + tokio::time::sleep(Duration::from_millis(300)).await; + } + app.unlisten(listener); + if let Err(error) = std::fs::write(output, serde_json::json!(samples).to_string()) { + tracing::error!(%error, "Could not write picker benchmark results"); + } + app.exit(0); + }); +} diff --git a/apps/desktop/src-tauri/src/recording.rs b/apps/desktop/src-tauri/src/recording.rs index e5a6039a88..2a3a178870 100644 --- a/apps/desktop/src-tauri/src/recording.rs +++ b/apps/desktop/src-tauri/src/recording.rs @@ -1073,18 +1073,15 @@ pub fn get_microphone_info(name: String) -> Option { return None; } - microphone::MicrophoneFeed::list() - .into_iter() - .find(|(n, _)| *n == name) - .map(|(name, (device, config))| { - let formats = microphone_format_infos(&device); - MicrophoneInfo { - name, - sample_rate: config.sample_rate().0, - channels: config.channels(), - formats, - } - }) + microphone::MicrophoneFeed::device_with_settings(&name, None).map(|(device, config)| { + let formats = microphone_format_infos(&device); + MicrophoneInfo { + name, + sample_rate: config.sample_rate().0, + channels: config.channels(), + formats, + } + }) } fn microphone_format_infos(device: &cpal::Device) -> Vec { diff --git a/apps/desktop/src-tauri/src/windows.rs b/apps/desktop/src-tauri/src/windows.rs index 92ba119ade..c0a1663422 100644 --- a/apps/desktop/src-tauri/src/windows.rs +++ b/apps/desktop/src-tauri/src/windows.rs @@ -1989,6 +1989,12 @@ impl ShowCapWindow { "window.__CAP__ = window.__CAP__ ?? {{}}; window.__CAP__.cameraWsPort = {camera_ws_port};" )); + #[cfg(debug_assertions)] + if crate::picker_benchmark::enabled() { + window_builder = + window_builder.initialization_script(crate::picker_benchmark::SCRIPT); + } + #[cfg(target_os = "macos")] { let position = display.raw_handle().logical_position(); diff --git a/apps/desktop/src/routes/(window-chrome)/new-main/index.tsx b/apps/desktop/src/routes/(window-chrome)/new-main/index.tsx index 1cfb085c8f..e3fb64eac9 100644 --- a/apps/desktop/src/routes/(window-chrome)/new-main/index.tsx +++ b/apps/desktop/src/routes/(window-chrome)/new-main/index.tsx @@ -2583,8 +2583,8 @@ function Page() { if (!targetMode) scheduleTargetListPrewarm(); if (rawOptions.micName) { - setMicInput - .mutateAsync(rawOptions.micName) + commands + .setMicInput(rawOptions.micName) .catch((error) => console.error("Failed to set mic input:", error)); } @@ -2598,7 +2598,7 @@ function Page() { !cameraRestoreDisposed && getCameraRevision() === restoreRevision && JSON.stringify(rawOptions.cameraID) === cameraKey, - () => setCamera.mutateAsync({ model }), + () => setCamera.rawMutate(model), ).catch((error) => console.error("Failed to restore camera input:", error), ); diff --git a/apps/desktop/src/routes/target-select-overlay.tsx b/apps/desktop/src/routes/target-select-overlay.tsx index 209e0c4d18..42c116b904 100644 --- a/apps/desktop/src/routes/target-select-overlay.tsx +++ b/apps/desktop/src/routes/target-select-overlay.tsx @@ -1934,30 +1934,36 @@ function RecordingControls(props: { const permissions = createMemo(() => devices.data?.permissions); const setMicInput = createMicrophoneMutation(); const setCamera = createCameraMutation(); + const [restoringInputs, setRestoringInputs] = createSignal(true); onMount(async () => { - if (rawOptions.micName) { - setMicInput - .mutateAsync(rawOptions.micName) - .catch((error) => console.error("Failed to set mic input:", error)); - } + const restoreMicrophone = rawOptions.micName + ? commands + .setMicInput(rawOptions.micName) + .catch((error) => + console.error("Failed to restore mic input:", error), + ) + : Promise.resolve(); const isCameraOnly = props.target.variant === "cameraOnly"; - if (rawOptions.cameraID && "ModelID" in rawOptions.cameraID) - await setCamera.mutateAsync({ - model: { ModelID: rawOptions.cameraID.ModelID }, - skipCameraWindow: isCameraOnly, - }); - else if (rawOptions.cameraID && "DeviceID" in rawOptions.cameraID) - await setCamera.mutateAsync({ - model: { DeviceID: rawOptions.cameraID.DeviceID }, - skipCameraWindow: isCameraOnly, - }); + const restoreCamera = async () => { + if (rawOptions.cameraID) { + await setCamera.rawMutate({ ...rawOptions.cameraID }, isCameraOnly); + } - if (isCameraOnly) { - const win = await getCameraWindow(); - if (win) win.close(); - } + if (isCameraOnly) { + const win = await getCameraWindow(); + if (win) await win.close(); + } + }; + + await Promise.all([ + restoreMicrophone, + restoreCamera().catch((error) => + console.error("Failed to restore camera input:", error), + ), + ]); + if (!controlsDisposed) setRestoringInputs(false); }); const selectedCamera = createMemo(() => { @@ -2007,6 +2013,7 @@ function RecordingControls(props: { const startLoading = () => devices.isPending || recordingStartSafety.isPending || + restoringInputs() || setMicInput.isPending || setCamera.isPending; const startDisabled = () => !!props.disabled || startLoading(); @@ -2309,7 +2316,6 @@ function RecordingControls(props: { {(() => { if (rawOptions.mode === "instant" && !auth.data) return "Sign In To Use"; - if (startLoading()) return "Preparing..."; if (rawOptions.mode === "screenshot") return "Take Screenshot"; return "Start Recording"; diff --git a/crates/recording/src/feeds/camera.rs b/crates/recording/src/feeds/camera.rs index bf40a88b4f..28d822b4bf 100644 --- a/crates/recording/src/feeds/camera.rs +++ b/crates/recording/src/feeds/camera.rs @@ -135,6 +135,8 @@ struct AttachedState { id: DeviceOrModelID, camera_info: cap_camera::CameraInfo, video_info: VideoInfo, + settings: Option, + last_frame_at: Option, done_tx: mpsc::SyncSender<()>, pending_release: Option>, } @@ -145,6 +147,7 @@ impl AttachedState { done_tx, camera_info, video_info, + settings, .. } = data; @@ -152,6 +155,8 @@ impl AttachedState { id, camera_info, video_info, + settings, + last_frame_at: None, done_tx, pending_release: None, } @@ -162,12 +167,15 @@ impl AttachedState { done_tx, camera_info, video_info, + settings, .. } = data; self.id = id; self.camera_info = camera_info; self.video_info = video_info; + self.settings = settings; + self.last_frame_at = None; self.done_tx = done_tx; } @@ -282,6 +290,11 @@ pub struct SetInput { pub settings: Option, } +pub struct CheckInput { + pub id: DeviceOrModelID, + pub settings: Option, +} + pub struct RemoveInput; pub struct AddSender(pub flume::Sender); @@ -307,6 +320,7 @@ struct InputConnected { done_tx: SyncSender<()>, camera_info: cap_camera::CameraInfo, video_info: VideoInfo, + settings: Option, } type ReadyFuture = Shared>>; @@ -328,6 +342,7 @@ struct LockedCameraInputReconnected { id: DeviceOrModelID, camera_info: cap_camera::CameraInfo, video_info: VideoInfo, + settings: Option, done_tx: SyncSender<()>, } @@ -506,6 +521,7 @@ fn spawn_camera_setup(args: CameraSetupArgs) -> (ReadyFuture, SyncSender<()>) { id: id.clone(), camera_info: camera_info.clone(), video_info, + settings, done_tx: done_tx_thread.clone(), }; @@ -525,6 +541,7 @@ fn spawn_camera_setup(args: CameraSetupArgs) -> (ReadyFuture, SyncSender<()>) { id: id.clone(), camera_info, video_info, + settings, done_tx: done_tx_thread.clone(), }) .await; @@ -1176,6 +1193,26 @@ async fn setup_camera( }) } +impl Message for CameraFeed { + type Reply = bool; + + async fn handle(&mut self, msg: CheckInput, _: &mut Context) -> Self::Reply { + let State::Open(OpenState { + connecting: None, + attached: Some(attached), + }) = &self.state + else { + return false; + }; + !self.setup_cancel.is_cancelled() + && attached.id == msg.id + && attached.settings == msg.settings + && attached + .last_frame_at + .is_some_and(|received| received.elapsed() < Duration::from_millis(250)) + } +} + impl Message for CameraFeed { type Reply = Result>, SetInputError>; @@ -1511,6 +1548,13 @@ impl Message for CameraFeed { type Reply = (); async fn handle(&mut self, msg: NewFrame, _: &mut Context) -> Self::Reply { + if let State::Open(OpenState { + connecting: None, + attached: Some(attached), + }) = &mut self.state + { + attached.last_frame_at = Some(std::time::Instant::now()); + } let frame_num = CAMERA_FRAME_COUNTER.fetch_add(1, std::sync::atomic::Ordering::Relaxed); if send_frame_to_camera_senders(&mut self.senders, msg.0, frame_num, "Camera") { @@ -1531,6 +1575,13 @@ impl Message for CameraFeed { msg: NewNativeFrame, _: &mut Context, ) -> Self::Reply { + if let State::Open(OpenState { + connecting: None, + attached: Some(attached), + }) = &mut self.state + { + attached.last_frame_at = Some(std::time::Instant::now()); + } let frame_num = NATIVE_CAMERA_FRAME_COUNTER.fetch_add(1, std::sync::atomic::Ordering::Relaxed); @@ -1712,6 +1763,7 @@ impl Message for CameraFeed { done_tx: msg.done_tx, camera_info: msg.camera_info, video_info: msg.video_info, + settings: msg.settings, }, ); true @@ -1793,12 +1845,95 @@ mod tests { })) .unwrap(), video_info: VideoInfo::from_raw_ffmpeg(ffmpeg::format::Pixel::BGRA, 16, 16, 30), + settings: None, }; (connection, done_rx) } struct AttachedCamera; + #[tokio::test] + async fn camera_reuse_requires_matching_settings_and_recent_frames() { + for (age, matching_id, matching_settings, cancelled, expected) in [ + (Some(0), true, true, false, true), + (None, true, true, false, false), + (Some(500), true, true, false, false), + (Some(0), false, true, false, false), + (Some(0), true, false, false, false), + (Some(0), true, true, true, false), + ] { + let (connection, stopped) = test_camera_connection(1); + let id = connection.id.clone(); + let mut attached = AttachedState::new(id.clone(), connection); + attached.last_frame_at = age + .and_then(|age| std::time::Instant::now().checked_sub(Duration::from_millis(age))); + let camera = CameraFeed { + state: State::Open(OpenState { + connecting: None, + attached: Some(attached), + }), + ..CameraFeed::default() + }; + if cancelled { + camera.setup_cancel.cancel(); + } + let feed = CameraFeed::spawn(camera); + let reusable = feed + .ask(CheckInput { + id: if matching_id { + id + } else { + DeviceOrModelID::DeviceID("different-camera".into()) + }, + settings: (!matching_settings).then_some(CameraDeviceSettings { + width: Some(1280), + height: Some(720), + frame_rate: Some(30.0), + }), + }) + .await + .unwrap(); + assert_eq!(reusable, expected); + assert!(matches!(stopped.try_recv(), Err(mpsc::TryRecvError::Empty))); + feed.kill(); + feed.wait_for_stop().await; + } + } + + #[tokio::test] + async fn camera_reuse_does_not_bypass_a_pending_selection_or_recording_lock() { + for locked in [false, true] { + let (connection, _stopped) = test_camera_connection(1); + let id = connection.id.clone(); + let mut attached = AttachedState::new(id.clone(), connection.clone()); + attached.last_frame_at = Some(std::time::Instant::now()); + let token = Arc::new(()); + let state = if locked { + State::Locked { + inner: attached, + token: Arc::downgrade(&token), + } + } else { + State::Open(OpenState { + connecting: Some(ConnectingState { + id: id.clone(), + generation: 2, + ready: futures::future::pending().boxed(), + done_tx: connection.done_tx, + }), + attached: Some(attached), + }) + }; + let feed = CameraFeed::spawn(CameraFeed { + state, + ..CameraFeed::default() + }); + assert!(!feed.ask(CheckInput { id, settings: None }).await.unwrap()); + feed.kill(); + feed.wait_for_stop().await; + } + } + impl Message for CameraFeed { type Reply = Option; @@ -2268,6 +2403,7 @@ mod tests { id: replacement.id, camera_info: replacement.camera_info, video_info: replacement.video_info, + settings: replacement.settings, done_tx: replacement.done_tx, }) .await diff --git a/crates/recording/src/feeds/microphone.rs b/crates/recording/src/feeds/microphone.rs index 6eadd4053c..73cea2dd92 100644 --- a/crates/recording/src/feeds/microphone.rs +++ b/crates/recording/src/feeds/microphone.rs @@ -154,7 +154,6 @@ pub struct MicrophoneDeviceSettings { #[derive(Clone)] pub struct MicrophoneSamples { - #[cfg(any(target_os = "linux", windows))] pub(crate) stream_id: u32, pub data: Vec, pub format: SampleFormat, @@ -872,6 +871,7 @@ pub struct MicrophoneFeed { input_id_counter: u32, lock_generation: u64, state: State, + last_samples: Option<(u32, Instant)>, senders: Vec, error_sender: flume::Sender, dropped_message_count: Arc, @@ -1132,6 +1132,7 @@ impl MicrophoneFeed { connecting: None, attached: None, }), + last_samples: None, senders: Vec::new(), error_sender, dropped_message_count: Arc::new(AtomicU64::new(0)), @@ -1186,7 +1187,7 @@ impl MicrophoneFeed { device_map } - fn device_with_settings( + pub fn device_with_settings( label: &str, settings: Option<&MicrophoneDeviceSettings>, ) -> Option<(Device, SupportedStreamConfig)> { @@ -1413,7 +1414,6 @@ impl MicrophoneFeed { ), ); let samples = MicrophoneSamples { - #[cfg(any(target_os = "linux", windows))] stream_id: id, data: data.bytes().to_vec(), format: data.sample_format(), @@ -1878,6 +1878,26 @@ pub struct SetInput { pub settings: Option, } +pub struct CheckInput(pub String); + +impl Message for MicrophoneFeed { + type Reply = bool; + + async fn handle(&mut self, msg: CheckInput, _: &mut Context) -> Self::Reply { + let State::Open(OpenState { + connecting: None, + attached: Some(attached), + }) = &self.state + else { + return false; + }; + attached.label == msg.0 + && self.last_samples.is_some_and(|(id, received)| { + id == attached.id && received.elapsed() < Duration::from_millis(250) + }) + } +} + pub struct RemoveInput; pub struct AddSender(pub flume::Sender); @@ -2562,6 +2582,7 @@ impl Message for MicrophoneFeed { ) -> Self::Reply { let mut to_remove = vec![]; let now = Instant::now(); + self.last_samples = Some((msg.stream_id, now)); let stall_emit_interval = Duration::from_secs(5); for (i, sender) in self.senders.iter_mut().enumerate() { @@ -3171,6 +3192,46 @@ mod tests { assert_eq!(configured, ["requested microphone"]); } + #[tokio::test] + async fn microphone_reuse_requires_recent_samples_from_the_selected_stream() { + for (stream_id, age, label, expected) in [ + (1, Some(0), "selected", true), + (2, Some(0), "selected", false), + (1, Some(500), "selected", false), + (1, None, "selected", false), + (1, Some(0), "different", false), + ] { + let (errors, _errors_rx) = flume::bounded(1); + let (done_tx, done_rx) = mpsc::sync_channel(1); + let mut microphone = MicrophoneFeed::new(errors); + microphone.state = State::Open(OpenState { + connecting: None, + attached: Some(AttachedState { + id: 1, + label: "selected".into(), + config: SupportedStreamConfig::new( + 1, + cpal::SampleRate(48_000), + cpal::SupportedBufferSize::Unknown, + SampleFormat::F32, + ), + buffer_size_frames: None, + done_tx, + }), + }); + microphone.last_samples = age.and_then(|age| { + Instant::now() + .checked_sub(Duration::from_millis(age)) + .map(|received| (stream_id, received)) + }); + let feed = MicrophoneFeed::spawn(microphone); + assert_eq!(feed.ask(CheckInput(label.into())).await.unwrap(), expected); + assert!(matches!(done_rx.try_recv(), Err(mpsc::TryRecvError::Empty))); + feed.kill(); + feed.wait_for_stop().await; + } + } + #[test] fn named_microphone_selection_preserves_first_usable_duplicate() { let mut configured = Vec::new();