Skip to content

Commit bfecade

Browse files
Schreezerfactory-droid[bot]cjpais
authored
Apple intel integration (#391)
* feat: add Apple Intelligence post-processing provider Co-authored-by: factory-droid[bot] <138933559+factory-droid[bot]@users.noreply.github.com> * feat: guide apple intelligence output Co-authored-by: factory-droid[bot] <138933559+factory-droid[bot]@users.noreply.github.com> * fix(build): add fallback stub for Apple Intelligence on older SDKs - Checks if 'FoundationModels.framework' is present in the macOS SDK. - If missing, compiles a stub Swift file that returns 'unavailable' errors instead of failing the build. - Prevents build errors on older Xcode versions (or macOS versions < 26.0) where the macros are not supported. * fix(ui): hide Apple Intelligence option when unavailable - Checks for runtime availability of Apple Intelligence (via 'check_apple_intelligence_availability') in 'settings.rs'. - Only adds the Apple Intelligence provider to the default settings if it is actually available on the device. - Ensures the option does not appear in the UI for unsupported Macs (e.g., Intel or older macOS versions), even if the app was built with support enabled. * move some files around as well as some ui text * format --------- Co-authored-by: factory-droid[bot] <138933559+factory-droid[bot]@users.noreply.github.com> Co-authored-by: CJ Pais <[email protected]>
1 parent 933c1b1 commit bfecade

File tree

11 files changed

+625
-60
lines changed

11 files changed

+625
-60
lines changed

‎src-tauri/build.rs‎

Lines changed: 125 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,128 @@
11
fn main() {
2+
#[cfg(all(target_os = "macos", target_arch = "aarch64"))]
3+
build_apple_intelligence_bridge();
4+
25
tauri_build::build()
36
}
7+
8+
#[cfg(all(target_os = "macos", target_arch = "aarch64"))]
9+
fn build_apple_intelligence_bridge() {
10+
use std::env;
11+
use std::path::{Path, PathBuf};
12+
use std::process::Command;
13+
14+
const REAL_SWIFT_FILE: &str = "swift/apple_intelligence.swift";
15+
const STUB_SWIFT_FILE: &str = "swift/apple_intelligence_stub.swift";
16+
const BRIDGE_HEADER: &str = "swift/apple_intelligence_bridge.h";
17+
18+
println!("cargo:rerun-if-changed={REAL_SWIFT_FILE}");
19+
println!("cargo:rerun-if-changed={STUB_SWIFT_FILE}");
20+
println!("cargo:rerun-if-changed={BRIDGE_HEADER}");
21+
22+
let out_dir = PathBuf::from(env::var("OUT_DIR").expect("OUT_DIR not set"));
23+
let object_path = out_dir.join("apple_intelligence.o");
24+
let static_lib_path = out_dir.join("libapple_intelligence.a");
25+
26+
let sdk_path = String::from_utf8(
27+
Command::new("xcrun")
28+
.args(["--sdk", "macosx", "--show-sdk-path"])
29+
.output()
30+
.expect("Failed to locate macOS SDK")
31+
.stdout,
32+
)
33+
.expect("SDK path is not valid UTF-8")
34+
.trim()
35+
.to_string();
36+
37+
// Check if the SDK supports FoundationModels (required for Apple Intelligence)
38+
let framework_path =
39+
Path::new(&sdk_path).join("System/Library/Frameworks/FoundationModels.framework");
40+
let has_foundation_models = framework_path.exists();
41+
42+
let source_file = if has_foundation_models {
43+
println!("cargo:warning=Building with Apple Intelligence support.");
44+
REAL_SWIFT_FILE
45+
} else {
46+
println!("cargo:warning=Apple Intelligence SDK not found. Building with stubs.");
47+
STUB_SWIFT_FILE
48+
};
49+
50+
if !Path::new(source_file).exists() {
51+
panic!("Source file {} is missing!", source_file);
52+
}
53+
54+
let swiftc_path = String::from_utf8(
55+
Command::new("xcrun")
56+
.args(["--find", "swiftc"])
57+
.output()
58+
.expect("Failed to locate swiftc")
59+
.stdout,
60+
)
61+
.expect("swiftc path is not valid UTF-8")
62+
.trim()
63+
.to_string();
64+
65+
let toolchain_swift_lib = Path::new(&swiftc_path)
66+
.parent()
67+
.and_then(|p| p.parent())
68+
.map(|root| root.join("lib/swift/macosx"))
69+
.expect("Unable to determine Swift toolchain lib directory");
70+
let sdk_swift_lib = Path::new(&sdk_path).join("usr/lib/swift");
71+
72+
let status = Command::new("xcrun")
73+
.args([
74+
"swiftc",
75+
"-target",
76+
"arm64-apple-macosx26.0",
77+
"-sdk",
78+
&sdk_path,
79+
"-O",
80+
"-import-objc-header",
81+
BRIDGE_HEADER,
82+
"-c",
83+
source_file,
84+
"-o",
85+
object_path
86+
.to_str()
87+
.expect("Failed to convert object path to string"),
88+
])
89+
.status()
90+
.expect("Failed to invoke swiftc for Apple Intelligence bridge");
91+
92+
if !status.success() {
93+
panic!("swiftc failed to compile {source_file}");
94+
}
95+
96+
let status = Command::new("libtool")
97+
.args([
98+
"-static",
99+
"-o",
100+
static_lib_path
101+
.to_str()
102+
.expect("Failed to convert static lib path to string"),
103+
object_path
104+
.to_str()
105+
.expect("Failed to convert object path to string"),
106+
])
107+
.status()
108+
.expect("Failed to create static library for Apple Intelligence bridge");
109+
110+
if !status.success() {
111+
panic!("libtool failed for Apple Intelligence bridge");
112+
}
113+
114+
println!("cargo:rustc-link-search=native={}", out_dir.display());
115+
println!("cargo:rustc-link-lib=static=apple_intelligence");
116+
println!(
117+
"cargo:rustc-link-search=native={}",
118+
toolchain_swift_lib.display()
119+
);
120+
println!("cargo:rustc-link-search=native={}", sdk_swift_lib.display());
121+
println!("cargo:rustc-link-lib=framework=Foundation");
122+
123+
if has_foundation_models {
124+
println!("cargo:rustc-link-lib=framework=FoundationModels");
125+
}
126+
127+
println!("cargo:rustc-link-arg=-Wl,-rpath,/usr/lib/swift");
128+
}

‎src-tauri/src/actions.rs‎

Lines changed: 45 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,10 @@
1+
#[cfg(all(target_os = "macos", target_arch = "aarch64"))]
2+
use crate::apple_intelligence;
13
use crate::audio_feedback::{play_feedback_sound, play_feedback_sound_blocking, SoundType};
24
use crate::managers::audio::AudioRecordingManager;
35
use crate::managers::history::HistoryManager;
46
use crate::managers::transcription::TranscriptionManager;
5-
use crate::settings::{get_settings, AppSettings};
7+
use crate::settings::{get_settings, AppSettings, APPLE_INTELLIGENCE_PROVIDER_ID};
68
use crate::shortcut;
79
use crate::tray::{change_tray_icon, TrayIconState};
810
use crate::utils::{self, show_recording_overlay, show_transcribing_overlay};
@@ -86,12 +88,6 @@ async fn maybe_post_process_transcription(
8688
return None;
8789
}
8890

89-
let api_key = settings
90-
.post_process_api_keys
91-
.get(&provider.id)
92-
.cloned()
93-
.unwrap_or_default();
94-
9591
debug!(
9692
"Starting LLM post-processing with provider '{}' (model: {})",
9793
provider.id, model
@@ -101,6 +97,48 @@ async fn maybe_post_process_transcription(
10197
let processed_prompt = prompt.replace("${output}", transcription);
10298
debug!("Processed prompt length: {} chars", processed_prompt.len());
10399

100+
if provider.id == APPLE_INTELLIGENCE_PROVIDER_ID {
101+
#[cfg(all(target_os = "macos", target_arch = "aarch64"))]
102+
{
103+
if !apple_intelligence::check_apple_intelligence_availability() {
104+
debug!("Apple Intelligence selected but not currently available on this device");
105+
return None;
106+
}
107+
108+
let token_limit = model.trim().parse::<i32>().unwrap_or(0);
109+
return match apple_intelligence::process_text(&processed_prompt, token_limit) {
110+
Ok(result) => {
111+
if result.trim().is_empty() {
112+
debug!("Apple Intelligence returned an empty response");
113+
None
114+
} else {
115+
debug!(
116+
"Apple Intelligence post-processing succeeded. Output length: {} chars",
117+
result.len()
118+
);
119+
Some(result)
120+
}
121+
}
122+
Err(err) => {
123+
error!("Apple Intelligence post-processing failed: {}", err);
124+
None
125+
}
126+
};
127+
}
128+
129+
#[cfg(not(all(target_os = "macos", target_arch = "aarch64")))]
130+
{
131+
debug!("Apple Intelligence provider selected on unsupported platform");
132+
return None;
133+
}
134+
}
135+
136+
let api_key = settings
137+
.post_process_api_keys
138+
.get(&provider.id)
139+
.cloned()
140+
.unwrap_or_default();
141+
104142
// Create OpenAI-compatible client
105143
let client = match crate::llm_client::create_client(&provider, api_key) {
106144
Ok(client) => client,

‎src-tauri/src/apple_intelligence.rs‎

Lines changed: 71 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,71 @@
1+
use std::ffi::{CStr, CString};
2+
use std::os::raw::{c_char, c_int};
3+
4+
// Define the response structure from Swift
5+
#[repr(C)]
6+
pub struct AppleLLMResponse {
7+
pub response: *mut c_char,
8+
pub success: c_int,
9+
pub error_message: *mut c_char,
10+
}
11+
12+
// Link to the Swift functions
13+
extern "C" {
14+
pub fn is_apple_intelligence_available() -> c_int;
15+
pub fn process_text_with_apple_llm(
16+
prompt: *const c_char,
17+
max_tokens: i32,
18+
) -> *mut AppleLLMResponse;
19+
pub fn free_apple_llm_response(response: *mut AppleLLMResponse);
20+
}
21+
22+
// Safe wrapper functions
23+
pub fn check_apple_intelligence_availability() -> bool {
24+
unsafe { is_apple_intelligence_available() == 1 }
25+
}
26+
27+
pub fn process_text(prompt: &str, max_tokens: i32) -> Result<String, String> {
28+
let prompt_cstr = CString::new(prompt).map_err(|e| e.to_string())?;
29+
30+
let response_ptr = unsafe { process_text_with_apple_llm(prompt_cstr.as_ptr(), max_tokens) };
31+
32+
if response_ptr.is_null() {
33+
return Err("Null response from Apple LLM".to_string());
34+
}
35+
36+
let response = unsafe { &*response_ptr };
37+
38+
let result = if response.success == 1 {
39+
if response.response.is_null() {
40+
Ok(String::new())
41+
} else {
42+
let c_str = unsafe { CStr::from_ptr(response.response) };
43+
let rust_str = c_str.to_string_lossy().into_owned();
44+
Ok(rust_str)
45+
}
46+
} else {
47+
let error_c_str = if !response.error_message.is_null() {
48+
unsafe { CStr::from_ptr(response.error_message) }
49+
} else {
50+
CStr::from_bytes_with_nul(b"Unknown error\0").unwrap()
51+
};
52+
let error_msg = error_c_str.to_string_lossy().into_owned();
53+
Err(error_msg)
54+
};
55+
56+
// Clean up the response
57+
unsafe { free_apple_llm_response(response_ptr) };
58+
59+
result
60+
}
61+
62+
#[cfg(test)]
63+
mod tests {
64+
use super::*;
65+
66+
#[test]
67+
fn test_availability() {
68+
let available = check_apple_intelligence_availability();
69+
println!("Apple Intelligence available: {}", available);
70+
}
71+
}

‎src-tauri/src/lib.rs‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,6 @@
11
mod actions;
2+
#[cfg(all(target_os = "macos", target_arch = "aarch64"))]
3+
mod apple_intelligence;
24
mod audio_feedback;
35
pub mod audio_toolkit;
46
mod clipboard;

0 commit comments

Comments
 (0)