worker: prefer macOS P-cores via pthread QoS USER_INITIATED

nativelink-worker/BUILD.bazel

@@ -13,6 +13,7 @@ rust_library(
         "src/directory_cache.rs",
         "src/lib.rs",
         "src/local_worker.rs",
+        "src/qos.rs",
         "src/running_actions_manager.rs",
         "src/worker_api_client_wrapper.rs",
         "src/worker_utils.rs",
@@ -51,6 +52,7 @@ rust_library(
         "@crates//:uuid",
     ] + select({
         "@platforms//os:linux": ["@crates//:libc"],
+        "@platforms//os:macos": ["@crates//:libc"],
         "//conditions:default": [],
     }),
 )
@@ -98,6 +100,7 @@ rust_test_suite(
         "@crates//:which",
     ] + select({
         "@platforms//os:linux": ["@crates//:libc"],
+        "@platforms//os:macos": ["@crates//:libc"],
         "//conditions:default": [],
     }),
 )

nativelink-worker/Cargo.toml

@@ -58,6 +58,9 @@ uuid = { version = "1.16.0", default-features = false, features = [
 [target.'cfg(target_os = "linux")'.dependencies]
 libc = { version = "0.2.183", default-features = false }
 
+[target.'cfg(target_os = "macos")'.dependencies]
+libc = { version = "0.2.183", default-features = false }
+
 [dev-dependencies]
 nativelink-macro = { path = "../nativelink-macro" }
 

nativelink-worker/src/lib.rs

@@ -16,6 +16,7 @@ pub mod directory_cache;
 pub mod local_worker;
 #[cfg(target_os = "linux")]
 pub mod namespace_utils;
+pub mod qos;
 pub mod running_actions_manager;
 pub mod worker_api_client_wrapper;
 pub mod worker_utils;

nativelink-worker/src/local_worker.rs

@@ -798,6 +798,14 @@ impl<T: WorkerApiClientTrait + 'static, U: RunningActionsManager> LocalWorker<T,
         mut self,
         mut shutdown_rx: broadcast::Receiver<ShutdownGuard>,
     ) -> Result<(), Error> {
+        // Belt-and-suspenders QoS bump: the main binary already calls
+        // this before runtime creation so the tokio worker threads
+        // inherit P-core preference via pthread QoS inheritance, but
+        // any thread that reaches this point should also be tagged in
+        // case it was spawned by a path that bypassed `on_thread_start`.
+        // No-op on non-macOS.
+        let _ = crate::qos::set_user_initiated();
+
         let sleep_fn = self
             .sleep_fn
             .take()

nativelink-worker/src/qos.rs

@@ -0,0 +1,166 @@
+// Copyright 2024 The NativeLink Authors. All rights reserved.
+//
+// Licensed under the Functional Source License, Version 1.1, Apache 2.0 Future License (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    See LICENSE file for details
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! Darwin `QoS` (Quality of Service) helpers for worker scheduling.
+//!
+//! Apple Silicon (M-series) CPUs have a heterogeneous topology with
+//! performance ("P") and efficiency ("E") cores. XNU's scheduler routes
+//! threads to P or E cores in part based on the thread's `QoS` class. The
+//! default class assigned to long-running background daemons is typically
+//! `UTILITY` or `BACKGROUND`, both of which the scheduler may park on
+//! E-cores.
+//!
+//! Single-thread-bursty workloads such as `swift-frontend` and `clang`
+//! invocations (typical in iOS RBE builds) can run 2x–3x slower when
+//! pinned to an E-core. Tagging the worker process with
+//! `QOS_CLASS_USER_INITIATED` tells the scheduler to treat its threads
+//! as foreground-equivalent and bias placement toward P-cores.
+//!
+//! On Linux and Windows these helpers compile away to nothing — they are
+//! intentionally not behind a runtime branch so non-macOS builds never
+//! emit a call.
+
+/// Sets the calling thread's `QoS` class to `USER_INITIATED` on macOS.
+///
+/// Returns `true` if the underlying `pthread_set_qos_class_self_np`
+/// call succeeded; returns `false` if it failed.
+///
+/// Safe to call from any thread, including tokio runtime worker threads
+/// via `Builder::on_thread_start`.
+#[cfg(target_os = "macos")]
+#[inline]
+pub fn set_user_initiated() -> bool {
+    // SAFETY: `pthread_set_qos_class_self_np` is a thread-local
+    // setter with no preconditions on the caller; passing a valid
+    // enum variant and relative priority 0 is always defined.
+    let ret = unsafe {
+        libc::pthread_set_qos_class_self_np(libc::qos_class_t::QOS_CLASS_USER_INITIATED, 0)
+    };
+    ret == 0
+}
+
+/// Compile-time no-op on non-macOS targets.
+///
+/// Always returns `true`. The call site expands to nothing after
+/// inlining / dead-code elimination, so non-macOS builds never emit
+/// a runtime branch or a libc call.
+#[cfg(not(target_os = "macos"))]
+#[inline]
+pub const fn set_user_initiated() -> bool {
+    true
+}
+
+#[cfg(all(test, target_os = "macos"))]
+mod macos_tests {
+    use super::set_user_initiated;
+
+    /// Reads the current thread's `QoS` class via `pthread_get_qos_class_np`.
+    /// Panics with a contextual message on failure (only called from tests).
+    fn current_qos_class() -> libc::qos_class_t {
+        let mut class: libc::qos_class_t = libc::qos_class_t::QOS_CLASS_UNSPECIFIED;
+        let mut rel_prio: libc::c_int = 0;
+        // SAFETY: out-pointers point to stack-allocated, properly sized
+        // and aligned storage owned by this thread.
+        let ret = unsafe {
+            libc::pthread_get_qos_class_np(
+                libc::pthread_self(),
+                core::ptr::from_mut(&mut class),
+                core::ptr::from_mut(&mut rel_prio),
+            )
+        };
+        assert_eq!(ret, 0, "pthread_get_qos_class_np failed: {ret}");
+        class
+    }
+
+    /// Proves the `QoS` call is wired up on macOS and the underlying
+    /// Darwin symbol resolves at link time. A failure here means the
+    /// worker would silently keep running on E-cores.
+    #[test]
+    fn sets_user_initiated_on_current_thread() {
+        assert!(
+            set_user_initiated(),
+            "pthread_set_qos_class_self_np(USER_INITIATED) returned non-zero",
+        );
+        // `qos_class_t` is a `#[repr(u32)]` C enum that does not derive
+        // `PartialEq` in libc, so compare the underlying discriminants.
+        assert_eq!(
+            current_qos_class() as u32,
+            libc::qos_class_t::QOS_CLASS_USER_INITIATED as u32,
+            "`QoS` class did not update; thread will be eligible for E-core scheduling",
+        );
+    }
+
+    /// Validates the load-bearing claim that tokio worker threads created
+    /// with a `Builder::on_thread_start` hook calling `set_user_initiated`
+    /// observe `QOS_CLASS_USER_INITIATED` from inside spawned tasks. This
+    /// mirrors the wiring in `src/bin/nativelink.rs::main`. Without this
+    /// test the entire `QoS` scheme is unverified at the integration level.
+    ///
+    /// This is the one place in the worker crate that must construct a
+    /// fresh `tokio::runtime::Builder::new_multi_thread()` and drive it
+    /// with `block_on` — the unit under test *is* the `on_thread_start`
+    /// hook on a custom-built runtime, which `nativelink-util::task` and
+    /// `#[nativelink_test]` do not expose. The `#[expect]` mirrors the
+    /// same justified escape used in `src/bin/nativelink.rs::main`.
+    #[test]
+    #[expect(
+        clippy::disallowed_methods,
+        reason = "test exercises `Builder::on_thread_start` + `block_on`; \
+                  no util wrapper exposes a custom-built runtime with a thread-start hook"
+    )]
+    fn tokio_worker_threads_inherit_user_initiated_via_on_thread_start() {
+        // Deliberately build a fresh runtime in-test (do not reuse a
+        // global one) so the hook is exercised on freshly-spawned
+        // worker threads with whatever class they were born with.
+        let rt = tokio::runtime::Builder::new_multi_thread()
+            .worker_threads(2)
+            .on_thread_start(|| {
+                assert!(set_user_initiated(), "hook failed in worker thread");
+            })
+            .enable_all()
+            .build()
+            .expect("build tokio runtime");
+
+        let observed: u32 = rt.block_on(async {
+            // Force execution on a worker thread (not the caller).
+            tokio::spawn(async { current_qos_class() as u32 })
+                .await
+                .expect("join spawned task")
+        });
+
+        assert_eq!(
+            observed,
+            libc::qos_class_t::QOS_CLASS_USER_INITIATED as u32,
+            "tokio worker thread did not inherit USER_INITIATED from on_thread_start",
+        );
+    }
+}
+
+#[cfg(all(test, not(target_os = "macos")))]
+mod non_macos_tests {
+    use super::set_user_initiated;
+
+    /// On Linux/Windows the function must be a true no-op that always
+    /// reports success — there is no runtime cost and no platform call.
+    #[test]
+    fn is_a_noop_on_non_macos() {
+        assert!(set_user_initiated());
+    }
+}
+
+/// Compile-time assertion: when `target_os` is not `macos`, this module
+/// must not reference any libc symbol. Reviewers can `grep "extern crate
+/// libc"` or inspect this constant to verify the no-op story.
+#[cfg(not(target_os = "macos"))]
+pub const NON_MACOS_IS_NOOP: () = ();

src/bin/nativelink.rs

@@ -718,8 +718,19 @@ fn get_config() -> Result<CasConfig, Error> {
 }
 
 fn main() -> Result<(), Box<dyn core::error::Error>> {
+    // Set QoS to USER_INITIATED on the main thread *before* the tokio
+    // runtime is built so the spawned worker threads inherit P-core
+    // scheduling preference via pthread QoS inheritance on Apple
+    // Silicon. `on_thread_start` below is a belt-and-suspenders hook
+    // for any thread that misses the inherited class (e.g. tokio
+    // blocking pool threads created lazily). No-op on non-macOS.
+    let _ = nativelink_worker::qos::set_user_initiated();
+
     #[expect(clippy::disallowed_methods, reason = "starting main runtime")]
     let runtime = tokio::runtime::Builder::new_multi_thread()
+        .on_thread_start(|| {
+            let _ = nativelink_worker::qos::set_user_initiated();
+        })
         .enable_all()
         .build()?;