feat: audio-reactive orb in the widget

Port the procedural Metal orb from ElevenLabs components-swift (Apache-2.0),
trimmed to the volume-driven view, and drive it from the SDK's mic and agent
audio observers: a lock-protected RMS monitor per stream, sampled at 30 Hz so
PCM-rate audio never re-renders SwiftUI. The mic button reuses the same level.

Co-authored-by: Cursor <cursoragent@cursor.com>

Author: renal128

Package.swift

@@ -45,6 +45,15 @@ let package = Package(
             name: "ElevenLabsWidget",
             dependencies: [
                 "ElevenLabs"
+            ],
+            resources: [
+                .process("Resources/OrbShader.metal")
+            ]
+        ),
+        .testTarget(
+            name: "ElevenLabsWidgetTests",
+            dependencies: [
+                "ElevenLabsWidget"
             ]
         ),
         .testTarget(

Package@swift-6.0.swift

@@ -45,6 +45,15 @@ let package = Package(
             name: "ElevenLabsWidget",
             dependencies: [
                 "ElevenLabs"
+            ],
+            resources: [
+                .process("Resources/OrbShader.metal")
+            ]
+        ),
+        .testTarget(
+            name: "ElevenLabsWidgetTests",
+            dependencies: [
+                "ElevenLabsWidget"
             ]
         ),
         .testTarget(

Package@swift-6.2.swift

@@ -45,6 +45,15 @@ let package = Package(
             name: "ElevenLabsWidget",
             dependencies: [
                 "ElevenLabs"
+            ],
+            resources: [
+                .process("Resources/OrbShader.metal")
+            ]
+        ),
+        .testTarget(
+            name: "ElevenLabsWidgetTests",
+            dependencies: [
+                "ElevenLabsWidget"
             ]
         ),
         .testTarget(

Sources/ElevenLabsWidget/Audio/ConversationAudioLevelMonitor.swift

@@ -0,0 +1,62 @@
+import Accelerate
+@preconcurrency import AVFoundation
+import ElevenLabs
+
+/// Tracks a normalized loudness level for one conversation audio stream.
+///
+/// Pull-based: `didReceive` runs on the audio thread and only holds the loudest
+/// level seen. Reading drains it, which is also what paces the decay, so the
+/// level keeps falling once a stream goes quiet and stops delivering buffers.
+final class ConversationAudioLevelMonitor: ConversationAudioObserver, @unchecked Sendable {
+    /// Fraction of the level kept per read: fast attack, slow release.
+    private static let release: Float = 0.75
+
+    private let lock = NSLock()
+    private var storedLevel: Float = 0
+    /// Only touched from the audio thread, which delivers buffers serially.
+    private var scratch: [Float] = []
+
+    /// The loudest level since the last read, leaving one release step behind.
+    func sample() -> Float {
+        lock.withLock {
+            defer { storedLevel *= Self.release }
+            return storedLevel
+        }
+    }
+
+    func didReceive(_ buffer: AVAudioPCMBuffer) {
+        guard let level = normalizedRMS(of: buffer) else { return }
+        lock.withLock { storedLevel = max(level, storedLevel) }
+    }
+
+    func reset() {
+        lock.withLock { storedLevel = 0 }
+    }
+
+    /// RMS of the first channel, mapped from the top 60 dB of headroom onto 0...1.
+    private func normalizedRMS(of buffer: AVAudioPCMBuffer) -> Float? {
+        let frames = Int(buffer.frameLength)
+        guard frames > 0 else { return nil }
+        let stride = buffer.format.isInterleaved ? vDSP_Stride(buffer.format.channelCount) : 1
+
+        var rms: Float = 0
+        if let channel = buffer.floatChannelData?[0] {
+            vDSP_rmsqv(channel, stride, &rms, vDSP_Length(frames))
+        } else if let channel = buffer.int16ChannelData?[0] {
+            // Grown, never reallocated per buffer: this is the audio thread.
+            if scratch.count < frames { scratch = [Float](repeating: 0, count: frames) }
+            scratch.withUnsafeMutableBufferPointer { floats in
+                guard let samples = floats.baseAddress else { return }
+                vDSP_vflt16(channel, stride, samples, 1, vDSP_Length(frames))
+                var scale = 1 / Float(Int16.max)
+                vDSP_vsmul(samples, 1, &scale, samples, 1, vDSP_Length(frames))
+                vDSP_rmsqv(samples, 1, &rms, vDSP_Length(frames))
+            }
+        } else {
+            return nil
+        }
+
+        let decibels = 20 * log10(max(rms, 0.000_001))
+        return min(max((decibels + 60) / 60, 0), 1)
+    }
+}

Sources/ElevenLabsWidget/Audio/OrbAudioLevels.swift

@@ -0,0 +1,58 @@
+import Combine
+import Foundation
+
+/// Samples the mic and agent level monitors at a display-friendly rate.
+///
+/// The monitors are updated at PCM rate on the audio thread; publishing from
+/// there would re-render SwiftUI thousands of times a second, so the levels are
+/// polled on a timer while a conversation is live instead.
+@MainActor
+final class OrbAudioLevels: ObservableObject {
+    @Published private(set) var input: Float = 0
+    @Published private(set) var output: Float = 0
+
+    let micMonitor = ConversationAudioLevelMonitor()
+    let agentMonitor = ConversationAudioLevelMonitor()
+
+    private static let interval: TimeInterval = 1.0 / 30
+    private var timer: Timer?
+
+    var isActive: Bool = false {
+        didSet {
+            guard isActive != oldValue else { return }
+            isActive ? start() : stop()
+        }
+    }
+
+    deinit { timer?.invalidate() }
+
+    private func start() {
+        // Clears anything the audio thread wrote after the last stop, which would
+        // otherwise surface as leftover loudness from the previous call.
+        micMonitor.reset()
+        agentMonitor.reset()
+        let timer = Timer(timeInterval: Self.interval, repeats: true) { [weak self] _ in
+            MainActor.assumeIsolated { self?.sample() }
+        }
+        // Common mode, so the levels keep updating while the transcript scrolls.
+        RunLoop.main.add(timer, forMode: .common)
+        self.timer = timer
+    }
+
+    private func stop() {
+        timer?.invalidate()
+        timer = nil
+        micMonitor.reset()
+        agentMonitor.reset()
+        input = 0
+        output = 0
+    }
+
+    /// Only publishes on change, so a silent conversation doesn't redraw the orb.
+    private func sample() {
+        let mic = micMonitor.sample()
+        let agent = agentMonitor.sample()
+        if mic != input { input = mic }
+        if agent != output { output = agent }
+    }
+}