feat: audio-reactive orb in the widget

Port the procedural Metal orb from ElevenLabs components-swift (Apache-2.0),
trimmed to the volume-driven view, and drive it from the SDK's mic and agent
audio observers: a lock-protected RMS monitor per stream, sampled at 30 Hz so
PCM-rate audio never re-renders SwiftUI. The mic button reuses the same level.

Co-authored-by: Cursor <cursoragent@cursor.com>

Author: renal128

Package.swift

@@ -45,6 +45,15 @@ let package = Package(
             name: "ElevenLabsWidget",
             dependencies: [
                 "ElevenLabs"
+            ],
+            resources: [
+                .process("Resources/OrbShader.metal")
+            ]
+        ),
+        .testTarget(
+            name: "ElevenLabsWidgetTests",
+            dependencies: [
+                "ElevenLabsWidget"
             ]
         ),
         .testTarget(

Package@swift-6.0.swift

@@ -45,6 +45,15 @@ let package = Package(
             name: "ElevenLabsWidget",
             dependencies: [
                 "ElevenLabs"
+            ],
+            resources: [
+                .process("Resources/OrbShader.metal")
+            ]
+        ),
+        .testTarget(
+            name: "ElevenLabsWidgetTests",
+            dependencies: [
+                "ElevenLabsWidget"
             ]
         ),
         .testTarget(

Package@swift-6.2.swift

@@ -45,6 +45,15 @@ let package = Package(
             name: "ElevenLabsWidget",
             dependencies: [
                 "ElevenLabs"
+            ],
+            resources: [
+                .process("Resources/OrbShader.metal")
+            ]
+        ),
+        .testTarget(
+            name: "ElevenLabsWidgetTests",
+            dependencies: [
+                "ElevenLabsWidget"
             ]
         ),
         .testTarget(

Sources/ElevenLabsWidget/Audio/ConversationAudioLevelMonitor.swift

@@ -0,0 +1,51 @@
+import Accelerate
+@preconcurrency import AVFoundation
+import ElevenLabs
+
+/// Tracks a normalized loudness level for one conversation audio stream.
+///
+/// Pull-based: `didReceive` runs on the audio thread and only updates a lock-
+/// protected level, which the UI samples at its own rate.
+final class ConversationAudioLevelMonitor: ConversationAudioObserver, @unchecked Sendable {
+    private let lock = NSLock()
+    private var storedLevel: Float = 0
+
+    var level: Float {
+        lock.withLock { storedLevel }
+    }
+
+    func didReceive(_ buffer: AVAudioPCMBuffer) {
+        guard let level = Self.normalizedRMS(of: buffer) else { return }
+        lock.withLock {
+            // Fast attack, slow release: reacts instantly to speech, decays smoothly.
+            storedLevel = max(level, storedLevel * 0.75)
+        }
+    }
+
+    func reset() {
+        lock.withLock { storedLevel = 0 }
+    }
+
+    /// RMS of the first channel, mapped from the top 60 dB of headroom onto 0...1.
+    private static func normalizedRMS(of buffer: AVAudioPCMBuffer) -> Float? {
+        let frames = Int(buffer.frameLength)
+        guard frames > 0 else { return nil }
+        let stride = buffer.format.isInterleaved ? vDSP_Stride(buffer.format.channelCount) : 1
+
+        var rms: Float = 0
+        if let channel = buffer.floatChannelData?[0] {
+            vDSP_rmsqv(channel, stride, &rms, vDSP_Length(frames))
+        } else if let channel = buffer.int16ChannelData?[0] {
+            var samples = [Float](repeating: 0, count: frames)
+            vDSP_vflt16(channel, stride, &samples, 1, vDSP_Length(frames))
+            var scale = 1 / Float(Int16.max)
+            vDSP_vsmul(samples, 1, &scale, &samples, 1, vDSP_Length(frames))
+            vDSP_rmsqv(samples, 1, &rms, vDSP_Length(frames))
+        } else {
+            return nil
+        }
+
+        let decibels = 20 * log10(max(rms, 0.000_001))
+        return min(max((decibels + 60) / 60, 0), 1)
+    }
+}

Sources/ElevenLabsWidget/Audio/OrbAudioLevels.swift

@@ -0,0 +1,51 @@
+import Combine
+import Foundation
+
+/// Samples the mic and agent level monitors at a display-friendly rate.
+///
+/// The monitors are updated at PCM rate on the audio thread; publishing from
+/// there would re-render SwiftUI thousands of times a second, so the levels are
+/// polled on a timer while a conversation is live instead.
+@MainActor
+final class OrbAudioLevels: ObservableObject {
+    @Published private(set) var input: Float = 0
+    @Published private(set) var output: Float = 0
+
+    let micMonitor = ConversationAudioLevelMonitor()
+    let agentMonitor = ConversationAudioLevelMonitor()
+
+    private static let interval: TimeInterval = 1.0 / 30
+    private var timer: Timer?
+
+    var isActive: Bool = false {
+        didSet {
+            guard isActive != oldValue else { return }
+            isActive ? start() : stop()
+        }
+    }
+
+    deinit { timer?.invalidate() }
+
+    private func start() {
+        timer = Timer.scheduledTimer(withTimeInterval: Self.interval, repeats: true) { [weak self] _ in
+            MainActor.assumeIsolated { self?.sample() }
+        }
+    }
+
+    private func stop() {
+        timer?.invalidate()
+        timer = nil
+        micMonitor.reset()
+        agentMonitor.reset()
+        input = 0
+        output = 0
+    }
+
+    /// Only publishes on change, so a silent conversation doesn't redraw the orb.
+    private func sample() {
+        let mic = micMonitor.level
+        let agent = agentMonitor.level
+        if mic != input { input = mic }
+        if agent != output { output = agent }
+    }
+}