Add Swift API for source separation (#3426)

This pull request expands the Swift API by integrating offline audio source separation capabilities. It provides developers with tools to easily separate audio into different stems (e.g., vocals and accompaniment) using both Spleeter and UVR models, enhancing the framework's utility for audio processing applications.

Author: csukuangfj

.github/scripts/test-swift.sh

@@ -7,6 +7,15 @@ echo "pwd: $PWD"
 cd swift-api-examples
 ls -lh
 
+./run-source-separation-spleeter.sh
+rm -rf sherpa-onnx-spleeter-*
+rm vocals.wav accompaniment.wav
+
+./run-source-separation-uvr.sh
+rm -rf UVR-MDX-NET-Voc_FT.onnx
+rm uvr-vocals.wav uvr-non-vocals.wav
+rm -f qi-feng-le-zh.wav
+
 ./run-qwen3-asr.sh
 rm -rf sherpa-onnx-qwen3-*
 

.gitignore

@@ -183,3 +183,4 @@ sherpa-onnx-kws-zipformer-wenetspeech-3.3M-2024-01-01-mobile
 sherpa-onnx-punct-ct-transformer-zh-en-vocab272727-2024-04-12-int8
 doxygen-docs
 sherpa-onnx-qwen3-asr-0.6B-int8-2026-03-25
+sherpa-onnx-spleeter-2stems-fp16

swift-api-examples/.gitignore

@@ -35,3 +35,5 @@ fire-red-asr-ctc
 moonshine-v2-asr
 tts-supertonic-en
 tts-zipvoice
+source-separation-spleeter
+source-separation-uvr

swift-api-examples/SherpaOnnx.swift

@@ -1122,7 +1122,8 @@ func sherpaOnnxOfflineTtsModelConfig(
   kitten: SherpaOnnxOfflineTtsKittenModelConfig = sherpaOnnxOfflineTtsKittenModelConfig(),
   zipvoice: SherpaOnnxOfflineTtsZipvoiceModelConfig = sherpaOnnxOfflineTtsZipvoiceModelConfig(),
   pocket: SherpaOnnxOfflineTtsPocketModelConfig = sherpaOnnxOfflineTtsPocketModelConfig(),
-  supertonic: SherpaOnnxOfflineTtsSupertonicModelConfig = sherpaOnnxOfflineTtsSupertonicModelConfig()
+  supertonic: SherpaOnnxOfflineTtsSupertonicModelConfig =
+    sherpaOnnxOfflineTtsSupertonicModelConfig()
 ) -> SherpaOnnxOfflineTtsModelConfig {
   return SherpaOnnxOfflineTtsModelConfig(
     vits: vits,
@@ -1353,7 +1354,8 @@ class SherpaOnnxOfflineTtsWrapper {
       let p = Unmanaged<SherpaOnnxCallbackPair>.fromOpaque(rawArg!).takeUnretainedValue()
       return p.cb!(samples, n, p.arg)
     }
-    let result = generateWithConfig(text: text, config: config, callback: wrapper, arg: unmanaged.toOpaque())
+    let result = generateWithConfig(
+      text: text, config: config, callback: wrapper, arg: unmanaged.toOpaque())
     unmanaged.release()
     return result
   }
@@ -2037,3 +2039,163 @@ func getSherpaOnnxGitSha1() -> String {
 func getSherpaOnnxGitDate() -> String {
   return String(cString: SherpaOnnxGetGitDate())
 }
+//---------------------------
+// Source separation
+//---------------------------
+
+// MARK: - Audio Buffer
+struct AudioBuffer {
+  private enum Storage {
+    case owned([Float])
+    case wrapped(ManagedWave)
+  }
+
+  private class ManagedWave {
+    let pointer: UnsafePointer<SherpaOnnxMultiChannelWave>
+    init(_ p: UnsafePointer<SherpaOnnxMultiChannelWave>) { self.pointer = p }
+    deinit { SherpaOnnxFreeMultiChannelWave(pointer) }
+  }
+
+  private let storage: Storage
+  let channelCount: Int
+  let samplesPerChannel: Int
+  let sampleRate: Int
+
+  init(samples: [Float], channelCount: Int, sampleRate: Int) {
+    self.storage = .owned(samples)
+    self.channelCount = channelCount
+    self.sampleRate = sampleRate
+    self.samplesPerChannel = channelCount > 0 ? samples.count / channelCount : 0
+  }
+
+  init?(filename: String) {
+    guard let ptr = SherpaOnnxReadWaveMultiChannel(filename) else { return nil }
+    self.storage = .wrapped(ManagedWave(ptr))
+    self.channelCount = Int(ptr.pointee.num_channels)
+    self.samplesPerChannel = Int(ptr.pointee.num_samples)
+    self.sampleRate = Int(ptr.pointee.sample_rate)
+  }
+
+  func withUnsafeBufferPointer<R>(_ body: (UnsafeBufferPointer<Float>) -> R) -> R {
+    switch storage {
+    case .owned(let array):
+      return array.withUnsafeBufferPointer(body)
+    case .wrapped(let managed):
+      let total = Int(managed.pointer.pointee.num_channels * managed.pointer.pointee.num_samples)
+      // Ensure we start from the first channel's pointer
+      return body(UnsafeBufferPointer(start: managed.pointer.pointee.samples[0], count: total))
+    }
+  }
+
+  @discardableResult
+  func save(to filename: String) -> Bool {
+    return withUnsafeBufferPointer { buf in
+      guard let base = buf.baseAddress else { return false }
+      // FIX: Explicitly type the array as Optional pointers to match C 'float* const*'
+      var ptrs: [UnsafePointer<Float>?] = (0..<channelCount).map { base + ($0 * samplesPerChannel) }
+
+      return SherpaOnnxWriteWaveMultiChannel(
+        &ptrs,
+        Int32(samplesPerChannel),
+        Int32(sampleRate),
+        Int32(channelCount),
+        filename
+      ) == 1
+    }
+  }
+}
+
+// MARK: - Configuration Struct
+struct SourceSeparationConfig {
+  struct Spleeter {
+    var vocals: String
+    var accompaniment: String
+  }
+  struct Uvr { var model: String }
+
+  var spleeter: Spleeter?
+  var uvr: Uvr?
+  var numThreads: Int = 1
+  var debug: Bool = false
+  var provider: String = "cpu"
+
+  func withCConfig<R>(_ body: (UnsafePointer<SherpaOnnxOfflineSourceSeparationConfig>) -> R) -> R {
+    var cConfig = SherpaOnnxOfflineSourceSeparationConfig()
+    cConfig.model.num_threads = Int32(self.numThreads)
+    cConfig.model.debug = self.debug ? 1 : 0
+
+    var s: [String: [Int8]] = [:]
+    func b(_ k: String, _ v: String?) -> UnsafePointer<Int8>? {
+      guard let v = v else { return nil }
+      s[k] = Array(v.utf8CString)
+      return s[k]!.withUnsafeBufferPointer { $0.baseAddress }
+    }
+
+    cConfig.model.provider = b("provider", self.provider)
+    cConfig.model.spleeter.vocals = b("spleeter.vocals", self.spleeter?.vocals)
+    cConfig.model.spleeter.accompaniment = b("spleeter.accompaniment", self.spleeter?.accompaniment)
+    cConfig.model.uvr.model = b("uvr.model", self.uvr?.model)
+
+    return body(&cConfig)
+  }
+}
+
+class SourceSeparator {
+  private var engine: OpaquePointer?
+
+  init?(config: SourceSeparationConfig) {
+    self.engine = config.withCConfig { SherpaOnnxCreateOfflineSourceSeparation($0) }
+
+    if self.engine == nil { return nil }
+  }
+
+  deinit {
+    if let e = engine {
+      SherpaOnnxDestroyOfflineSourceSeparation(e)
+    }
+  }
+
+  func process(buffer: AudioBuffer) -> [AudioBuffer]? {
+    guard let engine = engine else { return nil }
+
+    return buffer.withUnsafeBufferPointer { flatBuf in
+      guard let base = flatBuf.baseAddress else { return nil }
+      var ptrs: [UnsafePointer<Float>?] = (0..<buffer.channelCount).map {
+        base + ($0 * buffer.samplesPerChannel)
+      }
+
+      guard
+        let raw = SherpaOnnxOfflineSourceSeparationProcess(
+          engine,
+          &ptrs,
+          Int32(buffer.channelCount),
+          Int32(buffer.samplesPerChannel),
+          Int32(buffer.sampleRate)
+        )
+      else { return nil }
+
+      let stemCount = Int(raw.pointee.num_stems)
+      let result = (0..<stemCount).map { i in
+        let stem = raw.pointee.stems[i]
+        let chs = Int(stem.num_channels)
+        let n = Int(stem.n)
+        var flat = [Float](repeating: 0, count: chs * n)
+
+        for c in 0..<chs {
+          if let src = stem.samples[c] {
+            let offset = c * n
+            flat.withUnsafeMutableBufferPointer { dest in
+              let destPtr = dest.baseAddress!.advanced(by: offset)
+              destPtr.initialize(from: src, count: n)
+            }
+          }
+        }
+        return AudioBuffer(
+          samples: flat, channelCount: chs, sampleRate: Int(raw.pointee.sample_rate))
+      }
+
+      SherpaOnnxDestroySourceSeparationOutput(raw)
+      return result
+    }
+  }
+}

swift-api-examples/run-source-separation-spleeter.sh

@@ -0,0 +1,38 @@
+#!/usr/bin/env bash
+
+set -ex
+
+if [ ! -d ../build-swift-macos ]; then
+  echo "Please run ../build-swift-macos.sh first!"
+  exit 1
+fi
+
+if [ ! -f ./sherpa-onnx-spleeter-2stems-fp16/vocals.fp16.onnx ]; then
+  curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/source-separation-models/sherpa-onnx-spleeter-2stems-fp16.tar.bz2
+  tar xjf sherpa-onnx-spleeter-2stems-fp16.tar.bz2
+  rm sherpa-onnx-spleeter-2stems-fp16.tar.bz2
+fi
+
+if [ ! -f ./qi-feng-le-zh.wav ]; then
+  curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/source-separation-models/qi-feng-le-zh.wav
+fi
+
+if [ ! -e ./source-separation-spleeter ]; then
+  # Note: We use -lc++ to link against libc++ instead of libstdc++
+  swiftc \
+    -lc++ \
+    -I ../build-swift-macos/install/include \
+    -import-objc-header ./SherpaOnnx-Bridging-Header.h \
+    ./source-separation-spleeter.swift ./SherpaOnnx.swift \
+    -L ../build-swift-macos/install/lib/ \
+    -l sherpa-onnx \
+    -l onnxruntime \
+    -o source-separation-spleeter
+
+  strip source-separation-spleeter
+else
+  echo "./source-separation-spleeter exists - skip building"
+fi
+
+export DYLD_LIBRARY_PATH=$PWD/../build-swift-macos/install/lib:$DYLD_LIBRARY_PATH
+./source-separation-spleeter

swift-api-examples/run-source-separation-uvr.sh

@@ -0,0 +1,36 @@
+#!/usr/bin/env bash
+
+set -ex
+
+if [ ! -d ../build-swift-macos ]; then
+  echo "Please run ../build-swift-macos.sh first!"
+  exit 1
+fi
+
+if [ ! -f ./UVR-MDX-NET-Voc_FT.onnx ]; then
+  curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/source-separation-models/UVR-MDX-NET-Voc_FT.onnx
+fi
+
+if [ ! -f ./qi-feng-le-zh.wav ]; then
+  curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/source-separation-models/qi-feng-le-zh.wav
+fi
+
+if [ ! -e ./source-separation-uvr ]; then
+  # Note: We use -lc++ to link against libc++ instead of libstdc++
+  swiftc \
+    -lc++ \
+    -I ../build-swift-macos/install/include \
+    -import-objc-header ./SherpaOnnx-Bridging-Header.h \
+    ./source-separation-uvr.swift ./SherpaOnnx.swift \
+    -L ../build-swift-macos/install/lib/ \
+    -l sherpa-onnx \
+    -l onnxruntime \
+    -o source-separation-uvr
+
+  strip source-separation-uvr
+else
+  echo "./source-separation-uvr exists - skip building"
+fi
+
+export DYLD_LIBRARY_PATH=$PWD/../build-swift-macos/install/lib:$DYLD_LIBRARY_PATH
+./source-separation-uvr

swift-api-examples/source-separation-spleeter.swift

@@ -0,0 +1,67 @@
+// swift-api-examples/source-separation-spleeter.swift
+//
+// Copyright (c)  2026  Xiaomi Corporation
+
+// This file demonstrates how to use source separation with the
+// Spleeter 2-stems model (UNet-based).
+//
+// Please refer to
+// https://github.com/k2-fsa/sherpa-onnx/releases/tag/source-separation-models
+// to download files used in this script
+
+import Foundation
+
+func run() {
+  let modelDir = "./sherpa-onnx-spleeter-2stems-fp16"
+
+  let config = SourceSeparationConfig(
+    spleeter: .init(
+      vocals: "\(modelDir)/vocals.fp16.onnx",
+      accompaniment: "\(modelDir)/accompaniment.fp16.onnx"
+    ),
+    numThreads: 1,
+    debug: true
+  )
+
+  guard let separator = SourceSeparator(config: config) else {
+    print("Failed to create SourceSeparator")
+    return
+  }
+
+  guard let input = AudioBuffer(filename: "./qi-feng-le-zh.wav") else {
+    print("Failed to read ./qi-feng-le-zh.wav")
+    return
+  }
+  print(
+    "Input: channels=\(input.channelCount), samples=\(input.samplesPerChannel), sampleRate=\(input.sampleRate)"
+  )
+
+  let start = CFAbsoluteTimeGetCurrent()
+  guard let stems = separator.process(buffer: input) else {
+    print("Source separation failed")
+    return
+  }
+  let elapsed = CFAbsoluteTimeGetCurrent() - start
+  let audioDuration = Double(input.samplesPerChannel) / Double(input.sampleRate)
+  let rtf = elapsed / audioDuration
+
+  print("Output: \(stems.count) stems, sampleRate=\(input.sampleRate)")
+  print("Elapsed: \(String(format: "%.2f", elapsed))s, Audio: \(String(format: "%.2f", audioDuration))s, RTF = \(String(format: "%.3f", rtf))")
+
+  let stemNames = ["vocals", "accompaniment"]
+  for i in 0..<min(stems.count, stemNames.count) {
+    let filename = "\(stemNames[i]).wav"
+    if stems[i].save(to: filename) {
+      print("Saved \(filename)")
+    } else {
+      print("Failed to save \(filename)")
+    }
+  }
+}
+
+@main
+struct App {
+  static func main() {
+    run()
+  }
+}