Acceleration improvements and fixes (#241)

* Update Whisper.cpp and add Linux acceleration (source builds only)

Update Whisper.cpp from v1.6.2 to v1.8.1

Supported Linux acceleration methods:
* CUDA
* hipblas
* Vulkan
* OpenCL

If suitable development libraries are installed for any of the above when building from source, then support for them
will be built into both whisper.cpp and the LocalVocal plugin

* Test behaviour with combinations of multiple backends

* Use found dependencies to decide which acceleration features to enable

Also try to build Whispercpp from source on Windows with acceleration (not working)

* Add GPU device selection option in plugin settings

Allows the selection of which GPU backend to use if more than one is
available, as well as disabling GPU backends in case the only
available ones don't work or are somehow slower than the CPU alone

* Add OpenCL support for Linux source build

* Comment out seemingly unused DISABLE_ONNX_RUNTIME_GPU option

* Update prebuilt Whisper deps to 0.0.9 (Whisper 1.8.2) and refactor cmake file

* Handle new GGML IGPU device type

* Bump whisper deps to 0.0.9-1 to fix hipblas

* Try to avoid running apt in CI when packages are cached

* Add option to enable/disable flash attention

* Install Vulkan SDK properly for CI build and try adding OpenCL and hip dev libs

* Build with shared libs on linux and allow use of dynamic backends

* Update deps to 0.0.10 (runtime module linking/importing is likely broken)

* Update deps to 0.0.10-2; try to fix linking & other platform specific issues

* Bump version number to 0.5.0

* Fix Ubuntu packaging

* Fix dynamic linking on MacOS

* Update README

* Cleanup CMake files a bit and make prebuilt build on Linux the default over full source build

* Fix Linux full source build

Author: Tabby

.github/actions/build-plugin/action.yaml

@@ -11,7 +11,7 @@ inputs:
   acceleration:
     description: 'Enable acceleration'
     required: false
-    default: 'cpu'
+    default: 'generic'
   codesign:
     description: 'Enable codesigning (macOS only)'
     required: false
@@ -59,6 +59,8 @@ runs:
       if: runner.os == 'Linux'
       shell: zsh --no-rcs --errexit --pipefail {0}
       working-directory: ${{ inputs.workingDirectory }}
+      env:
+        ACCELERATION: ${{ inputs.acceleration }}
       run: |
         : Run Ubuntu Build
 

.github/scripts/.Aptfile

@@ -7,6 +7,8 @@ package 'libicu-dev'
 package 'libopenblas-dev'
 package 'libopenblas-openmp-dev'
 package 'libsimde-dev'
-package 'libvulkan-dev'
 package 'ninja-build', bin: 'ninja'
+package 'nvidia-opencl-dev'
+package 'ocl-icd-opencl-dev'
+package 'opencl-headers'
 package 'pkg-config'

.github/scripts/.build.zsh

@@ -65,7 +65,12 @@ build() {
     linux-aarch64
   )
   local target
-  local config='RelWithDebInfo'
+  if [[ "${+ACCELERATION}" == "amd" ]]
+  then
+    local config='Release'
+  else
+    local config='RelWithDebInfo'
+  fi
   local -r -a _valid_configs=(Debug RelWithDebInfo Release MinSizeRel)
   local -i codesign=0
 
@@ -184,7 +189,7 @@ ${_usage_host:-}"
     setup_ccache
   }
 
-  if [[ ${host_os} == linux ]] {
+  if [[ ${host_os} == linux && ("$(lsb_release -i)" == "Distributor ID: Ubuntu") ]] {
     autoload -Uz setup_linux && setup_linux
   }
 
@@ -230,7 +235,7 @@ ${_usage_host:-}"
           -DCODESIGN_IDENTITY=${CODESIGN_IDENT:--}
         )
 
-        cmake_build_args+=(--preset ${_preset} --parallel --config ${config} -- ONLY_ACTIVE_ARCH=YES)
+        cmake_build_args+=(--verbose --preset ${_preset} --parallel --config ${config} -- ONLY_ACTIVE_ARCH=YES)
         # check the MACOS_ARCH env var to determine the build architecture
         if [[ -n ${MACOS_ARCH} ]] {
           cmake_build_args+=(-arch ${MACOS_ARCH})

.github/scripts/.package.zsh

@@ -256,7 +256,7 @@ ${_usage_host:-}"
       popd
     } else {
       log_group "Archiving ${product_name}..."
-      local output_name="${product_name}-${product_version}-${target##*-}-linux-gnu"
+      local output_name="${product_name}-${product_version}-${target##*-}-${+ACCELERATION}-linux-gnu"
       local _tarflags='cJf'
       if (( _loglevel > 1 || ${+CI} )) _tarflags="v${_tarflags}"
 

.github/scripts/Package-Windows.ps1

@@ -4,8 +4,8 @@ param(
     [string] $Target = 'x64',
     [ValidateSet('Debug', 'RelWithDebInfo', 'Release', 'MinSizeRel')]
     [string] $Configuration = 'RelWithDebInfo',
-    [ValidateSet('cpu', 'hipblas', 'vulkan', 'cuda')]
-    [string] $Acceleration = 'cpu',
+    [ValidateSet('generic', 'nvidia', 'amd')]
+    [string] $Acceleration = 'generic',
     [switch] $BuildInstaller,
     [switch] $SkipDeps
 )

.github/scripts/utils.zsh/check_linux

@@ -18,43 +18,42 @@ if [[ -f /etc/os_release ]] {
 
 log_debug 'Checking for apt-get...'
 if (( ! ${+commands[apt-get]} )) {
-  log_error 'No apt-get command found. Please install apt'
-  return 2
+  log_warning 'No apt-get command found. Please ensure all dependencies are correctly installed'
 } else {
   log_debug "Apt-get located at ${commands[apt-get]}"
-}
 
-local -a dependencies=("${(fA)$(<${SCRIPT_HOME}/.Aptfile)}")
-local -a install_list
-local binary
+  local -a dependencies=("${(fA)$(<${SCRIPT_HOME}/.Aptfile)}")
+  local -a install_list
+  local binary
 
-sudo apt-get update -qq
+  sudo apt-get update -qq
 
-for dependency (${dependencies}) {
-  local -a tokens=(${=dependency//(,|:|\')/})
+  for dependency (${dependencies}) {
+    local -a tokens=(${=dependency//(,|:|\')/})
 
-  if [[ ! ${tokens[1]} == 'package' ]] continue
+    if [[ ! ${tokens[1]} == 'package' ]] continue
 
-  if [[ ${#tokens} -gt 2 && ${tokens[3]} == 'bin' ]] {
-    binary=${tokens[4]}
-  } else {
-    binary=${tokens[2]}
-  }
+    if [[ ${#tokens} -gt 2 && ${tokens[3]} == 'bin' ]] {
+      binary=${tokens[4]}
+    } else {
+      binary=${tokens[2]}
+    }
 
-  if (( ! ${+commands[${binary}]} )) install_list+=(${tokens[2]})
-}
+    if (( ! ${+commands[${binary}]} )) install_list+=(${tokens[2]})
+  }
 
-log_debug "List of dependencies to install: ${install_list}"
-if (( ${#install_list} )) {
-  if (( ! ${+CI} )) log_warning 'Dependency installation via apt may require elevated privileges'
+  log_debug "List of dependencies to install: ${install_list}"
+  if (( ${#install_list} )) {
+    if (( ! ${+CI} )) log_warning 'Dependency installation via apt may require elevated privileges'
 
-  local -a apt_args=(
-    ${CI:+-y}
-    --no-install-recommends
-  )
-  if (( _loglevel == 0 )) apt_args+=(--quiet)
+    local -a apt_args=(
+      ${CI:+-y}
+      --no-install-recommends
+    )
+    if (( _loglevel == 0 )) apt_args+=(--quiet)
 
-  sudo apt-get ${apt_args} install ${install_list}
+    sudo apt-get ${apt_args} install ${install_list}
+  }
 }
 
 rehash

.github/workflows/build-project.yaml

@@ -202,14 +202,42 @@ jobs:
     name: Build for Ubuntu 🐧
     runs-on: ubuntu-22.04
     needs: check-event
+    strategy:
+      matrix:
+        acceleration: [generic, nvidia, amd]
     defaults:
       run:
         shell: bash
+    env:
+      CI: 1
     steps:
+      - name: Free Disk Space (Ubuntu)
+        if: ${{ matrix.acceleration == 'amd' || matrix.acceleration == 'nvidia' }}
+        uses: jlumbroso/free-disk-space@main
+        with:
+          # this might remove tools that are actually needed,
+          # if set to "true" but frees about 6 GB
+          tool-cache: false
+          # Needs to be false to prevent running out of memory
+          swap-storage: false
+          # large-packages takes a long time so don't remove them unless we need to
+          large-packages: false
+
+          android: true
+          dotnet: true
+          haskell: true
+          docker-images: true
+
       - uses: actions/checkout@v4
         with:
           submodules: recursive
           fetch-depth: 0
+      
+      - name: "Setup additional apt repos, keys and pre-requisites"
+        run: |
+          wget -qO- https://packages.lunarg.com/lunarg-signing-key-pub.asc | sudo tee /etc/apt/trusted.gpg.d/lunarg.asc
+          sudo wget -qO /etc/apt/sources.list.d/lunarg-vulkan-jammy.list http://packages.lunarg.com/vulkan/lunarg-vulkan-jammy.list
+          sudo apt update
 
       - uses: awalsh128/cache-apt-pkgs-action@v1.6.0
         with:
@@ -218,17 +246,25 @@ jobs:
             libcurl4-openssl-dev
             libicu-dev
             libgles2-mesa-dev
+            libopenblas0
+            libopenblas0-openmp
+            libopenblas0-pthread
             libopenblas-dev
             libopenblas-openmp-dev
             libqt6svg6-dev
             libsimde-dev
-            libvulkan-dev
+            nvidia-opencl-dev
             obs-studio
+            ocl-icd-opencl-dev
             pkg-config
+            python3-setuptools
+            python3-wheel
             qt6-base-dev
             qt6-base-private-dev
+            vulkan-sdk
           add-repository: ppa:obsproject/obs-studio
           version: 1.0
+          execute_install_scripts: true
 
       - name: Set Up Environment 🔧
         id: setup
@@ -244,6 +280,29 @@ jobs:
 
       - uses: actions-rust-lang/setup-rust-toolchain@v1
 
+      # CUDA toolkit needed for linking against so it can be found when looking up package dependencies
+      - name: "Install CUDA toolkit 12.8"
+        if: ${{ matrix.acceleration == 'nvidia' }}
+        run: |
+          wget https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/x86_64/cuda-keyring_1.1-1_all.deb
+          sudo dpkg -i cuda-keyring_1.1-1_all.deb
+          sudo apt update
+          sudo apt -y install cuda-12-8
+          export PATH=/usr/local/cuda-12.8/bin${PATH:+:${PATH}}
+          export LD_LIBRARY_PATH=/usr/local/cuda-12.8/lib ${LD_LIBRARY_PATH:+:${LD_LIBRARY_PATH}}
+
+      # hip SDK needed for linking against so it can be found when looking up package dependencies
+      - name: "Install AMD hip SDK v6.4.2"
+        if: ${{ matrix.acceleration == 'amd' }}
+        run: |
+          wget https://repo.radeon.com/amdgpu-install/6.4.2/ubuntu/jammy/amdgpu-install_6.4.60402-1_all.deb
+          sudo apt install ./amdgpu-install_6.4.60402-1_all.deb
+          sudo apt update
+          sudo apt install python3-setuptools python3-wheel
+          sudo usermod -a -G render,video $LOGNAME # Add the current user to the render and video groups
+          sudo apt install rocm
+          sudo apt update && sudo apt install hipblas
+
       - uses: actions/cache@v4
         id: ccache-cache
         with:
@@ -260,6 +319,7 @@ jobs:
         with:
           target: x86_64
           config: ${{ needs.check-event.outputs.config }}
+          acceleration: ${{ matrix.acceleration }}
 
       - name: Package Plugin 📀
         uses: ./.github/actions/package-plugin
@@ -271,29 +331,29 @@ jobs:
       - name: Upload Source Tarball 🗜️
         uses: actions/upload-artifact@v4
         with:
-          name: ${{ steps.setup.outputs.pluginName }}-${{ steps.setup.outputs.pluginVersion }}-sources-${{ needs.check-event.outputs.commitHash }}
-          path: ${{ github.workspace }}/release/${{ steps.setup.outputs.pluginName }}-${{ steps.setup.outputs.pluginVersion }}-source.*
+          name: ${{ steps.setup.outputs.pluginName }}-${{ steps.setup.outputs.pluginVersion }}-${{ matrix.acceleration }}-sources-${{ needs.check-event.outputs.commitHash }}
+          path: ${{ github.workspace }}/release/${{ steps.setup.outputs.pluginName }}-${{ steps.setup.outputs.pluginVersion }}-${{ matrix.acceleration }}-source.*
 
       - name: Upload Artifacts 📡
         uses: actions/upload-artifact@v4
         with:
-          name: ${{ steps.setup.outputs.pluginName }}-${{ steps.setup.outputs.pluginVersion }}-ubuntu-22.04-x86_64-${{ needs.check-event.outputs.commitHash }}
-          path: ${{ github.workspace }}/release/${{ steps.setup.outputs.pluginName }}-${{ steps.setup.outputs.pluginVersion }}-x86_64*.*
+          name: ${{ steps.setup.outputs.pluginName }}-${{ steps.setup.outputs.pluginVersion }}-${{ matrix.acceleration }}-ubuntu-22.04-x86_64-${{ needs.check-event.outputs.commitHash }}
+          path: ${{ github.workspace }}/release/${{ steps.setup.outputs.pluginName }}-${{ steps.setup.outputs.pluginVersion }}-${{ matrix.acceleration }}-x86_64*.*
 
       - name: Upload debug symbol artifacts 🪲
         uses: actions/upload-artifact@v4
-        if: ${{ fromJSON(needs.check-event.outputs.package) }}
+        if: ${{ fromJSON(needs.check-event.outputs.package) && (matrix.acceleration != 'amd') }}
         with:
-          name: ${{ steps.setup.outputs.pluginName }}-${{ steps.setup.outputs.pluginVersion }}-ubuntu-22.04-x86_64-${{ needs.check-event.outputs.commitHash }}-dbgsym
-          path: ${{ github.workspace }}/release/${{ steps.setup.outputs.pluginName }}-${{ steps.setup.outputs.pluginVersion }}-x86_64*-dbgsym.ddeb
+          name: ${{ steps.setup.outputs.pluginName }}-${{ steps.setup.outputs.pluginVersion }}-${{ matrix.acceleration }}-ubuntu-22.04-x86_64-${{ needs.check-event.outputs.commitHash }}-dbgsym
+          path: ${{ github.workspace }}/release/${{ steps.setup.outputs.pluginName }}-${{ steps.setup.outputs.pluginVersion }}-${{ matrix.acceleration }}-x86_64*-dbgsym.ddeb
 
   windows-build:
     name: Build for Windows 🪟
     runs-on: windows-2022
     needs: check-event
     strategy:
       matrix:
-        acceleration: [cpu, cuda, vulkan, hipblas]
+        acceleration: [generic, nvidia, amd]
     defaults:
       run:
         shell: pwsh