Repository navigation
fix(ci): harden dedupe script to skip symlinks via lstat #17
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Release builds | |
| on: | |
| push: | |
| branches: [main] | |
| tags: ['v*'] | |
| workflow_dispatch: | |
| jobs: | |
| compute-version: | |
| runs-on: ubuntu-latest | |
| outputs: | |
| version: ${{ steps.v.outputs.version }} | |
| is_tag: ${{ steps.v.outputs.is_tag }} | |
| steps: | |
| - id: v | |
| run: | | |
| if [[ "$GITHUB_REF" == refs/tags/v* ]]; then | |
| VERSION="${GITHUB_REF#refs/tags/v}" | |
| echo "is_tag=true" >> "$GITHUB_OUTPUT" | |
| else | |
| VERSION="nightly-$(date -u +%Y%m%d)-${GITHUB_SHA::7}" | |
| echo "is_tag=false" >> "$GITHUB_OUTPUT" | |
| fi | |
| echo "version=$VERSION" >> "$GITHUB_OUTPUT" | |
| echo "Resolved version: $VERSION" | |
| build: | |
| needs: compute-version | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| include: | |
| # Linux builds pinned to 22.04 (jammy) for glibc 2.35 floor + LunarG apt availability. | |
| - { os: ubuntu-22.04, backend: cpu, name: linux-cpu } | |
| - { os: ubuntu-22.04, backend: vulkan, name: linux-vulkan } | |
| - { os: ubuntu-22.04, backend: cuda, name: linux-cuda } | |
| # Windows builds use windows-2022 (explicit) for reproducibility. | |
| - { os: windows-2022, backend: cpu, name: windows-cpu } | |
| - { os: windows-2022, backend: vulkan, name: windows-vulkan } | |
| - { os: windows-2022, backend: cuda, name: windows-cuda } | |
| # macOS arm64 with Metal backend (CoreML code predictor also on). | |
| - { os: macos-14, backend: metal, name: macos-arm64-metal } | |
| runs-on: ${{ matrix.os }} | |
| env: | |
| VERSION: ${{ needs.compute-version.outputs.version }} | |
| steps: | |
| - uses: actions/checkout@v5 | |
| with: | |
| submodules: recursive | |
| - name: Install build tools (Linux) | |
| if: runner.os == 'Linux' | |
| run: sudo apt-get update && sudo apt-get install -y cmake build-essential zip | |
| - name: Install Vulkan SDK (Linux) | |
| if: runner.os == 'Linux' && matrix.backend == 'vulkan' | |
| run: | | |
| set -e | |
| wget -qO - https://packages.lunarg.com/lunarg-signing-key-pub.asc | sudo gpg --dearmor -o /usr/share/keyrings/lunarg.gpg | |
| echo "deb [signed-by=/usr/share/keyrings/lunarg.gpg] https://packages.lunarg.com/vulkan/ jammy main" | sudo tee /etc/apt/sources.list.d/lunarg-vulkan.list | |
| sudo apt-get update | |
| sudo apt-get install -y vulkan-sdk | |
| - name: Install CUDA Toolkit (Linux) | |
| if: runner.os == 'Linux' && matrix.backend == 'cuda' | |
| run: | | |
| set -e | |
| wget -q https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/x86_64/cuda-keyring_1.1-1_all.deb | |
| sudo dpkg -i cuda-keyring_1.1-1_all.deb | |
| sudo apt-get update | |
| sudo apt-get install -y cuda-toolkit-12-5 | |
| echo "/usr/local/cuda/bin" >> "$GITHUB_PATH" | |
| echo "CUDA_PATH=/usr/local/cuda" >> "$GITHUB_ENV" | |
| echo "LD_LIBRARY_PATH=/usr/local/cuda/lib64:${LD_LIBRARY_PATH:-}" >> "$GITHUB_ENV" | |
| # libggml-cuda.so has a NEEDED entry for libcuda.so.1 (Driver API). | |
| # The toolkit ships only libcuda.so (no .1 suffix) in its stubs dir, | |
| # so add a symlink with the expected SONAME and pass rpath-link to ld | |
| # so transitive shared-lib checks succeed. At runtime, libcuda.so.1 | |
| # resolves from the end user's NVIDIA driver install. | |
| sudo ln -sf /usr/local/cuda/lib64/stubs/libcuda.so /usr/local/cuda/lib64/stubs/libcuda.so.1 | |
| echo "LDFLAGS=-Wl,-rpath-link,/usr/local/cuda/lib64/stubs -L/usr/local/cuda/lib64/stubs" >> "$GITHUB_ENV" | |
| - name: Install build tools (Windows) | |
| if: runner.os == 'Windows' | |
| shell: bash | |
| run: | | |
| # windows-2022 already has MSVC 17.x + CMake + PowerShell on PATH. | |
| # Sanity-check that they work; fail fast if the image changed. | |
| cmake --version | |
| cl.exe /? 2>&1 | head -1 || true | |
| - name: Install Vulkan SDK (Windows) | |
| if: runner.os == 'Windows' && matrix.backend == 'vulkan' | |
| shell: pwsh | |
| run: | | |
| $version = "1.3.290.0" | |
| $url = "https://sdk.lunarg.com/sdk/download/$version/windows/VulkanSDK-$version-Installer.exe" | |
| $installer = "$env:RUNNER_TEMP\VulkanSDK-Installer.exe" | |
| Write-Host "Downloading $url" | |
| Invoke-WebRequest -Uri $url -OutFile $installer | |
| Write-Host "Running silent install" | |
| Start-Process -FilePath $installer -ArgumentList "--accept-licenses","--default-answer","--confirm-command","install" -Wait -NoNewWindow | |
| $sdkRoot = "C:\VulkanSDK\$version" | |
| if (-not (Test-Path $sdkRoot)) { Write-Error "SDK install dir not found: $sdkRoot"; exit 1 } | |
| echo "VULKAN_SDK=$sdkRoot" >> $env:GITHUB_ENV | |
| echo "$sdkRoot\Bin" >> $env:GITHUB_PATH | |
| Write-Host "Installed Vulkan SDK at $sdkRoot" | |
| - name: Install CUDA Toolkit (Windows) | |
| if: runner.os == 'Windows' && matrix.backend == 'cuda' | |
| shell: pwsh | |
| run: | | |
| $version = "12.5.0" | |
| $installer = "$env:RUNNER_TEMP\cuda_installer.exe" | |
| $url = "https://developer.download.nvidia.com/compute/cuda/12.5.0/network_installers/cuda_12.5.0_windows_network.exe" | |
| Write-Host "Downloading $url" | |
| Invoke-WebRequest -Uri $url -OutFile $installer | |
| Write-Host "Running silent install (this takes ~5 min)" | |
| # Silent switches: -s (silent), then subpackage list without versions (the installer infers) | |
| $args = @("-s","nvcc_12.5","cudart_12.5","cublas_12.5","cublas_dev_12.5","thrust_12.5","visual_studio_integration_12.5") | |
| $proc = Start-Process -FilePath $installer -ArgumentList $args -Wait -PassThru -NoNewWindow | |
| if ($proc.ExitCode -ne 0) { | |
| Write-Error "CUDA installer exited $($proc.ExitCode)" | |
| exit $proc.ExitCode | |
| } | |
| $cudaRoot = "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v12.5" | |
| if (-not (Test-Path $cudaRoot)) { Write-Error "CUDA not installed at expected path"; exit 1 } | |
| echo "CUDA_PATH=$cudaRoot" >> $env:GITHUB_ENV | |
| echo "CUDA_PATH_V12_5=$cudaRoot" >> $env:GITHUB_ENV | |
| echo "$cudaRoot\bin" >> $env:GITHUB_PATH | |
| echo "$cudaRoot\libnvvp" >> $env:GITHUB_PATH | |
| Write-Host "Installed CUDA at $cudaRoot" | |
| - name: Configure ggml | |
| shell: bash | |
| run: | | |
| FLAGS="-DCMAKE_BUILD_TYPE=Release" | |
| case "${{ matrix.backend }}" in | |
| cpu) : ;; | |
| vulkan) FLAGS="$FLAGS -DGGML_VULKAN=ON" ;; | |
| cuda) FLAGS="$FLAGS -DGGML_CUDA=ON" ;; | |
| metal) FLAGS="$FLAGS -DGGML_METAL=ON" ;; | |
| esac | |
| if [ "${{ runner.os }}" = "Windows" ]; then | |
| cmake -S ggml -B ggml/build -G "Visual Studio 17 2022" -A x64 $FLAGS | |
| else | |
| cmake -S ggml -B ggml/build $FLAGS | |
| fi | |
| - name: Build ggml | |
| shell: bash | |
| run: | | |
| # CUDA nvcc is memory-heavy (~3 GB/process) — cap parallelism on the | |
| # 7 GB GitHub runner to avoid OOM kills. | |
| if [ "${{ matrix.backend }}" = "cuda" ]; then | |
| cmake --build ggml/build --config Release -j2 | |
| else | |
| cmake --build ggml/build --config Release -j | |
| fi | |
| - name: Configure project | |
| shell: bash | |
| run: | | |
| FLAGS="-DCMAKE_BUILD_TYPE=Release" | |
| if [ "${{ runner.os }}" != "macOS" ]; then | |
| FLAGS="$FLAGS -DQWEN3_TTS_COREML=OFF" | |
| fi | |
| if [ "${{ runner.os }}" = "Windows" ]; then | |
| cmake -S . -B build -G "Visual Studio 17 2022" -A x64 $FLAGS | |
| else | |
| cmake -S . -B build $FLAGS | |
| fi | |
| - name: Build project | |
| run: cmake --build build --config Release -j --target qwen3-tts-cli qwen3-tts-quantize | |
| - name: Bundle CUDA runtime libs (Linux) | |
| if: runner.os == 'Linux' && matrix.backend == 'cuda' | |
| shell: bash | |
| run: | | |
| mkdir -p cuda-runtime | |
| # Resolve each SONAME to its concrete file and copy just once under | |
| # the SONAME. The binary's DT_NEEDED references libfoo.so.12 (SONAME), | |
| # so that's the only name we need to ship. Skipping the unversioned | |
| # symlink (dev-only) and the full-version file avoids ~1 GB duplication. | |
| for soname in libcudart.so.12 libcublas.so.12 libcublasLt.so.12; do | |
| real="$(readlink -f /usr/local/cuda/lib64/$soname)" | |
| if [ -f "$real" ]; then | |
| cp "$real" "cuda-runtime/$soname" | |
| else | |
| echo "warn: $soname not found" >&2 | |
| fi | |
| done | |
| ls -la cuda-runtime/ | |
| - name: Package (Linux/macOS) | |
| if: runner.os != 'Windows' | |
| shell: bash | |
| run: | | |
| PKG="qwen3-tts-${VERSION}-${{ matrix.name }}" | |
| mkdir -p "dist/$PKG/examples" "dist/$PKG/scripts" | |
| cp build/qwen3-tts-cli "dist/$PKG/" | |
| cp build/qwen3-tts-quantize "dist/$PKG/" | |
| # GGML runtime shared libs (if any — static build produces none) | |
| find ggml/build -maxdepth 4 \( -name 'libggml*.so*' -o -name 'libggml*.dylib' \) -exec cp {} "dist/$PKG/" \; || true | |
| # CUDA runtime libs staged in previous step (CUDA builds only) | |
| if [ -d cuda-runtime ] && [ "${{ matrix.backend }}" = "cuda" ]; then | |
| cp -P cuda-runtime/* "dist/$PKG/" || true | |
| fi | |
| cp examples/*.wav "dist/$PKG/examples/" | |
| cp scripts/download_model.sh "dist/$PKG/scripts/" | |
| cp scripts/download_model.bat "dist/$PKG/scripts/" | |
| cp .github/packaging/README.txt "dist/$PKG/" | |
| cp LICENSE "dist/$PKG/" | |
| chmod +x "dist/$PKG/scripts/download_model.sh" | |
| chmod +x "dist/$PKG/qwen3-tts-cli" "dist/$PKG/qwen3-tts-quantize" | |
| # Dedupe GGML lib chains: same content, multiple names. GGML's build | |
| # produces 3 real files (libfoo.so[.X[.Y.Z]]) instead of symlink chains; | |
| # collapse them so only the longest-named file stays as a real file and | |
| # the rest become symlinks. Saves ~200 MB on CUDA zips. | |
| python3 - "dist/$PKG" <<'PY' | |
| import hashlib, os, sys, glob | |
| pkg = sys.argv[1] | |
| os.chdir(pkg) | |
| groups = {} | |
| for f in sorted(set(glob.glob("lib*.so*") + glob.glob("lib*.dylib") + glob.glob("lib*.*.dylib"))): | |
| # Use lstat so we don't follow symlinks (ggml's macOS build already | |
| # uses symlink chains; we only want to group the concrete files). | |
| try: | |
| st = os.lstat(f) | |
| except OSError: | |
| continue | |
| import stat as _stat | |
| if _stat.S_ISLNK(st.st_mode): | |
| continue | |
| if not _stat.S_ISREG(st.st_mode): | |
| continue | |
| try: | |
| with open(f, "rb") as fh: | |
| h = hashlib.sha256(fh.read()).hexdigest() | |
| except OSError as e: | |
| print(f" skip {f}: {e}") | |
| continue | |
| groups.setdefault(h, []).append(f) | |
| saved = 0 | |
| for h, files in groups.items(): | |
| if len(files) < 2: | |
| continue | |
| files.sort(key=len, reverse=True) | |
| real, others = files[0], files[1:] | |
| for other in others: | |
| try: | |
| size = os.path.getsize(other) | |
| os.unlink(other) | |
| os.symlink(real, other) | |
| saved += size | |
| print(f" symlinked {other} -> {real}") | |
| except OSError as e: | |
| print(f" skip {other}: {e}") | |
| print(f" saved {saved} bytes") | |
| PY | |
| # macOS: rewrite dylib IDs and add @executable_path to the binary rpath so | |
| # the bundled .dylib files are found from any install location. | |
| if [ "${{ runner.os }}" = "macOS" ]; then | |
| for dylib in "dist/$PKG/"*.dylib; do | |
| [ -e "$dylib" ] || continue | |
| [ -L "$dylib" ] && continue # dedupe may have replaced some with symlinks | |
| install_name_tool -id "@rpath/$(basename "$dylib")" "$dylib" || true | |
| done | |
| for bin in qwen3-tts-cli qwen3-tts-quantize; do | |
| install_name_tool -add_rpath "@executable_path" "dist/$PKG/$bin" 2>/dev/null || true | |
| done | |
| fi | |
| # -y preserves symlinks (without it, zip follows and stores duplicates) | |
| cd dist && zip -y -r "${PKG}.zip" "$PKG" | |
| ls -la "${PKG}.zip" | |
| - name: Package (Windows) | |
| if: runner.os == 'Windows' | |
| shell: bash | |
| run: | | |
| PKG="qwen3-tts-${VERSION}-${{ matrix.name }}" | |
| mkdir -p "dist/$PKG/examples" "dist/$PKG/scripts" | |
| cp build/Release/qwen3-tts-cli.exe "dist/$PKG/" | |
| cp build/Release/qwen3-tts-quantize.exe "dist/$PKG/" | |
| # GGML runtime DLLs (ggml.dll, ggml-base.dll, ggml-cpu.dll, plus backend if applicable) | |
| find ggml/build -maxdepth 5 -name 'ggml*.dll' -exec cp {} "dist/$PKG/" \; || true | |
| # CUDA runtime DLLs (windows-specific filenames) | |
| if [ "${{ matrix.backend }}" = "cuda" ]; then | |
| for pat in "cudart64_*.dll" "cublas64_*.dll" "cublasLt64_*.dll"; do | |
| find "$CUDA_PATH/bin" -maxdepth 1 -name "$pat" -exec cp {} "dist/$PKG/" \; 2>/dev/null || true | |
| done | |
| fi | |
| # Vulkan loader DLL — usually on PATH via system, but bundle it so end users | |
| # without the SDK installed can still run. Located at $VULKAN_SDK\Bin\vulkan-1.dll. | |
| if [ "${{ matrix.backend }}" = "vulkan" ]; then | |
| if [ -f "$VULKAN_SDK/Bin/vulkan-1.dll" ]; then | |
| cp "$VULKAN_SDK/Bin/vulkan-1.dll" "dist/$PKG/" | |
| fi | |
| fi | |
| cp examples/*.wav "dist/$PKG/examples/" | |
| cp scripts/download_model.sh "dist/$PKG/scripts/" | |
| cp scripts/download_model.bat "dist/$PKG/scripts/" | |
| cp .github/packaging/README.txt "dist/$PKG/" | |
| cp LICENSE "dist/$PKG/" | |
| # Use PowerShell's Compress-Archive (reliable on windows-2022, no zip.exe) | |
| powershell -Command "Compress-Archive -Path 'dist/$PKG/*' -DestinationPath 'dist/${PKG}.zip' -CompressionLevel Optimal" | |
| ls -la "dist/${PKG}.zip" | |
| - name: Upload artifact | |
| uses: actions/upload-artifact@v5 | |
| with: | |
| name: qwen3-tts-${{ env.VERSION }}-${{ matrix.name }} | |
| path: dist/qwen3-tts-${{ env.VERSION }}-${{ matrix.name }}.zip | |
| retention-days: 14 | |
| if-no-files-found: error | |
| release: | |
| needs: [compute-version, build] | |
| if: startsWith(github.ref, 'refs/tags/v') | |
| runs-on: ubuntu-22.04 | |
| permissions: | |
| contents: write | |
| steps: | |
| - name: Download all build artifacts | |
| uses: actions/download-artifact@v5 | |
| with: | |
| path: all-artifacts | |
| pattern: 'qwen3-tts-*' | |
| merge-multiple: true | |
| - name: List artifacts | |
| run: ls -la all-artifacts/ | |
| - name: Create GitHub Release | |
| uses: softprops/action-gh-release@v2 | |
| with: | |
| files: all-artifacts/*.zip | |
| generate_release_notes: true | |
| fail_on_unmatched_files: true | |
| draft: false | |
| prerelease: false |