Skip to content

Build-Native-ZSTD

Build-Native-ZSTD #285

name: Build-Native-ZSTD
# Follow Zstandard latest release by cron update
# 1. Check for Zstandard updates - compare submodule commit ID with latest release commit
# 2. If no changes, checkout in each build job, build and upload artifacts.
# 2". If changes exist, update submodule after checkout in each build job, build and upload artifacts
# 3. Upload packages to GitHub Actions summary
# 4. If changes exist, Create a branch and PR. Separate PRs for each Zstandard release are preferred (avoid updating in a single PR)
# PR includes following changes.
# - Commit that updates the submodule
# - Commit that zstd dynamic lib artifacts placed in src/NativeCompressions.Zstandard.Runtime/runtimes/{platform_name}/native/{lib}
on:
workflow_dispatch:
schedule:
- cron: "0 3 * * *" # Every day at 3:00 AM (UTC)
jobs:
check_new_release:
permissions:
contents: read
uses: ./.github/workflows/check-new-release.yaml
with:
repository: facebook/zstd
submodule_path: zstd
build-windows:
needs: [check_new_release]
if: ${{ needs.check_new_release.outputs.needs_update == 'true' || github.event_name == 'workflow_dispatch' }}
strategy:
matrix:
arch: [x64]
include:
- arch: x64
platform: x64
msys_env: MINGW64
permissions:
contents: read
runs-on: windows-2025
timeout-minutes: 5
steps:
- uses: Cysharp/Actions/.github/actions/checkout@main
with:
submodules: recursive
fetch-depth: 1
- name: Update submodule to latest release
if: ${{ needs.check_new_release.outputs.needs_update == 'true' }}
run: |
cd zstd
git fetch --tags
git checkout ${{ needs.check_new_release.outputs.latest_tag }}
cd ..
echo "Updated zstd submodule to ${{ needs.check_new_release.outputs.latest_tag }}"
- name: Setup MSYS2
uses: msys2/setup-msys2@4f806de0a5a7294ffabaff804b38a9b435a73bda # v2.30.0
with:
msystem: ${{ matrix.msys_env }}
update: true
install: >-
${{ matrix.arch == 'x64' && 'mingw-w64-x86_64-gcc mingw-w64-x86_64-make' || 'mingw-w64-clang-aarch64-gcc mingw-w64-clang-aarch64-make' }}
make
# Windows lib-mt-release build not apply -DZSTD_MULTITHREAD, so use lib-release instead + `MOREFLAGS="-DZSTD_MULTITHREAD -flto"`
# `-fPIC` is not needed on Windows
- name: Build DLL
shell: msys2 {0}
run: |
cd zstd/lib
make lib-release TARGET_SYSTEM=Windows CC=gcc MOREFLAGS="-DZSTD_MULTITHREAD" -j "$(nproc)"
- name: Prepare artifacts
shell: pwsh
run: |
New-Item -ItemType Directory -Force -Path "artifacts/win-${{ matrix.arch }}"
Copy-Item "zstd/lib/dll/*.dll" -Destination "artifacts/win-${{ matrix.arch }}/libzstd.dll"
- name: Upload Windows artifacts
uses: Cysharp/Actions/.github/actions/upload-artifact@main
with:
name: zstd-win-${{ matrix.arch }}
path: artifacts/win-${{ matrix.arch }}/*
build-windows-arm64:
needs: [check_new_release]
if: ${{ needs.check_new_release.outputs.needs_update == 'true' || github.event_name == 'workflow_dispatch' }}
permissions:
contents: read
runs-on: ubuntu-24.04
timeout-minutes: 5
steps:
- uses: Cysharp/Actions/.github/actions/checkout@main
with:
submodules: recursive
- name: Update submodule to latest release
if: ${{ needs.check_new_release.outputs.needs_update == 'true' }}
run: |
cd zstd
git fetch --tags
git checkout ${{ needs.check_new_release.outputs.latest_tag }}
cd ..
echo "Updated zstd submodule to ${{ needs.check_new_release.outputs.latest_tag }}"
# Windows lib-mt-release build not apply -DZSTD_MULTITHREAD, so use lib-release instead + `MOREFLAGS="-DZSTD_MULTITHREAD -flto"`
# `-fPIC` is not needed on Windows
- name: Build using Docker
run: |
docker run --rm -v "$PWD:/work" \
mstorsjo/llvm-mingw:latest \
bash -c "cd /work/zstd/lib && \
make lib-release TARGET_SYSTEM=Windows CC=aarch64-w64-mingw32-gcc MOREFLAGS='-DZSTD_MULTITHREAD'"
file zstd/lib/dll/libzstd.dll
- name: Prepare artifacts
run: |
mkdir -p artifacts/win-arm64
cp zstd/lib/dll/libzstd.dll artifacts/win-arm64/
- name: Upload artifacts
uses: Cysharp/Actions/.github/actions/upload-artifact@main
with:
name: zstd-win-arm64
path: artifacts/win-arm64/*
build-linux:
needs: [check_new_release]
if: ${{ needs.check_new_release.outputs.needs_update == 'true' || github.event_name == 'workflow_dispatch' }}
permissions:
contents: read
runs-on: ubuntu-24.04
timeout-minutes: 5
strategy:
matrix:
arch: [x64, arm64]
include:
- arch: x64
cc: gcc
- arch: arm64
cc: aarch64-linux-gnu-gcc
packages: gcc-aarch64-linux-gnu
steps:
- uses: Cysharp/Actions/.github/actions/checkout@main
with:
submodules: recursive
fetch-depth: 1
- name: Update submodule to latest release
if: ${{ needs.check_new_release.outputs.needs_update == 'true' }}
run: |
cd zstd
git fetch --tags
git checkout ${{ needs.check_new_release.outputs.latest_tag }}
cd ..
echo "Updated zstd submodule to ${{ needs.check_new_release.outputs.latest_tag }}"
- name: Install ARM64 cross-compiler
if: ${{ matrix.arch == 'arm64' }}
run: |
sudo apt-get update
sudo apt-get install -y ${{ matrix.packages }}
- name: Build shared library
run: |
cd zstd/lib
make lib-release CC="${{ matrix.cc }}" MOREFLAGS="-DZSTD_MULTITHREAD -fPIC -flto" -j "$(nproc)"
ls -la ./*.so*
file libzstd.so*
- name: Prepare artifacts
run: |
mkdir -p artifacts/linux-${{ matrix.arch }}
cp zstd/lib/libzstd.so* artifacts/linux-${{ matrix.arch }}/
cd artifacts/linux-${{ matrix.arch }}
if [ -L libzstd.so ]; then
cp -L libzstd.so libzstd.so.tmp
mv libzstd.so.tmp libzstd.so
fi
- name: Upload Linux artifacts
uses: Cysharp/Actions/.github/actions/upload-artifact@main
with:
name: zstd-linux-${{ matrix.arch }}
path: artifacts/linux-${{ matrix.arch }}/*
build-macos:
needs: [check_new_release]
if: ${{ needs.check_new_release.outputs.needs_update == 'true' || github.event_name == 'workflow_dispatch' }}
permissions:
contents: read
strategy:
matrix:
arch: [x64, arm64]
include:
- arch: x64
flags: "-arch x86_64"
- arch: arm64
flags: "-arch arm64"
runs-on: macos-15
timeout-minutes: 5
steps:
- uses: Cysharp/Actions/.github/actions/checkout@main
with:
submodules: recursive
- name: Update submodule to latest release
if: ${{ needs.check_new_release.outputs.needs_update == 'true' }}
run: |
cd zstd
git fetch --tags
git checkout ${{ needs.check_new_release.outputs.latest_tag }}
cd ..
echo "Updated zstd submodule to ${{ needs.check_new_release.outputs.latest_tag }}"
- name: Build dynamic library
run: |
cd zstd/lib
make lib-release MOREFLAGS="-DZSTD_MULTITHREAD ${{ matrix.flags }} -fPIC -flto" -j "$(sysctl -n hw.ncpu)"
lipo -info ./*.dylib
- name: Prepare artifacts
run: |
mkdir -p artifacts/osx-${{ matrix.arch }}
cp zstd/lib/*.dylib artifacts/osx-${{ matrix.arch }}/
cd artifacts/osx-${{ matrix.arch }}
for file in *.dylib; do
if [ -L "$file" ]; then
cp -L "$file" "$file.tmp"
mv "$file.tmp" "$file"
fi
done
- name: Upload artifacts
uses: Cysharp/Actions/.github/actions/upload-artifact@main
with:
name: zstd-osx-${{ matrix.arch }}
path: artifacts/osx-${{ matrix.arch }}/*
build-ios:
needs: [check_new_release]
if: ${{ needs.check_new_release.outputs.needs_update == 'true' || github.event_name == 'workflow_dispatch' }}
permissions:
contents: read
runs-on: macos-15
timeout-minutes: 10
strategy:
fail-fast: false
matrix:
arch: [arm64, x86_64]
include:
- arch: arm64
sdk: iphoneos
target: aarch64-apple-ios11.0
flags: "-arch arm64 -mios-version-min=11.0"
- arch: x86_64
sdk: iphonesimulator
target: x86_64-apple-ios11.0-simulator
flags: "-arch x86_64 -mios-simulator-version-min=11.0"
steps:
- uses: Cysharp/Actions/.github/actions/checkout@main
with:
submodules: recursive
fetch-depth: 1
- name: Update submodule to latest release
if: ${{ needs.check_new_release.outputs.needs_update == 'true' }}
run: |
cd zstd
git fetch --tags
git checkout ${{ needs.check_new_release.outputs.latest_tag }}
cd ..
echo "Updated zstd submodule to ${{ needs.check_new_release.outputs.latest_tag }}"
# `lib-release + -DZSTD_MULTITHREAD` build single-threaded static lib, so use `lib-mt + -O3` instead
# -fPIC is not needed on static library
- name: Build static library for iOS
run: |
SDK_PATH=$(xcrun --sdk ${{ matrix.sdk }} --show-sdk-path)
# Build static library (iOS App Store requires static linking)
cd zstd/lib
make lib-mt CC="clang" CFLAGS="-O3 ${{ matrix.flags }} -flto -isysroot $SDK_PATH" DEBUGFLAGS="" -j "$(sysctl -n hw.ncpu)"
ls -la libzstd.a
file libzstd.a
lipo -info libzstd.a
- name: Prepare artifacts
run: |
mkdir -p artifacts/ios-${{ matrix.arch }}
cp zstd/lib/libzstd.a artifacts/ios-${{ matrix.arch }}/
- name: Upload iOS artifacts
uses: Cysharp/Actions/.github/actions/upload-artifact@main
with:
name: zstd-ios-${{ matrix.arch }}
path: artifacts/ios-${{ matrix.arch }}/*
build-android:
needs: [check_new_release]
if: ${{ needs.check_new_release.outputs.needs_update == 'true' || github.event_name == 'workflow_dispatch' }}
permissions:
contents: read
runs-on: ubuntu-24.04
timeout-minutes: 10
strategy:
matrix:
abi: [armeabi-v7a, arm64-v8a, x86_64]
include:
- abi: armeabi-v7a
arch: arm # Unity require this for mobile app
target: armv7a-linux-androideabi21
cc: armv7a-linux-androideabi21-clang
- abi: arm64-v8a
arch: arm64 # Most mobile app
target: aarch64-linux-android21
cc: aarch64-linux-android21-clang
- abi: x86_64
arch: x86_64 # Chromebook and Google Play for PC (not emulation)
target: x86_64-linux-android21
cc: x86_64-linux-android21-clang
steps:
- uses: Cysharp/Actions/.github/actions/checkout@main
with:
submodules: recursive
fetch-depth: 1
- name: Update submodule to latest release
if: ${{ needs.check_new_release.outputs.needs_update == 'true' }}
run: |
cd zstd
git fetch --tags
git checkout ${{ needs.check_new_release.outputs.latest_tag }}
cd ..
echo "Updated zstd submodule to ${{ needs.check_new_release.outputs.latest_tag }}"
- name: Setup Android NDK
uses: nttld/setup-ndk@afb4c9964b521afb97c864b7d40b11e6911bd410 # v1.5.0
id: setup-ndk
with:
ndk-version: r26d
add-to-path: false
- name: Build shared library for Android
run: |
export PATH="${{ steps.setup-ndk.outputs.ndk-path }}/toolchains/llvm/prebuilt/linux-x86_64/bin:$PATH"
cd zstd/lib
make lib-release CC="${{ matrix.cc }}" MOREFLAGS="-DZSTD_MULTITHREAD -fPIC -flto" -j "$(nproc)"
ls -la libzstd.so*
file libzstd.so*
- name: Prepare artifacts
run: |
mkdir -p artifacts/android-${{ matrix.arch }}
cp zstd/lib/libzstd.so artifacts/android-${{ matrix.arch }}/
- name: Upload Android artifacts
uses: Cysharp/Actions/.github/actions/upload-artifact@main
with:
name: zstd-android-${{ matrix.arch }}
path: artifacts/android-${{ matrix.arch }}/*
package-all:
needs:
[
check_new_release,
build-windows,
build-windows-arm64,
build-linux,
build-macos,
build-ios,
build-android,
]
if: ${{ needs.check_new_release.outputs.needs_update == 'true' || github.event_name == 'workflow_dispatch' }}
permissions:
contents: write
runs-on: ubuntu-24.04
timeout-minutes: 5
steps:
- name: Download all artifacts
uses: Cysharp/Actions/.github/actions/download-artifact@main
with:
path: artifacts
- name: Create .NET runtime structure
run: |
mkdir -p release/runtimes/{win-x64,win-arm64}/native
mkdir -p release/runtimes/{linux-x64,linux-arm64}/native
mkdir -p release/runtimes/{osx-x64,osx-arm64}/native
mkdir -p release/runtimes/{android-arm,android-arm64,android-x64}/native
mkdir -p release/runtimes/{ios-arm64,ios-x64}/native
# Windows
if [ -f artifacts/zstd-win-x64/libzstd.dll ]; then
cp -f artifacts/zstd-win-x64/libzstd.dll release/runtimes/win-x64/native/libzstd.dll
fi
if [ -f artifacts/zstd-win-arm64/libzstd.dll ]; then
cp -f artifacts/zstd-win-arm64/libzstd.dll release/runtimes/win-arm64/native/libzstd.dll
fi
# Linux
if [ -f artifacts/zstd-linux-x64/libzstd.so ]; then
cp artifacts/zstd-linux-x64/libzstd.so release/runtimes/linux-x64/native/
fi
if [ -f artifacts/zstd-linux-arm64/libzstd.so ]; then
cp artifacts/zstd-linux-arm64/libzstd.so release/runtimes/linux-arm64/native/
fi
# macOS
for file in artifacts/zstd-osx-x64/*.dylib; do
[ -f "$file" ] && cp "$file" release/runtimes/osx-x64/native/libzstd.dylib && break
done
for file in artifacts/zstd-osx-arm64/*.dylib; do
[ -f "$file" ] && cp "$file" release/runtimes/osx-arm64/native/libzstd.dylib && break
done
# iOS
if [ -f artifacts/zstd-ios-arm64/libzstd.a ]; then
cp artifacts/zstd-ios-arm64/libzstd.a release/runtimes/ios-arm64/native/
fi
if [ -f artifacts/zstd-ios-x86_64/libzstd.a ]; then
cp artifacts/zstd-ios-x86_64/libzstd.a release/runtimes/ios-x64/native/
fi
# Android
if [ -f artifacts/zstd-android-arm/libzstd.so ]; then
cp artifacts/zstd-android-arm/libzstd.so release/runtimes/android-arm/native/
fi
if [ -f artifacts/zstd-android-arm64/libzstd.so ]; then
cp artifacts/zstd-android-arm64/libzstd.so release/runtimes/android-arm64/native/
fi
if [ -f artifacts/zstd-android-x86_64/libzstd.so ]; then
cp artifacts/zstd-android-x86_64/libzstd.so release/runtimes/android-x64/native/
fi
- name: Create archives
run: |
cd release
tar czf ../zstd-native-libraries.tar.gz .
zip -r ../zstd-native-libraries.zip .
- name: Upload package
uses: Cysharp/Actions/.github/actions/upload-artifact@main
with:
name: zstd-native-package
path: |
zstd-native-libraries.tar.gz
create-pr:
needs: [check_new_release, package-all]
if: ${{ needs.check_new_release.outputs.needs_update == 'true' }}
permissions:
contents: write
pull-requests: write
env:
NEW_BRANCH_NAME: "automate/zstd-${{ needs.check_new_release.outputs.latest_tag }}"
LATEST_TAG: ${{ needs.check_new_release.outputs.latest_tag }}
runs-on: ubuntu-24.04
timeout-minutes: 15
steps:
- uses: Cysharp/Actions/.github/actions/checkout@main
with:
submodules: recursive
- name: Configure Git
run: |
git config --local user.email "41898282+github-actions[bot]@users.noreply.github.com"
git config --local user.name "github-actions[bot]"
- name: Create new branch
run: git checkout -b "${NEW_BRANCH_NAME}"
# update submodule
- name: Update submodule to latest release
if: ${{ needs.check_new_release.outputs.needs_update == 'true' }}
run: |
echo "Updated zstd submodule to ${LATEST_TAG}"
cd zstd
git fetch --tags
git checkout "${LATEST_TAG}"
cd ..
git add zstd
git commit -m "automate: Update ZSTD submodule to ${LATEST_TAG}"
# update runtimes
- name: Download all artifacts
uses: Cysharp/Actions/.github/actions/download-artifact@main
with:
path: artifacts
- name: Place new runtimes
env:
DEST_BASE_PATH: src/NativeCompressions.Zstandard.Runtime/runtimes
run: |
echo "Updated zstd runtimes to ${LATEST_TAG}"
# Create directory structure
for arch in linux-x64 linux-arm64 osx-x64 osx-arm64 win-x64 win-arm64 ios-arm64 ios-x64 android-arm android-arm64 android-x64; do
mkdir -p "${DEST_BASE_PATH}/${arch}/native"
done
# Show current
echo "::group::Show artifacts/"
ls -lR artifacts/
echo "::endgroup::"
echo "::group::Show ${DEST_BASE_PATH}/"
ls -lR "${DEST_BASE_PATH}/"
echo "::endgroup::"
echo "::group::Copy new files"
# Windows
if [ -f artifacts/zstd-win-x64/libzstd.dll ]; then
cp -f artifacts/zstd-win-x64/libzstd.dll "${DEST_BASE_PATH}/win-x64/native/libzstd.dll"
fi
if [ -f artifacts/zstd-win-arm64/libzstd.dll ]; then
cp -f artifacts/zstd-win-arm64/libzstd.dll "${DEST_BASE_PATH}/win-arm64/native/libzstd.dll"
fi
# Linux
if [ -f artifacts/zstd-linux-x64/libzstd.so ]; then
cp -f artifacts/zstd-linux-x64/libzstd.so "${DEST_BASE_PATH}/linux-x64/native/libzstd.so"
fi
if [ -f artifacts/zstd-linux-arm64/libzstd.so ]; then
cp -f artifacts/zstd-linux-arm64/libzstd.so "${DEST_BASE_PATH}/linux-arm64/native/libzstd.so"
fi
# macOS
if [ -f artifacts/zstd-osx-x64/libzstd.dylib ]; then
cp -f artifacts/zstd-osx-x64/libzstd.dylib "${DEST_BASE_PATH}/osx-x64/native/libzstd.dylib"
fi
if [ -f artifacts/zstd-osx-arm64/libzstd.dylib ]; then
cp -f artifacts/zstd-osx-arm64/libzstd.dylib "${DEST_BASE_PATH}/osx-arm64/native/libzstd.dylib"
fi
# iOS
if [ -f artifacts/zstd-ios-arm64/libzstd.a ]; then
cp -f artifacts/zstd-ios-arm64/libzstd.a "${DEST_BASE_PATH}/ios-arm64/native/libzstd.a"
fi
if [ -f artifacts/zstd-ios-x86_64/libzstd.a ]; then
cp -f artifacts/zstd-ios-x86_64/libzstd.a "${DEST_BASE_PATH}/ios-x64/native/libzstd.a"
fi
# Android
if [ -f artifacts/zstd-android-arm/libzstd.so ]; then
cp -f artifacts/zstd-android-arm/libzstd.so "${DEST_BASE_PATH}/android-arm/native/libzstd.so"
fi
if [ -f artifacts/zstd-android-arm64/libzstd.so ]; then
cp -f artifacts/zstd-android-arm64/libzstd.so "${DEST_BASE_PATH}/android-arm64/native/libzstd.so"
fi
if [ -f artifacts/zstd-android-x86_64/libzstd.so ]; then
cp -f artifacts/zstd-android-x86_64/libzstd.so "${DEST_BASE_PATH}/android-x64/native/libzstd.so"
fi
echo "::endgroup::"
echo "::group::git commit"
# Commit runtime updates
git add "${DEST_BASE_PATH}"
git commit -m "automate: Update Zstandard native libraries to ${LATEST_TAG}"
echo "::endgroup::"
# create PR
- name: Push branch and create PR
id: create_pr
env:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
run: |
# Push the branch
git push origin "${NEW_BRANCH_NAME}"
cat <<'EOF' > artifacts/pr_body.txt
This PR updates Zstandard to the latest release `${{ needs.check_new_release.outputs.latest_tag }}` ([${{ needs.check_new_release.outputs.latest_commit }}](https://github.com/facebook/zstd/commit/${{ needs.check_new_release.outputs.latest_commit }}))
## Changes
- Updated Zstandard submodule to ${{ needs.check_new_release.outputs.latest_tag }}
- Updated native libraries for all platforms:
- Windows x64/ARM64
- Linux x64/ARM64
- macOS x64/ARM64
- Android ARM/ARM64/x64
- iOS ARM64/x64 (device/simulator)
## Build artifacts
All native libraries have been built and tested in CI.
@neuecc
EOF
# Create PR and capture PR number
PR_URL=$(gh pr create --title "automate: Update Zstandard to ${LATEST_TAG}" --base main --head "${NEW_BRANCH_NAME}" --body-file artifacts/pr_body.txt)
PR_NUMBER=$(echo "$PR_URL" | grep -o '[0-9]\+$')
echo "pr_number=$PR_NUMBER" | tee -a "$GITHUB_OUTPUT"
echo "Created PR #$PR_NUMBER: $PR_URL"
build:
needs: [check_new_release, create-pr]
if: ${{ needs.create-pr.result == 'success' }}
permissions:
contents: read
uses: ./.github/workflows/build-debug.yaml
with:
git-ref: "automate/zstd-${{ needs.check_new_release.outputs.latest_tag }}"