Skip to content

Build-Native-OpenZL #251

Build-Native-OpenZL

Build-Native-OpenZL #251

name: Build-Native-OpenZL
# Follow OpenZL latest release by cron update
# 1. Check for OpenZL updates - compare submodule commit ID with latest release commit
# 2. If no changes, checkout in each build job, build and upload artifacts.
# 2". If changes exist, update submodule after checkout in each build job, build and upload artifacts
# 3. Upload packages to GitHub Actions summary
# 4. If changes exist, Create a branch and PR. Separate PRs for each OpenZL release are preferred (avoid updating in a single PR)
# PR includes following changes.
# - Commit that updates the submodule
# - Commit that openzl dynamic lib artifacts placed in src/NativeCompressions.OpenZL.Runtime/runtimes/{platform_name}/native/{lib}
on:
workflow_dispatch:
schedule:
- cron: "0 3 * * *" # Every day at 3:00 AM (UTC)
jobs:
check_new_release:
permissions:
contents: read
uses: ./.github/workflows/check-new-release.yaml
with:
repository: facebook/openzl
submodule_path: openzl
build-windows:
needs: [check_new_release]
if: ${{ needs.check_new_release.outputs.needs_update == 'true' || github.event_name == 'workflow_dispatch' }}
strategy:
matrix:
arch: [x64]
include:
- arch: x64
platform: x64
msys_env: MINGW64
permissions:
contents: read
runs-on: windows-2025
timeout-minutes: 5
steps:
- uses: Cysharp/Actions/.github/actions/checkout@main
with:
submodules: recursive
fetch-depth: 1
- name: Update submodule to latest release
if: ${{ needs.check_new_release.outputs.needs_update == 'true' }}
run: |
cd openzl
git fetch --tags
git checkout ${{ needs.check_new_release.outputs.latest_tag }}
cd ..
echo "Updated OpenZL submodule to ${{ needs.check_new_release.outputs.latest_tag }}"
- name: Setup MSYS2
uses: msys2/setup-msys2@4f806de0a5a7294ffabaff804b38a9b435a73bda # v2.30.0
with:
msystem: ${{ matrix.msys_env }}
update: true
install: >-
${{ matrix.arch == 'x64' && 'mingw-w64-x86_64-gcc mingw-w64-x86_64-make' || 'mingw-w64-clang-aarch64-gcc mingw-w64-clang-aarch64-make' }}
make
# TODO: Currently, the build artifact extension is .so even on Windows. OpenZL may update the Makefile in the future to create .dll files for Windows, which could cause CI errors. Please update the file extensions accordingly if this change occurs.
- name: Build DLL
shell: msys2 {0}
run: |
cd openzl
make libopenzl.so BUILD_TYPE=OPT OS=Windows_NT MOREFLAGS="-flto" -j "$(nproc)"
ls -la libopenzl.so
file -L libopenzl.so
- name: Prepare artifacts
shell: bash
run: |
mkdir -p artifacts/win-${{ matrix.arch }}
cp -L openzl/libopenzl.so artifacts/win-${{ matrix.arch }}/libopenzl.dll
- name: Upload Windows artifacts
uses: Cysharp/Actions/.github/actions/upload-artifact@main
with:
name: openzl-win-${{ matrix.arch }}
path: artifacts/win-${{ matrix.arch }}/*
build-windows-arm64:
needs: [check_new_release]
if: ${{ needs.check_new_release.outputs.needs_update == 'true' || github.event_name == 'workflow_dispatch' }}
permissions:
contents: read
runs-on: ubuntu-24.04
timeout-minutes: 5
steps:
- uses: Cysharp/Actions/.github/actions/checkout@main
with:
submodules: recursive
- name: Update submodule to latest release
if: ${{ needs.check_new_release.outputs.needs_update == 'true' }}
run: |
cd openzl
git fetch --tags
git checkout ${{ needs.check_new_release.outputs.latest_tag }}
cd ..
echo "Updated openzl submodule to ${{ needs.check_new_release.outputs.latest_tag }}"
# TODO: Currently, the build artifact extension is .so even on Windows. OpenZL may update the Makefile in the future to create .dll files for Windows, which could cause CI errors. Please update the file extensions accordingly if this change occurs.
- name: Build using Docker
run: |
docker run --rm -v "$PWD:/work" \
mstorsjo/llvm-mingw:latest \
bash -c "
git config --global --add safe.directory /work/openzl && \
cd /work/openzl && \
export CC=aarch64-w64-mingw32-gcc && \
export WINDRES=aarch64-w64-mingw32-windres && \
export TARGET_OS=MINGW64 && \
export TARGET_SYSTEM=Windows_NT && \
make libopenzl.so BUILD_TYPE=OPT OS=Windows_NT \
CC=aarch64-w64-mingw32-gcc \
MOREFLAGS='-flto'"
# NOTE: Environment exports (CC, WINDRES, TARGET_OS, TARGET_SYSTEM) propagate to
# dependency sub-makes (lz4, zstd) whose MAKEOVERRIDES is cleared by OpenZL's Makefile.
# Command-line CC= is also needed for the parent make (highest priority in GNU Make).
# TARGET_OS=MINGW64 (not Windows_NT) is required because lz4's Makefile.inc produces
# 'liblz4.dll' for MINGW% but 'liblz4-1.dll' for Windows%, and OpenZL expects the former.
file -L openzl/libopenzl.so
- name: Prepare artifacts
run: |
mkdir -p artifacts/win-arm64
cp -L openzl/libopenzl.so artifacts/win-arm64/libopenzl.dll
- name: Upload artifacts
uses: Cysharp/Actions/.github/actions/upload-artifact@main
with:
name: openzl-win-arm64
path: artifacts/win-arm64/*
build-linux:
needs: [check_new_release]
if: ${{ needs.check_new_release.outputs.needs_update == 'true' || github.event_name == 'workflow_dispatch' }}
permissions:
contents: read
runs-on: ubuntu-24.04
timeout-minutes: 5
strategy:
matrix:
arch: [x64, arm64]
include:
- arch: x64
cc: gcc
- arch: arm64
cc: aarch64-linux-gnu-gcc
packages: gcc-aarch64-linux-gnu
steps:
- uses: Cysharp/Actions/.github/actions/checkout@main
with:
submodules: recursive
fetch-depth: 1
- name: Update submodule to latest release
if: ${{ needs.check_new_release.outputs.needs_update == 'true' }}
run: |
cd openzl
git fetch --tags
git checkout ${{ needs.check_new_release.outputs.latest_tag }}
cd ..
echo "Updated openzl submodule to ${{ needs.check_new_release.outputs.latest_tag }}"
- name: Install ARM64 cross-compiler
if: matrix.arch == 'arm64'
run: |
sudo apt-get update
sudo apt-get install -y ${{ matrix.packages }}
- name: Build shared library
run: |
cd openzl
make libopenzl.so BUILD_TYPE=OPT CC="${{ matrix.cc }}" MOREFLAGS="-flto" -j "$(nproc)"
ls -la ./*.so*
file -L libopenzl.so*
- name: Prepare artifacts
run: |
mkdir -p artifacts/linux-${{ matrix.arch }}
cp -L openzl/libopenzl.so artifacts/linux-${{ matrix.arch }}/
- name: Upload Linux artifacts
uses: Cysharp/Actions/.github/actions/upload-artifact@main
with:
name: openzl-linux-${{ matrix.arch }}
path: artifacts/linux-${{ matrix.arch }}/*
build-macos:
needs: [check_new_release]
if: ${{ needs.check_new_release.outputs.needs_update == 'true' || github.event_name == 'workflow_dispatch' }}
permissions:
contents: read
strategy:
matrix:
arch: [x64, arm64]
include:
- arch: x64
flags: "-arch x86_64"
- arch: arm64
flags: "-arch arm64"
runs-on: macos-15
timeout-minutes: 5
steps:
- uses: Cysharp/Actions/.github/actions/checkout@main
with:
submodules: recursive
- name: Update submodule to latest release
if: ${{ needs.check_new_release.outputs.needs_update == 'true' }}
run: |
cd openzl
git fetch --tags
git checkout ${{ needs.check_new_release.outputs.latest_tag }}
cd ..
echo "Updated openzl submodule to ${{ needs.check_new_release.outputs.latest_tag }}"
# TODO: Currently, the build artifact extension is .so even on macOS. OpenZL may update the Makefile in the future to create .dylib files for macOS, which could cause CI errors. Please update the file extensions accordingly if this change occurs.
# NOTE: Arch flags must reach dependency sub-makes (lz4, zstd) whose MAKEOVERRIDES is
# cleared by OpenZL's Makefile. CC with embedded arch is the most robust method because
# zstd uses SET_CACHE_DIRECTORY which re-invokes make and may not preserve env CFLAGS.
# CXXFLAGS is needed because OpenZL's addTargetAsmObject macro uses $(CC) $(CXXFLAGS)
# for .S files (not CFLAGS). MOREFLAGS only carries -flto for the parent make.
- name: Build dynamic library
env:
CC: "clang ${{ matrix.flags }}"
CFLAGS: ${{ matrix.flags }}
CXXFLAGS: ${{ matrix.flags }}
LDFLAGS: ${{ matrix.flags }}
run: |
cd openzl
make libopenzl.so BUILD_TYPE=OPT MOREFLAGS="-flto" -j "$(sysctl -n hw.ncpu)"
ls -la
lipo -info libopenzl.so
file -L libopenzl.so
- name: Prepare artifacts
run: |
mkdir -p artifacts/osx-${{ matrix.arch }}
cp -L openzl/libopenzl.so artifacts/osx-${{ matrix.arch }}/libopenzl.dylib
- name: Upload artifacts
uses: Cysharp/Actions/.github/actions/upload-artifact@main
with:
name: openzl-osx-${{ matrix.arch }}
path: artifacts/osx-${{ matrix.arch }}/*
build-ios:
needs: [check_new_release]
if: ${{ needs.check_new_release.outputs.needs_update == 'true' || github.event_name == 'workflow_dispatch' }}
permissions:
contents: read
runs-on: macos-15
timeout-minutes: 10
strategy:
fail-fast: false
matrix:
arch: [arm64, x86_64]
include:
- arch: arm64
sdk: iphoneos
target: aarch64-apple-ios11.0
cflags: "-arch arm64 -mios-version-min=11.0"
- arch: x86_64
sdk: iphonesimulator
target: x86_64-apple-ios11.0-simulator
cflags: "-arch x86_64 -mios-simulator-version-min=11.0"
steps:
- uses: Cysharp/Actions/.github/actions/checkout@main
with:
submodules: recursive
fetch-depth: 1
- name: Update submodule to latest release
if: ${{ needs.check_new_release.outputs.needs_update == 'true' }}
run: |
cd openzl
git fetch --tags
git checkout ${{ needs.check_new_release.outputs.latest_tag }}
cd ..
echo "Updated openzl submodule to ${{ needs.check_new_release.outputs.latest_tag }}"
- name: Build static library for iOS
run: |
SDK_PATH=$(xcrun --sdk ${{ matrix.sdk }} --show-sdk-path)
# Build static library (iOS App Store requires static linking)
cd openzl
make libopenzl.a BUILD_TYPE=OPT CC="clang" MOREFLAGS="${{ matrix.cflags }} -isysroot $SDK_PATH" -j "$(sysctl -n hw.ncpu)"
ls -la
file -L libopenzl.a
lipo -info libopenzl.a
- name: Prepare artifacts
run: |
mkdir -p artifacts/ios-${{ matrix.arch }}
cp -L openzl/libopenzl.a artifacts/ios-${{ matrix.arch }}/
- name: Upload iOS artifacts
uses: Cysharp/Actions/.github/actions/upload-artifact@main
with:
name: openzl-ios-${{ matrix.arch }}
path: artifacts/ios-${{ matrix.arch }}/*
build-android:
needs: [check_new_release]
if: ${{ needs.check_new_release.outputs.needs_update == 'true' || github.event_name == 'workflow_dispatch' }}
permissions:
contents: read
runs-on: ubuntu-24.04
timeout-minutes: 10
strategy:
matrix:
# TODO: Building with armeabi-v7a is currently not possible due to the following build error: `sizeof(HeapMeta) == 16: sizeof(HeapMeta) must be 16 to guarantee alignment, but the expression evaluates to '8 == 16'.`
# .This error indicates that OpenZL hasn't support for 32-bit architectures currently (including armeabi-v7a).
abi: [arm64-v8a, x86_64]
include:
# - abi: armeabi-v7a
# arch: arm # Unity require this for mobile app
# target: armv7a-linux-androideabi21
# cc: armv7a-linux-androideabi21-clang
- abi: arm64-v8a
arch: arm64 # Most mobile app
target: aarch64-linux-android21
cc: aarch64-linux-android21-clang
- abi: x86_64
arch: x86_64 # Chromebook and Google Play for PC (not emulation)
target: x86_64-linux-android21
cc: x86_64-linux-android21-clang
steps:
- uses: Cysharp/Actions/.github/actions/checkout@main
with:
submodules: recursive
fetch-depth: 1
- name: Update submodule to latest release
if: ${{ needs.check_new_release.outputs.needs_update == 'true' }}
run: |
cd openzl
git fetch --tags
git checkout ${{ needs.check_new_release.outputs.latest_tag }}
cd ..
echo "Updated openzl submodule to ${{ needs.check_new_release.outputs.latest_tag }}"
- name: Setup Android NDK
uses: nttld/setup-ndk@afb4c9964b521afb97c864b7d40b11e6911bd410 # v1.5.0
id: setup-ndk
with:
ndk-version: r26d
add-to-path: false
- name: Build shared library for Android
run: |
export PATH="${{ steps.setup-ndk.outputs.ndk-path }}/toolchains/llvm/prebuilt/linux-x86_64/bin:$PATH"
cd openzl
make libopenzl.so BUILD_TYPE=OPT CC="${{ matrix.cc }}" MOREFRAGS="-flto" -j "$(nproc)"
ls -la libopenzl.so
file -L libopenzl.so
- name: Prepare artifacts
run: |
mkdir -p artifacts/android-${{ matrix.arch }}
cp -L openzl/libopenzl.so artifacts/android-${{ matrix.arch }}/
- name: Upload Android artifacts
uses: Cysharp/Actions/.github/actions/upload-artifact@main
with:
name: openzl-android-${{ matrix.arch }}
path: artifacts/android-${{ matrix.arch }}/*
package-all:
needs:
[
check_new_release,
build-windows,
build-windows-arm64,
build-linux,
build-macos,
build-ios,
build-android,
]
if: ${{ needs.check_new_release.outputs.needs_update == 'true' || github.event_name == 'workflow_dispatch' }}
permissions:
contents: write
runs-on: ubuntu-24.04
timeout-minutes: 5
steps:
- name: Download all artifacts
uses: Cysharp/Actions/.github/actions/download-artifact@main
with:
path: artifacts
- name: Create .NET runtime structure
run: |
mkdir -p release/runtimes/{win-x64,win-arm64}/native
mkdir -p release/runtimes/{linux-x64,linux-arm64}/native
mkdir -p release/runtimes/{osx-x64,osx-arm64}/native
mkdir -p release/runtimes/{android-arm,android-arm64,android-x64}/native
mkdir -p release/runtimes/{ios-arm64,ios-x64}/native
# Windows
if [ -f artifacts/openzl-win-x64/libopenzl.dll ]; then
cp artifacts/openzl-win-x64/libopenzl.dll release/runtimes/win-x64/native/
fi
if [ -f artifacts/openzl-win-arm64/libopenzl.dll ]; then
cp artifacts/openzl-win-arm64/libopenzl.dll release/runtimes/win-arm64/native/
fi
# Linux
if [ -f artifacts/openzl-linux-x64/libopenzl.so ]; then
cp artifacts/openzl-linux-x64/libopenzl.so release/runtimes/linux-x64/native/
fi
if [ -f artifacts/openzl-linux-arm64/libopenzl.so ]; then
cp artifacts/openzl-linux-arm64/libopenzl.so release/runtimes/linux-arm64/native/
fi
# macOS
for file in artifacts/openzl-osx-x64/*.dylib; do
[ -f "$file" ] && cp "$file" release/runtimes/osx-x64/native/libopenzl.dylib && break
done
for file in artifacts/openzl-osx-arm64/*.dylib; do
[ -f "$file" ] && cp "$file" release/runtimes/osx-arm64/native/libopenzl.dylib && break
done
# iOS
if [ -f artifacts/openzl-ios-arm64/libopenzl.a ]; then
cp artifacts/openzl-ios-arm64/libopenzl.a release/runtimes/ios-arm64/native/
fi
if [ -f artifacts/openzl-ios-x86_64/libopenzl.a ]; then
cp artifacts/openzl-ios-x86_64/libopenzl.a release/runtimes/ios-x64/native/
fi
# Android
if [ -f artifacts/openzl-android-arm/libopenzl.so ]; then
cp artifacts/openzl-android-arm/libopenzl.so release/runtimes/android-arm/native/
fi
if [ -f artifacts/openzl-android-arm64/libopenzl.so ]; then
cp artifacts/openzl-android-arm64/libopenzl.so release/runtimes/android-arm64/native/
fi
if [ -f artifacts/openzl-android-x86_64/libopenzl.so ]; then
cp artifacts/openzl-android-x86_64/libopenzl.so release/runtimes/android-x64/native/
fi
- name: Create archives
run: |
cd release
tar czf ../openzl-native-libraries.tar.gz .
zip -r ../openzl-native-libraries.zip .
- name: Upload package
uses: Cysharp/Actions/.github/actions/upload-artifact@main
with:
name: openzl-native-package
path: |
openzl-native-libraries.tar.gz
create-pr:
needs: [check_new_release, package-all]
if: ${{ needs.check_new_release.outputs.needs_update == 'true' }}
permissions:
contents: write
pull-requests: write
env:
NEW_BRANCH_NAME: "automate/openzl-${{ needs.check_new_release.outputs.latest_tag }}"
LATEST_TAG: ${{ needs.check_new_release.outputs.latest_tag }}
runs-on: ubuntu-24.04
timeout-minutes: 15
steps:
- uses: Cysharp/Actions/.github/actions/checkout@main
with:
submodules: recursive
- name: Configure Git
run: |
git config --local user.email "41898282+github-actions[bot]@users.noreply.github.com"
git config --local user.name "github-actions[bot]"
- name: Create new branch
run: git checkout -b "${NEW_BRANCH_NAME}"
# update submodule
- name: Update submodule to latest release
if: ${{ needs.check_new_release.outputs.needs_update == 'true' }}
run: |
echo "Updated openzl submodule to ${LATEST_TAG}"
cd openzl
git fetch --tags
git checkout "${LATEST_TAG}"
cd ..
git add openzl
git commit -m "automate: Update OpenZL submodule to ${LATEST_TAG}"
# update runtimes
- name: Download all artifacts
uses: Cysharp/Actions/.github/actions/download-artifact@main
with:
path: artifacts
- name: Place new runtimes
env:
DEST_BASE_PATH: src/NativeCompressions.OpenZL.Runtime/runtimes
run: |
echo "Updated openzl runtimes to ${LATEST_TAG}"
for arch in linux-x64 linux-arm64 osx-x64 osx-arm64 win-x64 win-arm64 ios-arm64 ios-x64 android-arm android-arm64 android-x64; do
mkdir -p "${DEST_BASE_PATH}/${arch}/native"
done
# Show current
echo "::group::Show artifacts/"
ls -lR artifacts/
echo "::endgroup::"
echo "::group::Show ${DEST_BASE_PATH}/"
ls -lR "${DEST_BASE_PATH}/"
echo "::endgroup::"
echo "::group::Copy new files"
# Windows
if [ -f artifacts/openzl-win-x64/libopenzl.dll ]; then
cp -f artifacts/openzl-win-x64/libopenzl.dll "${DEST_BASE_PATH}/win-x64/native/libopenzl.dll"
fi
if [ -f artifacts/openzl-win-arm64/libopenzl.dll ]; then
cp -f artifacts/openzl-win-arm64/libopenzl.dll "${DEST_BASE_PATH}/win-arm64/native/libopenzl.dll"
fi
# Linux
if [ -f artifacts/openzl-linux-x64/libopenzl.so ]; then
cp -f artifacts/openzl-linux-x64/libopenzl.so "${DEST_BASE_PATH}/linux-x64/native/libopenzl.so"
fi
if [ -f artifacts/openzl-linux-arm64/libopenzl.so ]; then
cp -f artifacts/openzl-linux-arm64/libopenzl.so "${DEST_BASE_PATH}/linux-arm64/native/libopenzl.so"
fi
# macOS
if [ -f artifacts/openzl-osx-x64/libopenzl.dylib ]; then
cp -f artifacts/openzl-osx-x64/libopenzl.dylib "${DEST_BASE_PATH}/osx-x64/native/libopenzl.dylib"
fi
if [ -f artifacts/openzl-osx-arm64/libopenzl.dylib ]; then
cp -f artifacts/openzl-osx-arm64/libopenzl.dylib "${DEST_BASE_PATH}/osx-arm64/native/libopenzl.dylib"
fi
# iOS
if [ -f artifacts/openzl-ios-arm64/libopenzl.a ]; then
cp -f artifacts/openzl-ios-arm64/libopenzl.a "${DEST_BASE_PATH}/ios-arm64/native/libopenzl.a"
fi
if [ -f artifacts/openzl-ios-x86_64/libopenzl.a ]; then
cp -f artifacts/openzl-ios-x86_64/libopenzl.a "${DEST_BASE_PATH}/ios-x64/native/libopenzl.a"
fi
# Android
if [ -f artifacts/openzl-android-arm/libopenzl.so ]; then
cp -f artifacts/openzl-android-arm/libopenzl.so "${DEST_BASE_PATH}/android-arm/native/libopenzl.so"
fi
if [ -f artifacts/openzl-android-arm64/libopenzl.so ]; then
cp -f artifacts/openzl-android-arm64/libopenzl.so "${DEST_BASE_PATH}/android-arm64/native/libopenzl.so"
fi
if [ -f artifacts/openzl-android-x86_64/libopenzl.so ]; then
cp -f artifacts/openzl-android-x86_64/libopenzl.so "${DEST_BASE_PATH}/android-x64/native/libopenzl.so"
fi
echo "::endgroup::"
echo "::group::git commit"
# Commit runtime updates
git add "${DEST_BASE_PATH}"
git commit -m "automate: Update OpenZL native libraries to ${LATEST_TAG}"
echo "::endgroup::"
# create PR
- name: Push branch and create PR
id: create_pr
env:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
run: |
# Push the branch
git push origin "${NEW_BRANCH_NAME}"
cat <<'EOF' > artifacts/pr_body.txt
This PR updates OpenZL to the latest release `${{ needs.check_new_release.outputs.latest_tag }}` ([${{ needs.check_new_release.outputs.latest_commit }}](https://github.com/facebook/openzl/commit/${{ needs.check_new_release.outputs.latest_commit }}))
## Changes
- Updated OpenZL submodule to ${{ needs.check_new_release.outputs.latest_tag }}
- Updated native libraries for all platforms:
- Windows x64/ARM64
- Linux x64/ARM64
- macOS x64/ARM64
- Android ARM64/x64 (ARM not supported by OpenZL)
- iOS ARM64/x64 (device/simulator)
## Build artifacts
All native libraries have been built and tested in CI.
@neuecc
EOF
# Create PR and capture PR number
PR_URL=$(gh pr create --title "automate: Update OpenZL to ${LATEST_TAG}" --base main --head "${NEW_BRANCH_NAME}" --body-file artifacts/pr_body.txt)
PR_NUMBER=$(echo "$PR_URL" | grep -o '[0-9]\+$')
echo "pr_number=$PR_NUMBER" | tee -a "$GITHUB_OUTPUT"
echo "Created PR #$PR_NUMBER: $PR_URL"
build:
needs: [check_new_release, create-pr]
if: ${{ needs.create-pr.result == 'success' }}
permissions:
contents: read
uses: ./.github/workflows/build-debug.yaml
with:
git-ref: "automate/openzl-${{ needs.check_new_release.outputs.latest_tag }}"