Skip to content

Commit c492abf

Browse files
tazarovtazarov
committed
chore: reduce workflow redundancy with composite actions
Create reusable composite actions to eliminate duplication across CI workflows: - setup-rust: Standardizes Rust toolchain setup with caching - setup-cross-compilation: Handles cross-compilation dependencies - build-rust-library: Builds Rust library for any target platform - get-rust-library: Downloads release or builds library as fallback Workflow improvements: - Deleted cross-compile.yml (redundant with rust-ci.yml) - Updated all workflows to use new composite actions - Reduced overall workflow code by ~550 lines (~60% reduction) - Made workflows more maintainable and consistent Closes #11
1 parent 24abfe2 commit c492abf

11 files changed

Lines changed: 313 additions & 551 deletions

File tree

‎.github/actions/build-rust-library/action.yml‎

Lines changed: 87 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,87 @@
1+
name: Build Rust Library
2+
description: Build Rust library for specified target
3+
4+
inputs:
5+
target:
6+
description: 'Rust target triple (e.g., x86_64-unknown-linux-gnu)'
7+
required: true
8+
use-cross:
9+
description: 'Use cross for building (default: auto-detect for Linux)'
10+
required: false
11+
default: 'auto'
12+
use-zigbuild:
13+
description: 'Use cargo-zigbuild for building'
14+
required: false
15+
default: 'false'
16+
17+
outputs:
18+
library-path:
19+
description: 'Path to the built library file'
20+
value: ${{ steps.set-lib-path.outputs.path }}
21+
22+
runs:
23+
using: composite
24+
steps:
25+
- name: Clean target directory (Unix)
26+
if: runner.os != 'Windows'
27+
shell: bash
28+
run: rm -rf target
29+
30+
- name: Clean target directory (Windows)
31+
if: runner.os == 'Windows'
32+
shell: pwsh
33+
run: Remove-Item -Recurse -Force target -ErrorAction SilentlyContinue
34+
35+
- name: Build with cross (Linux)
36+
if: runner.os == 'Linux' && (inputs.use-cross == 'true' || inputs.use-cross == 'auto')
37+
shell: bash
38+
run: cross build --release --target ${{ inputs.target }}
39+
40+
- name: Build with zigbuild
41+
if: inputs.use-zigbuild == 'true'
42+
shell: bash
43+
run: cargo zigbuild --release --target ${{ inputs.target }}
44+
45+
- name: Build with cargo (native)
46+
if: (runner.os == 'macOS' || runner.os == 'Windows') && inputs.use-zigbuild == 'false'
47+
shell: bash
48+
run: cargo build --release --target ${{ inputs.target }}
49+
50+
- name: Set library path and verify
51+
id: set-lib-path
52+
shell: bash
53+
run: |
54+
set -euo pipefail
55+
56+
# Determine library name based on target
57+
case "${{ inputs.target }}" in
58+
*-apple-darwin)
59+
LIB_NAME="libtokenizers.dylib"
60+
;;
61+
*-unknown-linux-gnu)
62+
LIB_NAME="libtokenizers.so"
63+
;;
64+
*-unknown-linux-musl)
65+
LIB_NAME="libtokenizers.a"
66+
;;
67+
*-pc-windows-msvc)
68+
LIB_NAME="tokenizers.dll"
69+
;;
70+
*)
71+
echo "Unknown target: ${{ inputs.target }}"
72+
exit 1
73+
;;
74+
esac
75+
76+
LIB_PATH="target/${{ inputs.target }}/release/${LIB_NAME}"
77+
78+
echo "Expecting library at: $LIB_PATH"
79+
if [ -f "$LIB_PATH" ]; then
80+
echo "✓ Library found"
81+
ls -lh "$LIB_PATH"
82+
file "$LIB_PATH" || true
83+
echo "path=$LIB_PATH" >> $GITHUB_OUTPUT
84+
else
85+
echo "✗ Library not found at: $LIB_PATH"
86+
exit 1
87+
fi

‎.github/actions/get-rust-library/action.yml‎

Lines changed: 74 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,74 @@
1+
name: Get Rust Library
2+
description: Download latest Rust library release or build locally as fallback
3+
4+
inputs:
5+
github-token:
6+
description: 'GitHub token for downloading releases'
7+
required: true
8+
9+
runs:
10+
using: composite
11+
steps:
12+
- name: Download or build Rust library (Linux)
13+
if: runner.os == 'Linux'
14+
shell: bash
15+
env:
16+
GITHUB_TOKEN: ${{ inputs.github-token }}
17+
run: |
18+
LATEST_RELEASE=$(gh release list --repo ${{ github.repository }} --limit 10 | grep -E "^rust-v" | head -1 | cut -f1)
19+
if [ -z "$LATEST_RELEASE" ]; then
20+
echo "No Rust release found, building locally"
21+
cargo build --release
22+
echo "TOKENIZERS_LIB_PATH=$(pwd)/target/release/libtokenizers.so" >> $GITHUB_ENV
23+
else
24+
echo "Using Rust release: $LATEST_RELEASE"
25+
gh release download $LATEST_RELEASE --pattern "libtokenizers-x86_64-unknown-linux-gnu.tar.gz"
26+
mkdir -p test-lib
27+
tar -xzf libtokenizers-x86_64-unknown-linux-gnu.tar.gz -C test-lib
28+
echo "TOKENIZERS_LIB_PATH=$(pwd)/test-lib/libtokenizers.so" >> $GITHUB_ENV
29+
fi
30+
31+
- name: Download or build Rust library (macOS)
32+
if: runner.os == 'macOS'
33+
shell: bash
34+
env:
35+
GITHUB_TOKEN: ${{ inputs.github-token }}
36+
run: |
37+
ARCH=$(uname -m)
38+
if [ "$ARCH" = "arm64" ]; then
39+
ARCH="aarch64"
40+
elif [ "$ARCH" = "x86_64" ]; then
41+
ARCH="x86_64"
42+
fi
43+
LATEST_RELEASE=$(gh release list --repo ${{ github.repository }} --limit 10 | grep -E "^rust-v" | head -1 | cut -f1)
44+
if [ -z "$LATEST_RELEASE" ]; then
45+
echo "No Rust release found, building locally"
46+
cargo build --release
47+
echo "TOKENIZERS_LIB_PATH=$(pwd)/target/release/libtokenizers.dylib" >> $GITHUB_ENV
48+
else
49+
echo "Using Rust release: $LATEST_RELEASE"
50+
gh release download $LATEST_RELEASE --pattern "libtokenizers-${ARCH}-apple-darwin.tar.gz"
51+
mkdir -p test-lib
52+
tar -xzf libtokenizers-${ARCH}-apple-darwin.tar.gz -C test-lib
53+
echo "TOKENIZERS_LIB_PATH=$(pwd)/test-lib/libtokenizers.dylib" >> $GITHUB_ENV
54+
fi
55+
56+
- name: Download or build Rust library (Windows)
57+
if: runner.os == 'Windows'
58+
shell: pwsh
59+
env:
60+
GITHUB_TOKEN: ${{ inputs.github-token }}
61+
run: |
62+
$latestRelease = gh release list --repo ${{ github.repository }} --limit 10 | Select-String -Pattern "^rust-v" | Select-Object -First 1
63+
if ($latestRelease) {
64+
$releaseTag = ($latestRelease -split "`t")[0]
65+
Write-Host "Using Rust release: $releaseTag"
66+
gh release download $releaseTag --pattern "libtokenizers-x86_64-pc-windows-msvc.tar.gz"
67+
New-Item -ItemType Directory -Force -Path test-lib
68+
tar -xzf libtokenizers-x86_64-pc-windows-msvc.tar.gz -C test-lib
69+
"TOKENIZERS_LIB_PATH=$(Get-Location)\test-lib\tokenizers.dll" | Out-File -FilePath $env:GITHUB_ENV -Append
70+
} else {
71+
Write-Host "No Rust release found, building locally"
72+
cargo build --release
73+
"TOKENIZERS_LIB_PATH=$(Get-Location)\target\release\tokenizers.dll" | Out-File -FilePath $env:GITHUB_ENV -Append
74+
}

‎.github/actions/setup-cross-compilation/action.yml‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
name: Setup Cross-Compilation
2+
description: Install cross-compilation tools and dependencies
3+
4+
runs:
5+
using: composite
6+
steps:
7+
- name: Install cross-compilation dependencies (Linux)
8+
if: runner.os == 'Linux'
9+
shell: bash
10+
run: |
11+
sudo apt-get update
12+
sudo apt-get install -y \
13+
gcc-aarch64-linux-gnu \
14+
musl-tools \
15+
g++-aarch64-linux-gnu \
16+
libc6-dev-arm64-cross \
17+
pkg-config
18+
19+
- name: Install cross
20+
shell: bash
21+
run: cargo install cross --git https://github.com/cross-rs/cross

‎.github/actions/setup-rust/action.yml‎

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,37 @@
1+
name: Setup Rust Toolchain
2+
description: Install Rust toolchain with caching and optional components
3+
4+
inputs:
5+
components:
6+
description: 'Comma-separated list of Rust components (e.g., rustfmt, clippy)'
7+
required: false
8+
default: ''
9+
targets:
10+
description: 'Comma-separated list of Rust targets for cross-compilation'
11+
required: false
12+
default: ''
13+
cache-key-suffix:
14+
description: 'Additional suffix for cache key (e.g., target name)'
15+
required: false
16+
default: ''
17+
18+
runs:
19+
using: composite
20+
steps:
21+
- name: Install Rust toolchain
22+
uses: dtolnay/rust-toolchain@stable
23+
with:
24+
components: ${{ inputs.components }}
25+
targets: ${{ inputs.targets }}
26+
27+
- name: Cache cargo registry
28+
uses: actions/cache@v4
29+
with:
30+
path: |
31+
~/.cargo/registry
32+
~/.cargo/git
33+
target
34+
key: ${{ runner.os }}-cargo-${{ inputs.cache-key-suffix }}-${{ hashFiles('**/Cargo.lock') }}
35+
restore-keys: |
36+
${{ runner.os }}-cargo-${{ inputs.cache-key-suffix }}-
37+
${{ runner.os }}-cargo-

‎.github/workflows/benchmark.yml‎

Lines changed: 7 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@ on:
99
- 'Cargo.toml'
1010
- 'Cargo.lock'
1111
- '.github/workflows/benchmark.yml'
12+
- '.github/actions/**'
1213
pull_request:
1314
branches: [ main ]
1415
paths:
@@ -17,6 +18,7 @@ on:
1718
- 'Cargo.toml'
1819
- 'Cargo.lock'
1920
- '.github/workflows/benchmark.yml'
21+
- '.github/actions/**'
2022
workflow_dispatch:
2123

2224
permissions:
@@ -38,21 +40,8 @@ jobs:
3840
with:
3941
go-version: '1.24'
4042

41-
- name: Install Rust toolchain
42-
uses: dtolnay/rust-toolchain@stable
43-
44-
- name: Cache Rust dependencies
45-
uses: actions/cache@v4
46-
with:
47-
path: |
48-
~/.cargo/bin/
49-
~/.cargo/registry/index/
50-
~/.cargo/registry/cache/
51-
~/.cargo/git/db/
52-
target/
53-
key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }}
54-
restore-keys: |
55-
${{ runner.os }}-cargo-
43+
- name: Setup Rust
44+
uses: ./.github/actions/setup-rust
5645

5746
- name: Build Rust library
5847
run: cargo build --release
@@ -99,25 +88,12 @@ jobs:
9988
with:
10089
go-version: '1.24'
10190

102-
- name: Install Rust toolchain
103-
uses: dtolnay/rust-toolchain@stable
91+
- name: Setup Rust
92+
uses: ./.github/actions/setup-rust
10493

10594
- name: Install benchstat
10695
run: go install golang.org/x/perf/cmd/benchstat@latest
10796

108-
- name: Cache Rust dependencies
109-
uses: actions/cache@v4
110-
with:
111-
path: |
112-
~/.cargo/bin/
113-
~/.cargo/registry/index/
114-
~/.cargo/registry/cache/
115-
~/.cargo/git/db/
116-
target/
117-
key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }}
118-
restore-keys: |
119-
${{ runner.os }}-cargo-
120-
12197
- name: Build Rust library
12298
run: cargo build --release
12399

@@ -191,4 +167,4 @@ jobs:
191167
issue_number,
192168
body: comment
193169
});
194-
}
170+
}

0 commit comments

Comments
 (0)