Skip to content

Commit 0f5a66e

Browse files
chriskaliXchriskaliX
committed
ci(edriver): static musl build for x86_64 and aarch64
Both platforms use ubuntu:24.04 container with musl-gcc cross-compilation. vendored-libelf compiles elfutils --without-zstd on the glibc host, so obstack/fts/argp are all satisfied by glibc with zero extra packages. Approach validated against libbpf-sys own CI: their static+vendored builds all run on Ubuntu glibc hosts; Alpine is only used for dynamic-link tests. x86_64: ubuntu-latest runner / ubuntu:24.04 container / musl-gcc extra_cflags: -I/usr/include -I/usr/include/x86_64-linux-gnu aarch64: ubuntu-24.04-arm runner / ubuntu:24.04 container / musl-gcc extra_cflags: -I/usr/include -I/usr/include/aarch64-linux-gnu Both: RUSTFLAGS=-C target-feature=+crt-static, cargo feature vendored-libelf Other changes included: - edriver Cargo.toml: vendored-libelf as optional feature - edriver Makefile: CARGO_EXTRA_FEATURES variable - edriver .cargo/config.toml: musl targets, crt-static - remove co-re.yaml (superseded by this workflow)
1 parent 6fca81b commit 0f5a66e

18 files changed

Lines changed: 3962 additions & 3918 deletions

File tree

‎.github/workflows/ci-edriver.yaml‎

Lines changed: 67 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -17,18 +17,54 @@ jobs:
1717
edriver:
1818
name: "Build and test edriver / ${{ matrix.arch }}"
1919
runs-on: ${{ matrix.runner }}
20+
container:
21+
image: ${{ matrix.container }}
2022
strategy:
2123
fail-fast: false
2224
matrix:
2325
include:
2426
- arch: x86_64
2527
runner: ubuntu-latest
2628
target: x86_64-unknown-linux-musl
29+
# Use Ubuntu+musl-gcc (same strategy as aarch64). Building inside
30+
# Alpine is tempting but elfutils ./configure requires obstack/fts/
31+
# argp which are glibc built-ins; satisfying them on musl is fragile.
32+
# Ubuntu glibc host + vendored-libelf (--without-zstd) is clean.
33+
container: ubuntu:24.04
34+
cc: musl-gcc
35+
lib_path: ""
36+
extra_cflags: "-I/usr/include -I/usr/include/x86_64-linux-gnu"
37+
cargo_extra_features: ",vendored-libelf"
38+
rustflags: "-C target-feature=+crt-static"
2739
- arch: aarch64
2840
runner: ubuntu-24.04-arm
2941
target: aarch64-unknown-linux-musl
42+
# ubuntu:24.04 supports JavaScript actions on ARM64; Alpine does not.
43+
container: ubuntu:24.04
44+
cc: musl-gcc
45+
lib_path: ""
46+
extra_cflags: "-I/usr/include -I/usr/include/aarch64-linux-gnu"
47+
cargo_extra_features: ",vendored-libelf"
48+
rustflags: "-C target-feature=+crt-static"
3049
steps:
31-
- name: "Git checkout"
50+
# Both x86_64 and aarch64 run in ubuntu:24.04 containers with musl-gcc.
51+
# vendored-libelf compiles elfutils --without-zstd on the glibc host,
52+
# so obstack/fts/argp are all satisfied by glibc with no extra packages.
53+
- name: Install build dependencies
54+
env:
55+
DEBIAN_FRONTEND: noninteractive
56+
run: |
57+
apt-get update -qq
58+
apt-get install -y --no-install-recommends \
59+
bash curl git ca-certificates \
60+
build-essential linux-headers-generic \
61+
clang llvm \
62+
musl-tools musl-dev \
63+
protobuf-compiler \
64+
pkg-config \
65+
autoconf automake libtool autopoint flex bison gawk
66+
67+
- name: Git checkout
3268
uses: actions/checkout@v4
3369
with:
3470
submodules: true
@@ -37,33 +73,49 @@ jobs:
3773
uses: dtolnay/rust-toolchain@stable
3874
with:
3975
targets: ${{ matrix.target }}
76+
components: rustfmt
4077

41-
- name: Cache cargo registry & build
78+
- name: Cache cargo registry
4279
uses: actions/cache@v4
4380
with:
4481
path: |
4582
~/.cargo/registry
4683
~/.cargo/git
47-
plugins/edriver/target
48-
key: ${{ runner.os }}-${{ matrix.arch }}-cargo-edriver-${{ hashFiles('plugins/edriver/Cargo.lock') }}
49-
restore-keys: ${{ runner.os }}-${{ matrix.arch }}-cargo-edriver-
84+
key: ${{ matrix.arch }}-cargo-edriver-v7-${{ hashFiles('plugins/edriver/Cargo.lock') }}
85+
restore-keys: ${{ matrix.arch }}-cargo-edriver-v7-
5086

51-
- name: Install build dependencies
87+
- name: Verify toolchain
5288
run: |
53-
sudo apt-get update -qq
54-
sudo apt-get install -y --no-install-recommends build-essential pkgconf libelf-dev libzstd-dev musl-tools llvm-14 clang-14 protobuf-compiler
55-
for tool in clang llc llvm-strip
56-
do
57-
sudo rm -f /usr/bin/$tool
58-
sudo ln -s /usr/bin/${tool}-14 /usr/bin/$tool
59-
done
89+
which protoc && protoc --version
90+
which clang && clang --version | head -1
6091
6192
- name: Build
62-
run: cd plugins/edriver && make build
93+
run: |
94+
export PROTOC=$(which protoc)
95+
cd plugins/edriver && make build
6396
env:
6497
PLATFORM: ${{ matrix.arch }}
98+
LIBBPF_SYS_LIBRARY_PATH: ${{ matrix.lib_path }}
99+
# musl-gcc -nostdinc: re-add system includes for vendored libbpf.
100+
LIBBPF_SYS_EXTRA_CFLAGS: ${{ matrix.extra_cflags }}
101+
CC_x86_64_unknown_linux_musl: ${{ matrix.cc }}
102+
CC_aarch64_unknown_linux_musl: ${{ matrix.cc }}
103+
CARGO_TARGET_X86_64_UNKNOWN_LINUX_MUSL_LINKER: ${{ matrix.cc }}
104+
CARGO_TARGET_AARCH64_UNKNOWN_LINUX_MUSL_LINKER: ${{ matrix.cc }}
105+
RUSTFLAGS: ${{ matrix.rustflags }}
106+
CARGO_EXTRA_FEATURES: ${{ matrix.cargo_extra_features }}
65107

66108
- name: Test
67-
run: cd plugins/edriver && make test
109+
run: |
110+
export PROTOC=$(which protoc)
111+
cd plugins/edriver && make test
68112
env:
69113
PLATFORM: ${{ matrix.arch }}
114+
LIBBPF_SYS_LIBRARY_PATH: ${{ matrix.lib_path }}
115+
LIBBPF_SYS_EXTRA_CFLAGS: ${{ matrix.extra_cflags }}
116+
CC_x86_64_unknown_linux_musl: ${{ matrix.cc }}
117+
CC_aarch64_unknown_linux_musl: ${{ matrix.cc }}
118+
CARGO_TARGET_X86_64_UNKNOWN_LINUX_MUSL_LINKER: ${{ matrix.cc }}
119+
CARGO_TARGET_AARCH64_UNKNOWN_LINUX_MUSL_LINKER: ${{ matrix.cc }}
120+
RUSTFLAGS: ${{ matrix.rustflags }}
121+
CARGO_EXTRA_FEATURES: ${{ matrix.cargo_extra_features }}

‎.github/workflows/co-re.yaml‎

Lines changed: 0 additions & 59 deletions
This file was deleted.

‎.github/workflows/release-driver.yaml‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,9 @@ jobs:
1818
with:
1919
targets: x86_64-unknown-linux-musl
2020

21+
- name: Add musl target
22+
run: rustup target add x86_64-unknown-linux-musl
23+
2124
- name: Cache cargo registry & build
2225
uses: actions/cache@v4
2326
with:
@@ -43,7 +46,8 @@ jobs:
4346
cd plugins/edriver
4447
make build
4548
cd ../..
46-
49+
env:
50+
LIBBPF_SYS_LIBRARY_PATH: /usr/lib/x86_64-linux-gnu
4751
- name: Strip & checksum
4852
run: |
4953
strip plugins/edriver/target/x86_64-unknown-linux-musl/release/edriver

‎.gitignore‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,8 @@ agent/deploy/hades-agent
1818

1919
# server
2020
server/webconsole/frontend
21+
server/webconsole/frontend_backup
22+
server/webconsole/frontend_v2
2123
server/frontend
2224

2325
# ignore certs

‎plugins/edriver/.cargo/config.toml‎

Lines changed: 32 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -1,14 +1,37 @@
11
[env]
2-
LIBBPF_SYS_LIBRARY_PATH_x86_64_unknown_linux_musl = "/usr/lib/x86_64-linux-gnu:/usr/lib64:/usr/lib"
3-
LIBBPF_SYS_LIBRARY_PATH_aarch64_unknown_linux_musl = "/usr/lib/aarch64-linux-gnu:/usr/lib64:/usr/lib"
2+
LIBBPF_SYS_LIBRARY_PATH_x86_64_unknown_linux_gnu = "/usr/lib/x86_64-linux-gnu:/usr/lib64:/usr/lib"
3+
LIBBPF_SYS_LIBRARY_PATH_aarch64_unknown_linux_gnu = "/usr/lib/aarch64-linux-gnu:/usr/lib64:/usr/lib"
4+
LIBBPF_SYS_LIBRARY_PATH_x86_64_unknown_linux_musl = "/usr/lib/x86_64-linux-musl:/usr/lib/x86_64-linux-gnu:/usr/lib64:/usr/lib"
5+
LIBBPF_SYS_LIBRARY_PATH_aarch64_unknown_linux_musl = "/usr/lib/aarch64-linux-musl:/usr/lib/aarch64-linux-gnu:/usr/lib64:/usr/lib"
6+
# libbpf-sys compiles libbpf C sources using the native gcc.
7+
# For musl targets, we must use musl-gcc so libbpf C code is compiled against
8+
# musl headers (not glibc headers). With glibc headers, _FORTIFY_SOURCE
9+
# injects __snprintf_chk/__sprintf_chk/__realpath_chk etc., which musl doesn't
10+
# provide. musl-gcc wraps gcc with the musl specs (-specs .../musl-gcc.specs)
11+
# and points include paths to musl headers.
12+
CC_x86_64_unknown_linux_musl = "musl-gcc"
13+
CC_aarch64_unknown_linux_musl = "musl-gcc"
414

15+
# libbpf-sys/static links the system libelf.a. On Ubuntu 24.04+ that libelf.a
16+
# is compiled with USE_ZSTD, referencing ZSTD_* symbols that libbpf-sys never
17+
# emits `cargo:rustc-link-lib=zstd` for. We add -lzstd manually:
18+
#
19+
# [build].rustflags applies to build-script binaries (HOST triple), which is
20+
# where libbpf-cargo -> libbpf-sys runs. Dynamic libzstd.so is fine for the
21+
# host build-script executable.
22+
[build]
23+
rustflags = ["-C", "link-arg=-lzstd"]
24+
25+
# For the final musl binary, libbpf-sys also emits `cargo:rustc-link-lib=static=elf`
26+
# and the same ZSTD symbols need resolving. We force static libzstd here so
27+
# the binary remains self-contained: -Wl,-Bstatic,-lzstd,-Bdynamic forces static
28+
# for zstd only; libzstd.a is supplied by libzstd-dev in CI.
29+
# libbpf-sys already emits cargo:rustc-link-search for the LIBBPF_SYS_LIBRARY_PATH
30+
# dirs above, so the linker can find libzstd.a in e.g. /usr/lib/x86_64-linux-gnu/.
531
[target.x86_64-unknown-linux-musl]
6-
linker = "x86_64-linux-musl-gcc"
7-
rustflags = ["-C", "target-feature=+crt-static"]
32+
linker = "musl-gcc"
33+
rustflags = ["-C", "target-feature=+crt-static", "-C", "link-arg=-Wl,-Bstatic,-lzstd,-Bdynamic"]
834

935
[target.aarch64-unknown-linux-musl]
10-
linker = "aarch64-linux-musl-gcc"
11-
rustflags = ["-C", "target-feature=+crt-static"]
12-
13-
[target.aarch64-unknown-linux-gnu]
14-
linker = "aarch64-linux-gnu-gcc"
36+
linker = "musl-gcc"
37+
rustflags = ["-C", "target-feature=+crt-static", "-C", "link-arg=-Wl,-Bstatic,-lzstd,-Bdynamic"]

‎plugins/edriver/Cargo.toml‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,12 @@ description = "Rust version of hades edriver"
88
[features]
99
debug = ["sdk/debug"]
1010
static = ["libbpf-rs/static"]
11+
# Enable vendored elfutils compilation via libbpf-sys.
12+
# Only needed on glibc hosts (Ubuntu) where the system libelf.a is compiled
13+
# with USE_ZSTD. On Alpine/musl the system libelf-static is already
14+
# musl-native and zstd-free, so this is unnecessary (and won't build on musl
15+
# because elfutils configure requires glibc-only argp/fts).
16+
vendored-libelf = ["libbpf-sys/vendored-libelf"]
1117

1218
[dependencies]
1319
anyhow = "1.0"
@@ -19,9 +25,27 @@ lazy_static = "1.5.0"
1925
twox-hash = "2.1"
2026
hex = "0.4"
2127
libbpf-rs = { version = "0.26.2", features = ["static"] }
28+
# Add vendored-zlib directly so libbpf-sys compiles zlib from its vendored
29+
# source (using CC_x86_64_unknown_linux_musl=musl-gcc set in .cargo/config.toml)
30+
# instead of linking the system libz.a. The system libz.a on Ubuntu is
31+
# compiled with glibc headers (_FORTIFY_SOURCE=2), which generates
32+
# __snprintf_chk / __vsnprintf_chk calls that musl doesn't provide.
33+
# Compiling zlib from source with musl-gcc uses musl headers and avoids these.
34+
# vendored-zlib: compile zlib from source so the .a is musl-compatible on
35+
# both Alpine and Ubuntu. vendored-libelf is NOT listed here; it is gated
36+
# behind the optional `vendored-libelf` feature above and activated only on
37+
# glibc CI (ubuntu:24.04 aarch64) via CARGO_EXTRA_FEATURES in the Makefile.
38+
libbpf-sys = { version = "1.7", features = ["vendored-zlib"] }
2239
bitflags = "2.11.1"
2340
moka = { version = "0.12", features = ["sync"] }
2441
sdk = { path = "../../SDK/rust" }
2542

2643
[build-dependencies]
2744
libbpf-cargo = "0.26.2"
45+
# NOTE: Do NOT add libbpf-sys with features=["static"] here.
46+
# libbpf-cargo depends on libbpf-rs (default-features=false) → libbpf-sys
47+
# (no static). If we pin static here, the HOST build-script binary links
48+
# system libelf.a which on Ubuntu 24.04+ is compiled with USE_ZSTD and
49+
# references ZSTD_* symbols that libbpf-sys never emits as link-lib.
50+
# The static feature is only needed for the final binary, which is covered
51+
# by [dependencies] libbpf-rs/static → libbpf-sys/static.

‎plugins/edriver/Makefile‎

Lines changed: 7 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,9 @@
11
# SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause)
22
CMD_CLANG ?= clang
3+
# Extra cargo features appended to every --features= flag.
4+
# Set to ",vendored-libelf" on glibc hosts (ubuntu aarch64 CI) where the
5+
# system libelf.a is glibc-compiled and vendored compilation is required.
6+
CARGO_EXTRA_FEATURES ?=
37

48
LIB_PATH := ../libs
59
LIBBPF_CFLAGS = "-fPIC"
@@ -28,14 +32,14 @@ build: \
2832
headers/libbpf/libbpf.a
2933

3034
cargo fmt
31-
cargo build --release --bin edriver --target $(PLATFORM)-unknown-linux-$(LIBC) --features=static
35+
cargo build --release --bin edriver --target $(PLATFORM)-unknown-linux-$(LIBC) --features=static$(CARGO_EXTRA_FEATURES)
3236

3337
debug: \
3438
headers/libbpf/libbpf.a
3539

3640
cargo fmt
37-
cargo build --release --bin edriver --target $(PLATFORM)-unknown-linux-$(LIBC) --features=static,debug
41+
cargo build --release --bin edriver --target $(PLATFORM)-unknown-linux-$(LIBC) --features=static,debug$(CARGO_EXTRA_FEATURES)
3842

3943
test:
4044
cargo fmt
41-
cargo test --target $(PLATFORM)-unknown-linux-$(LIBC) --features=static,debug -- --include-ignored --show-output
45+
cargo test --target $(PLATFORM)-unknown-linux-$(LIBC) --features=static,debug$(CARGO_EXTRA_FEATURES) -- --include-ignored --show-output

‎plugins/edriver/rust-toolchain.toml‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
[toolchain]
2+
channel = "stable"
3+
targets = [
4+
"x86_64-unknown-linux-musl",
5+
"aarch64-unknown-linux-musl",
6+
]

0 commit comments

Comments
 (0)