ci(edriver): static musl build for x86_64 and aarch64

Build the edriver BPF plugin as a fully static musl binary for both
x86_64 and aarch64.  Both jobs run in ubuntu:24.04 containers using
musl-gcc; vendored-libelf compiles elfutils --without-zstd so the
resulting libelf.a carries no zstd dependency.

Key issues solved along the way:

1. elfutils ./configure (invoked by libbpf-sys build.rs with
   CC=musl-gcc) requires argp_parse, fts_close, and _obstack_free,
   which are absent from musl libc.  Fix: add a 'Build musl compat
   libs' CI step that compiles the three void-linux musl-compat shims
   (identical to Alpine's argp-standalone / musl-fts-dev /
   musl-obstack-dev packages) with musl-gcc and installs them into
   musl-gcc's default sysroot search path
   /usr/lib/${ARCH}-linux-musl/ before cargo build runs.

2. libbpf C compilation (make -C libbpf/src) needs gelf.h at BPF-
   skeleton-generation time; add libelf-dev and zlib1g-dev to apt
   install so clang can find the header.

3. .cargo/config.toml: set CC_*_musl=musl-gcc, linker=musl-gcc,
   target-feature=+crt-static for both musl targets.  Remove stale
   -lzstd link flags (vendored elfutils is --without-zstd).

CI matrix:
  x86_64  ubuntu-latest        → x86_64-unknown-linux-musl
  aarch64 ubuntu-24.04-arm     → aarch64-unknown-linux-musl
  (ubuntu:24.04 container for both; Alpine ARM64 has no Node.js so
   JS GitHub Actions don't work there)

Author: chriskaliX

.github/workflows/ci-edriver.yaml

@@ -17,18 +17,54 @@ jobs:
   edriver:
     name: "Build and test edriver / ${{ matrix.arch }}"
     runs-on: ${{ matrix.runner }}
+    container:
+      image: ${{ matrix.container }}
     strategy:
       fail-fast: false
       matrix:
         include:
           - arch: x86_64
             runner: ubuntu-latest
             target: x86_64-unknown-linux-musl
+            # Ubuntu + musl-gcc + vendored-libelf (--without-zstd).
+            # elfutils ./configure needs argp/fts/obstack, which are absent from
+            # musl libc.  We pre-build them (argp-standalone, musl-fts,
+            # musl-obstack) using musl-gcc and install to /usr/lib/x86_64-linux-musl
+            # before running `cargo build`, so configure finds them.
+            container: ubuntu:24.04
+            cc: musl-gcc
+            lib_path: ""
+            extra_cflags: "-idirafter /usr/include -idirafter /usr/include/x86_64-linux-gnu"
+            cargo_extra_features: ",vendored-libelf"
+            rustflags: "-C target-feature=+crt-static"
           - arch: aarch64
             runner: ubuntu-24.04-arm
             target: aarch64-unknown-linux-musl
+            # Same approach as x86_64.  ubuntu:24.04 on arm64 supports JS
+            # actions; Alpine ARM64 does not (no Node.js ARM64 in Alpine).
+            container: ubuntu:24.04
+            cc: musl-gcc
+            lib_path: ""
+            extra_cflags: "-idirafter /usr/include -idirafter /usr/include/aarch64-linux-gnu"
+            cargo_extra_features: ",vendored-libelf"
+            rustflags: "-C target-feature=+crt-static"
     steps:
-      - name: "Git checkout"
+      - name: Install build dependencies
+        env:
+          DEBIAN_FRONTEND: noninteractive
+        run: |
+          apt-get update -qq
+          apt-get install -y --no-install-recommends \
+            bash curl git ca-certificates \
+            build-essential linux-headers-generic \
+            clang llvm \
+            libelf-dev zlib1g-dev \
+            musl-tools musl-dev \
+            protobuf-compiler \
+            pkg-config \
+            autoconf automake libtool autopoint gettext flex bison gawk
+
+      - name: Git checkout
         uses: actions/checkout@v4
         with:
           submodules: true
@@ -37,33 +73,104 @@ jobs:
         uses: dtolnay/rust-toolchain@stable
         with:
           targets: ${{ matrix.target }}
+          components: rustfmt
 
-      - name: Cache cargo registry & build
+      - name: Cache cargo registry
         uses: actions/cache@v4
         with:
           path: |
             ~/.cargo/registry
             ~/.cargo/git
-            plugins/edriver/target
-          key: ${{ runner.os }}-${{ matrix.arch }}-cargo-edriver-${{ hashFiles('plugins/edriver/Cargo.lock') }}
-          restore-keys: ${{ runner.os }}-${{ matrix.arch }}-cargo-edriver-
+          key: ${{ matrix.arch }}-cargo-edriver-v10-${{ hashFiles('plugins/edriver/Cargo.lock') }}
+          restore-keys: ${{ matrix.arch }}-cargo-edriver-v10-
 
-      - name: Install build dependencies
+      # elfutils ./configure (invoked by libbpf-sys vendored-libelf build.rs)
+      # runs with CC=musl-gcc.  musl libc lacks argp_parse, fts_close, and
+      # _obstack_free, causing configure to abort.  We pre-build the three
+      # void-linux musl-compat shims (identical to Alpine's argp-standalone,
+      # musl-fts-dev, musl-obstack-dev packages) and install the .a + headers
+      # into musl-gcc's default sysroot search path so configure finds them.
+      - name: Build musl compat libs (argp / fts / obstack)
         run: |
-          sudo apt-get update -qq
-          sudo apt-get install -y --no-install-recommends build-essential pkgconf libelf-dev libzstd-dev musl-tools llvm-14 clang-14 protobuf-compiler
-          for tool in clang llc llvm-strip
-          do
-            sudo rm -f /usr/bin/$tool
-            sudo ln -s /usr/bin/${tool}-14 /usr/bin/$tool
-          done
+          set -eux
+          ARCH=$(uname -m)                              # x86_64 or aarch64
+          MUSL_LIB=/usr/lib/${ARCH}-linux-musl
+          MUSL_INC=/usr/include/${ARCH}-linux-musl
+
+          # 1. argp-standalone – provides argp_parse
+          # NOTE: Makefile.am uses noinst_LIBRARIES so `make install` does NOT
+          # install libargp.a.  We build and copy manually.
+          git clone --depth=1 https://github.com/ericonr/argp-standalone /tmp/argp-standalone
+          cd /tmp/argp-standalone
+          autoreconf -fiv
+          CC=musl-gcc ./configure --prefix=/usr
+          make -j$(nproc)
+          cp libargp.a ${MUSL_LIB}/
+          cp argp.h    ${MUSL_INC}/
+
+          # 2. musl-fts – provides fts_close (NetBSD implementation)
+          git clone --depth=1 https://github.com/void-linux/musl-fts /tmp/musl-fts
+          cd /tmp/musl-fts
+          ./bootstrap.sh
+          CC=musl-gcc ./configure --enable-static --disable-shared \
+            --prefix=/usr --libdir=${MUSL_LIB} --includedir=${MUSL_INC}
+          make -j$(nproc) && make install
+
+          # 3. musl-obstack – provides _obstack_free (from gcc libiberty)
+          git clone --depth=1 https://github.com/void-linux/musl-obstack /tmp/musl-obstack
+          cd /tmp/musl-obstack
+          ./bootstrap.sh
+          CC=musl-gcc ./configure --enable-static --disable-shared \
+            --prefix=/usr --libdir=${MUSL_LIB} --includedir=${MUSL_INC}
+          make -j$(nproc) && make install
+
+          # Verify all three libs landed in the musl sysroot
+          ls -la ${MUSL_LIB}/libargp.a ${MUSL_LIB}/libfts.a ${MUSL_LIB}/libobstack.a
+
+      - name: Verify toolchain
+        run: |
+          which protoc && protoc --version
+          which clang   && clang --version | head -1
 
       - name: Build
-        run: cd plugins/edriver && make build
+        run: |
+          export PROTOC=$(which protoc)
+          # Capture full output; write to job summary on failure for debugging.
+          BUILD_LOG=/tmp/edriver-build.log
+          set +e
+          cd plugins/edriver && make build 2>&1 | tee "$BUILD_LOG"
+          RC=${PIPESTATUS[0]}
+          if [ $RC -ne 0 ]; then
+            echo "## Build failure (exit $RC)" >> "$GITHUB_STEP_SUMMARY"
+            echo '```' >> "$GITHUB_STEP_SUMMARY"
+            tail -150 "$BUILD_LOG" >> "$GITHUB_STEP_SUMMARY"
+            echo '```' >> "$GITHUB_STEP_SUMMARY"
+            exit $RC
+          fi
         env:
           PLATFORM: ${{ matrix.arch }}
+          LIBBPF_SYS_LIBRARY_PATH: ${{ matrix.lib_path }}
+          # Use -idirafter (not -I) so musl sysroot headers keep priority;
+          # linux/bpf.h etc. are still found via /usr/include as last resort.
+          LIBBPF_SYS_EXTRA_CFLAGS: ${{ matrix.extra_cflags }}
+          CC_x86_64_unknown_linux_musl: ${{ matrix.cc }}
+          CC_aarch64_unknown_linux_musl: ${{ matrix.cc }}
+          CARGO_TARGET_X86_64_UNKNOWN_LINUX_MUSL_LINKER: ${{ matrix.cc }}
+          CARGO_TARGET_AARCH64_UNKNOWN_LINUX_MUSL_LINKER: ${{ matrix.cc }}
+          RUSTFLAGS: ${{ matrix.rustflags }}
+          CARGO_EXTRA_FEATURES: ${{ matrix.cargo_extra_features }}
 
       - name: Test
-        run: cd plugins/edriver && make test
+        run: |
+          export PROTOC=$(which protoc)
+          cd plugins/edriver && make test
         env:
           PLATFORM: ${{ matrix.arch }}
+          LIBBPF_SYS_LIBRARY_PATH: ${{ matrix.lib_path }}
+          LIBBPF_SYS_EXTRA_CFLAGS: ${{ matrix.extra_cflags }}
+          CC_x86_64_unknown_linux_musl: ${{ matrix.cc }}
+          CC_aarch64_unknown_linux_musl: ${{ matrix.cc }}
+          CARGO_TARGET_X86_64_UNKNOWN_LINUX_MUSL_LINKER: ${{ matrix.cc }}
+          CARGO_TARGET_AARCH64_UNKNOWN_LINUX_MUSL_LINKER: ${{ matrix.cc }}
+          RUSTFLAGS: ${{ matrix.rustflags }}
+          CARGO_EXTRA_FEATURES: ${{ matrix.cargo_extra_features }}

.github/workflows/co-re.yaml

@@ -18,6 +18,9 @@ jobs:
         with:
           targets: x86_64-unknown-linux-musl
 
+      - name: Add musl target
+        run: rustup target add x86_64-unknown-linux-musl
+
       - name: Cache cargo registry & build
         uses: actions/cache@v4
         with:
@@ -43,7 +46,8 @@ jobs:
           cd plugins/edriver
           make build
           cd ../..
-
+        env:
+          LIBBPF_SYS_LIBRARY_PATH: /usr/lib/x86_64-linux-gnu
       - name: Strip & checksum
         run: |
           strip plugins/edriver/target/x86_64-unknown-linux-musl/release/edriver

.github/workflows/release-driver.yaml

@@ -18,6 +18,8 @@ agent/deploy/hades-agent
 
 # server
 server/webconsole/frontend
+server/webconsole/frontend_backup
+server/webconsole/frontend_v2
 server/frontend
 
 # ignore certs

.gitignore

@@ -1,14 +1,22 @@
 [env]
-LIBBPF_SYS_LIBRARY_PATH_x86_64_unknown_linux_musl = "/usr/lib/x86_64-linux-gnu:/usr/lib64:/usr/lib"
-LIBBPF_SYS_LIBRARY_PATH_aarch64_unknown_linux_musl = "/usr/lib/aarch64-linux-gnu:/usr/lib64:/usr/lib"
+LIBBPF_SYS_LIBRARY_PATH_x86_64_unknown_linux_gnu  = "/usr/lib/x86_64-linux-gnu:/usr/lib64:/usr/lib"
+LIBBPF_SYS_LIBRARY_PATH_aarch64_unknown_linux_gnu = "/usr/lib/aarch64-linux-gnu:/usr/lib64:/usr/lib"
+LIBBPF_SYS_LIBRARY_PATH_x86_64_unknown_linux_musl  = "/usr/lib/x86_64-linux-musl:/usr/lib/x86_64-linux-gnu:/usr/lib64:/usr/lib"
+LIBBPF_SYS_LIBRARY_PATH_aarch64_unknown_linux_musl = "/usr/lib/aarch64-linux-musl:/usr/lib/aarch64-linux-gnu:/usr/lib64:/usr/lib"
+# libbpf-sys compiles libbpf C sources using the native gcc.
+# For musl targets, we must use musl-gcc so libbpf C code is compiled against
+# musl headers (not glibc headers). With glibc headers, _FORTIFY_SOURCE
+# injects __snprintf_chk/__sprintf_chk/__realpath_chk etc., which musl doesn't
+# provide. musl-gcc wraps gcc with the musl specs (-specs .../musl-gcc.specs)
+# and points include paths to musl headers.
+CC_x86_64_unknown_linux_musl  = "musl-gcc"
+CC_aarch64_unknown_linux_musl = "musl-gcc"
 
+# vendored-libelf builds elfutils --without-zstd; no zstd dependency anywhere.
 [target.x86_64-unknown-linux-musl]
-linker = "x86_64-linux-musl-gcc"
+linker = "musl-gcc"
 rustflags = ["-C", "target-feature=+crt-static"]
 
 [target.aarch64-unknown-linux-musl]
-linker = "aarch64-linux-musl-gcc"
+linker = "musl-gcc"
 rustflags = ["-C", "target-feature=+crt-static"]
-
-[target.aarch64-unknown-linux-gnu]
-linker = "aarch64-linux-gnu-gcc"

plugins/edriver/.cargo/config.toml

@@ -8,6 +8,12 @@ description = "Rust version of hades edriver"
 [features]
 debug = ["sdk/debug"]
 static = ["libbpf-rs/static"]
+# Enable vendored elfutils compilation via libbpf-sys.
+# Only needed on glibc hosts (Ubuntu) where the system libelf.a is compiled
+# with USE_ZSTD.  On Alpine/musl the system libelf-static is already
+# musl-native and zstd-free, so this is unnecessary (and won't build on musl
+# because elfutils configure requires glibc-only argp/fts).
+vendored-libelf = ["libbpf-sys/vendored-libelf"]
 
 [dependencies]
 anyhow = "1.0"
@@ -19,9 +25,31 @@ lazy_static = "1.5.0"
 twox-hash = "2.1"
 hex = "0.4"
 libbpf-rs = { version = "0.26.2", features = ["static"] }
+# Add vendored-zlib directly so libbpf-sys compiles zlib from its vendored
+# source (using CC_x86_64_unknown_linux_musl=musl-gcc set in .cargo/config.toml)
+# instead of linking the system libz.a.  The system libz.a on Ubuntu is
+# compiled with glibc headers (_FORTIFY_SOURCE=2), which generates
+# __snprintf_chk / __vsnprintf_chk calls that musl doesn't provide.
+# Compiling zlib from source with musl-gcc uses musl headers and avoids these.
+# vendored-zlib: compile zlib from source so the .a is musl-compatible on
+# both Alpine and Ubuntu.  vendored-libelf is NOT listed here; it is gated
+# behind the optional `vendored-libelf` feature above and activated only on
+# glibc CI (ubuntu:24.04 aarch64) via CARGO_EXTRA_FEATURES in the Makefile.
+libbpf-sys = { version = "1.7", features = ["vendored-zlib"] }
 bitflags = "2.11.1"
 moka = { version = "0.12", features = ["sync"] }
 sdk = { path = "../../SDK/rust" }
 
 [build-dependencies]
 libbpf-cargo = "0.26.2"
+# libbpf-cargo default → libbpf-rs/default → libbpf-sys/vendored-libbpf →
+# static-libbpf for the HOST build-script. libbpf's make install
+# BUILD_STATIC_ONLY=y creates a fat liblibbpf.a that embeds libelf object
+# files. Without vendored-libelf here, it embeds the SYSTEM libelf.a which
+# on Ubuntu 24.04+ is built with USE_ZSTD, causing ZSTD_* undefined symbol
+# errors when linking the HOST build-script binary.
+# vendored-libelf forces elfutils to be built with --without-zstd so the
+# embedded libelf objects are ZSTD-free.
+# NOTE: Do NOT add features=["static"] here (without vendored-libelf) since
+# that would make the HOST link system libelf.a directly with ZSTD refs.
+libbpf-sys = { version = "1.7", features = ["vendored-libelf", "vendored-zlib"] }

plugins/edriver/Cargo.toml

@@ -1,5 +1,9 @@
 # SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause)
 CMD_CLANG ?= clang
+# Extra cargo features appended to every --features= flag.
+# Set to ",vendored-libelf" on glibc hosts (ubuntu aarch64 CI) where the
+# system libelf.a is glibc-compiled and vendored compilation is required.
+CARGO_EXTRA_FEATURES ?=
 
 LIB_PATH := ../libs
 LIBBPF_CFLAGS = "-fPIC"
@@ -28,14 +32,14 @@ build: \
 	headers/libbpf/libbpf.a
 
 	cargo fmt
-	cargo build --release --bin edriver --target $(PLATFORM)-unknown-linux-$(LIBC) --features=static
+	cargo build --release --bin edriver --target $(PLATFORM)-unknown-linux-$(LIBC) --features=static$(CARGO_EXTRA_FEATURES)
 
 debug: \
 	headers/libbpf/libbpf.a
 
 	cargo fmt
-	cargo build --release --bin edriver --target $(PLATFORM)-unknown-linux-$(LIBC) --features=static,debug
+	cargo build --release --bin edriver --target $(PLATFORM)-unknown-linux-$(LIBC) --features=static,debug$(CARGO_EXTRA_FEATURES)
 
 test:
 	cargo fmt
-	cargo test --target $(PLATFORM)-unknown-linux-$(LIBC) --features=static,debug -- --include-ignored --show-output
+	cargo test --target $(PLATFORM)-unknown-linux-$(LIBC) --features=static,debug$(CARGO_EXTRA_FEATURES) -- --include-ignored --show-output

plugins/edriver/Makefile

@@ -0,0 +1,6 @@
+[toolchain]
+channel = "stable"
+targets = [
+  "x86_64-unknown-linux-musl",
+  "aarch64-unknown-linux-musl",
+]