Skip to content

Commit de06640

Browse files
shiptuxshiptux
authored
[CICD] Add RPM packaging support for RHEL/Rocky/OpenEuler (#476)
1 parent 0b18e89 commit de06640

8 files changed

Lines changed: 461 additions & 0 deletions

File tree

‎.github/workflows/build-rpm.yml‎

Lines changed: 104 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,104 @@
1+
name: Build RPM Packages
2+
3+
on:
4+
push:
5+
tags:
6+
- 'v*'
7+
pull_request:
8+
branches: [ main ]
9+
paths:
10+
- 'flagcx/**'
11+
- 'packaging/rpm/**'
12+
- '.github/workflows/build-rpm.yml'
13+
workflow_dispatch:
14+
15+
jobs:
16+
# Build on GitHub-hosted runners. The RPMs are produced entirely inside
17+
# backend-specific Docker images pulled from public registries (nvcr.io,
18+
# Docker Hub, repos.metax-tech.com), so the build is host-agnostic and needs
19+
# neither the internal h20 runner nor a real GPU. This keeps PR checks green
20+
# without depending on the internal network's flaky public egress.
21+
#
22+
# ascend targets aarch64 (the CANN 910 base image is arm64), so it runs on an
23+
# arm64 runner; nvidia and metax target x86_64.
24+
build-rpm-packages:
25+
strategy:
26+
fail-fast: false
27+
matrix:
28+
include:
29+
- backend: nvidia
30+
runner: ubuntu-latest
31+
- backend: metax
32+
runner: ubuntu-latest
33+
- backend: ascend
34+
runner: ubuntu-24.04-arm
35+
runs-on: ${{ matrix.runner }}
36+
37+
steps:
38+
- name: Checkout repository
39+
uses: actions/checkout@v4
40+
with:
41+
submodules: recursive
42+
43+
- name: Build ${{ matrix.backend }} RPM packages
44+
run: ./packaging/rpm/build-flagcx-rpm.sh ${{ matrix.backend }}
45+
46+
- name: Upload ${{ matrix.backend }} RPM packages
47+
uses: actions/upload-artifact@v4
48+
with:
49+
name: flagcx-${{ matrix.backend }}-rpm-packages
50+
path: rpm-packages/${{ matrix.backend }}/**/*.rpm
51+
retention-days: 7
52+
53+
# Publish on the internal h20 runner, which can reach the internal Nexus.
54+
# Tag-only and never triggered by pull_request, so fork PRs are never handed
55+
# the Nexus credentials (GitHub withholds secrets from fork PRs anyway; the
56+
# explicit tag guard is a second layer).
57+
#
58+
# TODO(unverified): this is the first design where h20 pulls a same-run
59+
# artifact via download-artifact (results.actions blob, a different endpoint
60+
# than codeload). It cannot be validated until the first v* tag is pushed; if
61+
# h20's egress to that endpoint is also unreliable, move this step to an
62+
# internal host with stable GitHub access, or fix h20's public egress.
63+
publish-rpm-packages:
64+
needs: build-rpm-packages
65+
if: startsWith(github.ref, 'refs/tags/v')
66+
runs-on: h20
67+
68+
strategy:
69+
fail-fast: false
70+
matrix:
71+
backend: [nvidia, metax, ascend]
72+
73+
steps:
74+
- name: Download ${{ matrix.backend }} RPM packages
75+
uses: actions/download-artifact@v4
76+
with:
77+
name: flagcx-${{ matrix.backend }}-rpm-packages
78+
path: rpm-packages/${{ matrix.backend }}
79+
80+
- name: Publish ${{ matrix.backend }} RPMs to Nexus YUM repository
81+
env:
82+
NEXUS_USERNAME: ${{ secrets.REGISTRY_USERNAME }}
83+
NEXUS_PASSWORD: ${{ secrets.CONTAINER_REGISTRY }}
84+
NEXUS_REPO_URL: https://resource.flagos.net/repository/flagos-yum-hosted
85+
BACKEND: ${{ matrix.backend }}
86+
run: |
87+
set -euo pipefail
88+
89+
uploaded=0
90+
while IFS= read -r -d '' rpm; do
91+
# rel keeps the RPMS/<arch>/<file> or SRPMS/<file> layout under the backend
92+
rel="${rpm#rpm-packages/${BACKEND}/}"
93+
echo "Uploading ${rpm} -> ${BACKEND}/${rel}"
94+
curl -f -u "${NEXUS_USERNAME}:${NEXUS_PASSWORD}" \
95+
--upload-file "$rpm" \
96+
"${NEXUS_REPO_URL}/${BACKEND}/${rel}"
97+
uploaded=$((uploaded + 1))
98+
done < <(find "rpm-packages/${BACKEND}" -name '*.rpm' -print0)
99+
100+
if [ "$uploaded" -eq 0 ]; then
101+
echo "No RPMs found for ${BACKEND}"
102+
exit 1
103+
fi
104+
echo "Uploaded ${uploaded} ${BACKEND} RPM(s) to Nexus YUM repository"

‎.gitignore‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@ build
33
plugin/*/build
44
test/*/build
55
debian-packages
6+
rpm-packages
67

78
# Ignore compiled Python files and shared object files
89
plugin/*/*.so

‎packaging/rpm/build-flagcx-rpm.sh‎

Lines changed: 106 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,106 @@
1+
#!/bin/bash
2+
set -e
3+
4+
# FlagCX RPM package build script
5+
# Usage: ./build-flagcx-rpm.sh <backend> [base_image_version]
6+
# Supported backends: nvidia, metax, ascend
7+
8+
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
9+
PROJECT_DIR="$(dirname "$(dirname "$SCRIPT_DIR")")"
10+
BACKEND="${1:-}"
11+
BASE_IMAGE_VERSION="${2:-}"
12+
13+
# Colors
14+
RED='\033[0;31m'
15+
GREEN='\033[0;32m'
16+
YELLOW='\033[1;33m'
17+
BLUE='\033[0;34m'
18+
NC='\033[0m'
19+
20+
log_info() { echo -e "${GREEN}[INFO]${NC} $1"; }
21+
log_warn() { echo -e "${YELLOW}[WARN]${NC} $1"; }
22+
log_error() { echo -e "${RED}[ERROR]${NC} $1"; }
23+
log_step() { echo -e "${BLUE}[STEP]${NC} $1"; }
24+
25+
# Show usage
26+
if [ -z "$BACKEND" ]; then
27+
log_error "No backend specified"
28+
echo ""
29+
echo "Usage: $0 <backend> [base_image_version]"
30+
echo ""
31+
echo "Supported backends:"
32+
echo " nvidia - Build RPM packages for NVIDIA GPUs"
33+
echo " metax - Build RPM packages for MetaX accelerators"
34+
echo " ascend - Build RPM packages for Ascend NPUs"
35+
echo ""
36+
echo "Examples:"
37+
echo " $0 nvidia"
38+
echo " $0 ascend 8.5.0-910-openeuler24.03-py3.11"
39+
exit 1
40+
fi
41+
42+
# Validate backend and set base image
43+
case "$BACKEND" in
44+
nvidia)
45+
BASE_IMAGE="nvcr.io/nvidia/cuda"
46+
[ -z "$BASE_IMAGE_VERSION" ] && BASE_IMAGE_VERSION="12.4.1-devel-rockylinux8"
47+
DOCKERFILE="${SCRIPT_DIR}/dockerfiles/Dockerfile.rpm.nvidia"
48+
;;
49+
metax)
50+
BASE_IMAGE="rockylinux"
51+
[ -z "$BASE_IMAGE_VERSION" ] && BASE_IMAGE_VERSION="8"
52+
DOCKERFILE="${SCRIPT_DIR}/dockerfiles/Dockerfile.rpm.metax"
53+
;;
54+
ascend)
55+
BASE_IMAGE="ascendai/cann"
56+
[ -z "$BASE_IMAGE_VERSION" ] && BASE_IMAGE_VERSION="8.5.0-910-openeuler24.03-py3.11"
57+
DOCKERFILE="${SCRIPT_DIR}/dockerfiles/Dockerfile.rpm.ascend"
58+
;;
59+
*)
60+
log_error "Invalid backend: $BACKEND"
61+
echo "Supported backends: nvidia, metax, ascend"
62+
exit 1
63+
;;
64+
esac
65+
66+
log_info "Building FlagCX RPM packages for $BACKEND backend"
67+
log_info "Using base image: ${BASE_IMAGE}:${BASE_IMAGE_VERSION}"
68+
69+
# Sync changelog from CHANGELOG.md
70+
log_step "Synchronizing changelog..."
71+
if [ -f "${PROJECT_DIR}/packaging/sync-changelog.py" ]; then
72+
python3 "${PROJECT_DIR}/packaging/sync-changelog.py" || log_warn "Failed to sync changelog"
73+
else
74+
log_warn "sync-changelog.py not found, skipping changelog sync"
75+
fi
76+
77+
# Build Docker image using backend-specific Dockerfile with shared RPM logic.
78+
log_step "Building Docker image..."
79+
docker build \
80+
--network=host \
81+
--build-arg BASE_IMAGE_VERSION="${BASE_IMAGE_VERSION}" \
82+
-f "${DOCKERFILE}" \
83+
-t "flagcx-rpm-${BACKEND}:${BASE_IMAGE_VERSION}" \
84+
"${PROJECT_DIR}"
85+
86+
# Extract RPM packages
87+
log_step "Extracting RPM packages..."
88+
OUTPUT_DIR="${PROJECT_DIR}/rpm-packages/${BACKEND}"
89+
mkdir -p "${OUTPUT_DIR}"
90+
91+
CONTAINER_ID=$(docker create "flagcx-rpm-${BACKEND}:${BASE_IMAGE_VERSION}")
92+
docker cp "${CONTAINER_ID}:/root/rpmbuild/RPMS/" "${OUTPUT_DIR}/"
93+
docker cp "${CONTAINER_ID}:/root/rpmbuild/SRPMS/" "${OUTPUT_DIR}/"
94+
docker rm "${CONTAINER_ID}"
95+
96+
# Fail loudly if no RPMs were extracted, so CI doesn't silently upload empty artifacts.
97+
if ! find "${OUTPUT_DIR}" -name '*.rpm' | grep -q .; then
98+
log_error "No RPM packages found under ${OUTPUT_DIR}"
99+
exit 1
100+
fi
101+
102+
log_info "✓ Packages built successfully for ${BACKEND}:"
103+
echo ""
104+
find "${OUTPUT_DIR}" -name "*.rpm" -exec ls -lh {} \;
105+
106+
log_info "Build complete! Packages in: ${OUTPUT_DIR}"

‎packaging/rpm/dockerfiles/Dockerfile.rpm.ascend‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
# Build FlagCX Ascend RPM packages on CANN OpenEuler images.
2+
3+
ARG BASE_IMAGE_VERSION=8.5.0-910-openeuler24.03-py3.11
4+
FROM ascendai/cann:${BASE_IMAGE_VERSION}
5+
6+
WORKDIR /workspace
7+
COPY . /workspace/
8+
9+
RUN bash packaging/rpm/dockerfiles/build-rpm-common.sh ascend
10+
11+
CMD ["/bin/bash"]

‎packaging/rpm/dockerfiles/Dockerfile.rpm.metax‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
# Build FlagCX MetaX RPM packages on Rocky Linux.
2+
3+
ARG BASE_IMAGE_VERSION=8
4+
FROM rockylinux:${BASE_IMAGE_VERSION}
5+
6+
WORKDIR /workspace
7+
COPY . /workspace/
8+
9+
# MetaX packages are served from the public MACA yum repository.
10+
# TODO: switch gpgcheck=1 once MetaX publishes a stable GPG key for
11+
# repos.metax-tech.com. Today this repo serves unsigned packages.
12+
RUN printf '[maca-sdk]\nname=MACA SDK Yum Repository\nbaseurl=https://repos.metax-tech.com/r/maca-sdk-rpm-x86_64/\nenabled=1\ngpgcheck=0\n' \
13+
> /etc/yum.repos.d/maca-sdk-rpm.repo && \
14+
yum makecache && \
15+
yum install -y maca_sdk && \
16+
yum clean all
17+
18+
RUN bash packaging/rpm/dockerfiles/build-rpm-common.sh metax
19+
20+
CMD ["/bin/bash"]

‎packaging/rpm/dockerfiles/Dockerfile.rpm.nvidia‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
# Build FlagCX NVIDIA RPM packages on Rocky Linux based CUDA images.
2+
3+
ARG BASE_IMAGE_VERSION=12.4.1-devel-rockylinux8
4+
FROM nvcr.io/nvidia/cuda:${BASE_IMAGE_VERSION}
5+
6+
WORKDIR /workspace
7+
COPY . /workspace/
8+
9+
RUN bash packaging/rpm/dockerfiles/build-rpm-common.sh nvidia
10+
11+
CMD ["/bin/bash"]

‎packaging/rpm/dockerfiles/build-rpm-common.sh‎

Lines changed: 58 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,58 @@
1+
#!/bin/bash
2+
set -euo pipefail
3+
4+
BACKEND="${1:-}"
5+
6+
if [ -z "${BACKEND}" ]; then
7+
echo "ERROR: backend is required" >&2
8+
exit 1
9+
fi
10+
11+
case "${BACKEND}" in
12+
nvidia|metax|ascend)
13+
;;
14+
*)
15+
echo "ERROR: unsupported backend: ${BACKEND}" >&2
16+
exit 1
17+
;;
18+
esac
19+
20+
PKG_MANAGER="$(command -v dnf || command -v yum || true)"
21+
if [ -z "${PKG_MANAGER}" ]; then
22+
echo "ERROR: neither dnf nor yum is available in the base image" >&2
23+
exit 1
24+
fi
25+
26+
"${PKG_MANAGER}" install -y epel-release || \
27+
echo "EPEL not available for this base image, continuing without it"
28+
29+
"${PKG_MANAGER}" install -y \
30+
rpm-build \
31+
rpmdevtools \
32+
gcc-c++ \
33+
make \
34+
cmake \
35+
patchelf
36+
37+
"${PKG_MANAGER}" install -y json-devel 2>/dev/null \
38+
|| "${PKG_MANAGER}" install -y nlohmann-json-devel 2>/dev/null \
39+
|| { echo "ERROR: neither json-devel nor nlohmann-json-devel is available; rpmbuild requires nlohmann::json headers" >&2; exit 1; }
40+
41+
"${PKG_MANAGER}" clean all
42+
43+
rpmdev-setuptree
44+
45+
SPEC_VERSION="$(awk '/^Version:/ {print $2; exit}' /workspace/packaging/rpm/specs/flagcx.spec)"
46+
tar czf "/root/rpmbuild/SOURCES/flagcx-${SPEC_VERSION}.tar.gz" \
47+
--transform "s,^\.,flagcx-${SPEC_VERSION}," \
48+
--exclude='.git' \
49+
--exclude='build' \
50+
--exclude='debian-packages' \
51+
--exclude='rpm-packages' \
52+
.
53+
54+
rpmbuild -ba \
55+
--define "backend ${BACKEND}" \
56+
/workspace/packaging/rpm/specs/flagcx.spec
57+
58+
ls -lh /root/rpmbuild/RPMS/*/*.rpm

0 commit comments

Comments
 (0)