Skip to content

Commit 3b0b549

Browse files
author
Changming Sun
committed
Move Linux CI pipelines to Github Actions
1 parent 24ece47 commit 3b0b549

File tree

7 files changed

+257
-353
lines changed

7 files changed

+257
-353
lines changed

‎.github/workflows/linux-dnnl.yml‎

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,32 @@
1+
# This workflow builds and tests the ONNX Runtime for Linux for DNNL EP
2+
# It leverages a reusable workflow (`reusable_linux_build.yml`) to handle the core build and test logic
3+
# within Docker containers, ensuring a consistent environment.
4+
# This file is very similar to linux_ci.yml, but much simpler
5+
6+
7+
name: Linux CI
8+
9+
on:
10+
push:
11+
branches: [ main, 'rel-*']
12+
pull_request:
13+
branches: [ main, 'rel-*']
14+
15+
concurrency:
16+
group: ${{ github.workflow }}-${{ github.ref }}
17+
cancel-in-progress: true
18+
19+
jobs:
20+
build-linux-x64-release-dnnl:
21+
name: Build Linux x64 Release (DNNL EP)
22+
uses: ./.github/workflows/reusable_linux_build.yml
23+
with:
24+
pool_name: "onnxruntime-github-Ubuntu2204-AMD-CPU"
25+
build_config: Release
26+
architecture: x64
27+
dockerfile_path: tools/ci_build/github/linux/docker/Dockerfile.manylinux2_28_cpu
28+
docker_image_repo: onnxruntimecpubuildpythonx64
29+
extra_build_flags: '--use_binskim_compliant_compile_flags --build_wheel --build_nuget --use_dnnl'
30+
python_path_prefix: 'PATH=/opt/python/cp310-cp310/bin:\$PATH'
31+
secrets:
32+
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}

‎.github/workflows/linux_ci.yml‎

Lines changed: 81 additions & 142 deletions
Original file line numberDiff line numberDiff line change
@@ -1,23 +1,20 @@
1-
# This workflow builds and tests the ONNX Runtime for Linux in both Debug and Release configurations.
2-
# It uses a Docker container to provide a consistent build environment.
1+
# This workflow builds and tests the ONNX Runtime for Linux on multiple architectures and configurations.
2+
# It leverages a reusable workflow (`reusable_linux_build.yml`) to handle the core build and test logic
3+
# within Docker containers, ensuring a consistent environment.
34
#
4-
# The workflow consists of two jobs:
5-
# - build-debug: Builds and tests the Debug configuration.
6-
# - Uses the 'Debug' build configuration.
7-
# - Enables AddressSanitizer for memory error detection.
8-
# - Builds and runs tests.
9-
# - build-release: Builds and tests the Release configuration.
10-
# - Uses the 'Release' build configuration.
11-
# - Includes additional flags for release builds:
12-
# - --use_binskim_compliant_compile_flags
13-
# - --build_wheel
14-
# - --build_csharp
15-
# - --enable_transformers_tool_test
16-
# - --cmake_extra_defines onnxruntime_BUILD_BENCHMARKS=ON
17-
# - Builds and runs tests.
5+
# The workflow consists of four parallel jobs targeting different combinations:
6+
# - build-linux-x64-debug: Builds/tests Debug config on Linux x64 (AMD CPU pool), enables AddressSanitizer.
7+
# - build-linux-x64-release: Builds/tests Release config on Linux x64 (AMD CPU pool), includes wheel/C#/benchmark flags.
8+
# - build-linux-arm64-debug: Builds/tests Debug config on Linux arm64 (ARM CPU pool), no AddressSanitizer because otherwise it will take more than 4 hours.
9+
# - build-linux-arm64-release: Builds/tests Release config on Linux arm64 (ARM CPU pool), includes wheel/C#/benchmark flags.
1810
#
19-
# The two jobs run in parallel to reduce the overall build time.
20-
# Both jobs use the same Docker image, built in a separate step.
11+
# Each job calls the reusable workflow, passing specific parameters:
12+
# - target architecture (x64 or arm64)
13+
# - build configuration (Debug or Release)
14+
# - runner pool name
15+
# - path to the appropriate Dockerfile
16+
# - Docker image name for caching/use
17+
# - configuration-specific build flags (e.g., --enable_address_sanitizer for Debug, --build_wheel for Release)
2118

2219
name: Linux CI
2320

@@ -32,131 +29,73 @@ concurrency:
3229
cancel-in-progress: true
3330

3431
jobs:
35-
linux-cpu-asan-debug: # Job for building and testing the Debug configuration
36-
runs-on: ["self-hosted", "1ES.Pool=onnxruntime-github-Ubuntu2204-AMD-CPU"]
37-
permissions:
38-
actions: read
39-
contents: read
32+
# --- x64 Builds ---
33+
build-linux-x64-debug:
34+
name: Build Linux x64 Debug (ASan)
35+
uses: ./.github/workflows/reusable_linux_build.yml
36+
with:
37+
pool_name: "onnxruntime-github-Ubuntu2204-AMD-CPU"
38+
build_config: Debug
39+
architecture: x64
40+
dockerfile_path: tools/ci_build/github/linux/docker/inference/x86_64/default/cpu/Dockerfile
41+
docker_image_repo: onnxruntimecpubuildcix64
42+
extra_build_flags: '--enable_address_sanitizer'
43+
# python_path_prefix: '' # Default empty string is fine, no prefix needed
44+
secrets:
45+
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
4046

41-
steps:
42-
- name: Checkout code
43-
uses: actions/checkout@v3
47+
build-linux-x64-release:
48+
name: Build Linux x64 Release
49+
uses: ./.github/workflows/reusable_linux_build.yml
50+
with:
51+
pool_name: "onnxruntime-github-Ubuntu2204-AMD-CPU"
52+
build_config: Release
53+
architecture: x64
54+
dockerfile_path: tools/ci_build/github/linux/docker/Dockerfile.manylinux2_28_cpu
55+
docker_image_repo: onnxruntimecpubuildpythonx64
56+
extra_build_flags: '--use_binskim_compliant_compile_flags --build_wheel --build_nuget --enable_transformers_tool_test --cmake_extra_defines onnxruntime_BUILD_BENCHMARKS=ON'
57+
python_path_prefix: 'PATH=/opt/python/cp310-cp310/bin:\$PATH'
58+
secrets:
59+
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
4460

45-
- name: Set up Python 3.x
46-
uses: actions/setup-python@v4
47-
with:
48-
python-version: '3.x'
61+
orttraining-linux-ci-pipeline:
62+
name: Build Linux x64 Release with training
63+
uses: ./.github/workflows/reusable_linux_build.yml
64+
with:
65+
pool_name: "onnxruntime-github-Ubuntu2204-AMD-CPU"
66+
build_config: Release
67+
architecture: x64
68+
dockerfile_path: tools/ci_build/github/linux/docker/Dockerfile.manylinux2_28_cpu
69+
docker_image_repo: onnxruntimecpubuildpythonx64
70+
extra_build_flags: '--enable_training --use_binskim_compliant_compile_flags --build_wheel --build_nuget --enable_transformers_tool_test --cmake_extra_defines onnxruntime_BUILD_BENCHMARKS=ON'
71+
python_path_prefix: 'PATH=/opt/python/cp310-cp310/bin:\$PATH'
72+
secrets:
73+
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
4974

50-
- name: Build Docker Image
51-
uses: microsoft/onnxruntime-github-actions/build-docker-image@v0.0.1
52-
with:
53-
Dockerfile: ${{ github.workspace }}/tools/ci_build/github/linux/docker/inference/x86_64/default/cpu/Dockerfile
54-
Repository: 'onnxruntimecpubuildcentos8x64'
55-
env:
56-
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} # Needed for context
75+
# --- arm64 Builds ---
76+
build-linux-arm64-debug:
77+
name: Build Linux arm64 Debug
78+
uses: ./.github/workflows/reusable_linux_build.yml
79+
with:
80+
pool_name: "onnxruntime-github-Ubuntu2204-ARM-CPU"
81+
build_config: Debug
82+
architecture: arm64
83+
dockerfile_path: tools/ci_build/github/linux/docker/inference/aarch64/default/cpu/Dockerfile
84+
docker_image_repo: onnxruntimecpubuildciaarch64
85+
extra_build_flags: '--use_binskim_compliant_compile_flags --build_wheel'
86+
secrets:
87+
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
5788

58-
- name: Create .onnx directory
59-
run: mkdir -p $HOME/.onnx
60-
61-
# Build and Test ONNX Runtime in Docker (Debug)
62-
- name: Build and Test ONNX Runtime in Docker (Debug)
63-
env:
64-
ALLOW_RELEASED_ONNX_OPSET_ONLY: 0
65-
NIGHTLY_BUILD: 1 # Assuming you want nightly build for both Debug and Release
66-
run: |
67-
docker run --rm \
68-
--volume /data/onnx:/data/onnx:ro \
69-
--volume /data/models:/data/models:ro \
70-
--volume ${{ github.workspace }}:/onnxruntime_src \
71-
--volume $HOME/.onnx:/home/onnxruntimedev/.onnx \
72-
-w /onnxruntime_src \
73-
-e ALLOW_RELEASED_ONNX_OPSET_ONLY \
74-
-e NIGHTLY_BUILD \
75-
onnxruntimecpubuildcentos8x64 \
76-
/bin/bash -c 'set -ex; \
77-
# Build with Debug configuration and AddressSanitizer enabled
78-
python3 tools/ci_build/build.py \
79-
--build_dir build/Debug --cmake_generator Ninja \
80-
--config Debug \
81-
--skip_submodule_sync \
82-
--build_shared_lib \
83-
--parallel \
84-
--use_vcpkg --use_vcpkg_ms_internal_asset_cache \
85-
--enable_onnx_tests \
86-
--enable_address_sanitizer \
87-
--update --build;
88-
# Run tests with Debug configuration
89-
python3 tools/ci_build/build.py \
90-
--build_dir build/Debug --cmake_generator Ninja \
91-
--config Debug \
92-
--skip_submodule_sync \
93-
--build_shared_lib \
94-
--parallel \
95-
--use_vcpkg --use_vcpkg_ms_internal_asset_cache \
96-
--enable_onnx_tests \
97-
--enable_address_sanitizer \
98-
--test;'
99-
100-
linux-cpu-release: # Job for building and testing the Release configuration
101-
runs-on: ["self-hosted", "1ES.Pool=onnxruntime-github-Ubuntu2204-AMD-CPU"]
102-
permissions:
103-
actions: read
104-
contents: read
105-
106-
steps:
107-
- name: Checkout code
108-
uses: actions/checkout@v3
109-
110-
- name: Set up Python 3.x
111-
uses: actions/setup-python@v4
112-
with:
113-
python-version: '3.x'
114-
115-
- name: Build Docker Image
116-
uses: microsoft/onnxruntime-github-actions/build-docker-image@v0.0.1
117-
with:
118-
Dockerfile: ${{ github.workspace }}/tools/ci_build/github/linux/docker/Dockerfile.manylinux2_28_cpu
119-
Repository: 'onnxruntimecpubuild'
120-
env:
121-
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} # Needed for context
122-
123-
- name: Create .onnx directory
124-
run: mkdir -p $HOME/.onnx
125-
# Build and Test ONNX Runtime in Docker (Release)
126-
- name: Build and Test ONNX Runtime in Docker (Release)
127-
env:
128-
ALLOW_RELEASED_ONNX_OPSET_ONLY: 0
129-
NIGHTLY_BUILD: 1
130-
run: |
131-
docker run --rm \
132-
--volume /data/onnx:/data/onnx:ro \
133-
--volume /data/models:/data/models:ro \
134-
--volume ${{ github.workspace }}:/onnxruntime_src \
135-
--volume $HOME/.onnx:/home/onnxruntimedev/.onnx \
136-
-w /onnxruntime_src \
137-
-e ALLOW_RELEASED_ONNX_OPSET_ONLY \
138-
-e NIGHTLY_BUILD \
139-
onnxruntimecpubuild \
140-
/bin/bash -c 'set -ex; \
141-
# Build with Release configuration and additional flags for release builds
142-
PATH=/opt/python/cp310-cp310/bin:$PATH python3 tools/ci_build/build.py \
143-
--build_dir build/Release --cmake_generator Ninja \
144-
--config Release \
145-
--skip_submodule_sync \
146-
--build_shared_lib \
147-
--parallel \
148-
--use_vcpkg --use_vcpkg_ms_internal_asset_cache \
149-
--enable_onnx_tests \
150-
--use_binskim_compliant_compile_flags --build_wheel --build_csharp --enable_transformers_tool_test --cmake_extra_defines onnxruntime_BUILD_BENCHMARKS=ON \
151-
--update --build;
152-
# Run tests with Release configuration
153-
PATH=/opt/python/cp310-cp310/bin:$PATH python3 tools/ci_build/build.py \
154-
--build_dir build/Release --cmake_generator Ninja \
155-
--config Release \
156-
--skip_submodule_sync \
157-
--build_shared_lib \
158-
--parallel \
159-
--use_vcpkg --use_vcpkg_ms_internal_asset_cache \
160-
--enable_onnx_tests \
161-
--use_binskim_compliant_compile_flags --build_wheel --build_csharp --enable_transformers_tool_test --cmake_extra_defines onnxruntime_BUILD_BENCHMARKS=ON \
162-
--test;'
89+
build-linux-arm64-release:
90+
name: Build Linux arm64 Release
91+
uses: ./.github/workflows/reusable_linux_build.yml
92+
with:
93+
pool_name: "onnxruntime-github-Ubuntu2204-ARM-CPU"
94+
build_config: Release
95+
architecture: arm64
96+
dockerfile_path: tools/ci_build/github/linux/docker/inference/aarch64/python/cpu/Dockerfile
97+
docker_image_repo: onnxruntimecpubuildpythonaarch64
98+
extra_build_flags: '--use_binskim_compliant_compile_flags --build_wheel --cmake_extra_defines onnxruntime_BUILD_BENCHMARKS=ON'
99+
python_path_prefix: 'PATH=/opt/python/cp310-cp310/bin:\$PATH'
100+
secrets:
101+
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}

‎.github/workflows/linux_training.yml‎

Lines changed: 0 additions & 55 deletions
This file was deleted.

0 commit comments

Comments
 (0)