initial multi-stage build

usertam · usertam · commit a3dbfaee8949 · 2024-06-28T03:14:12.000+08:00
disable bolt for now
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
@@ -4,22 +4,123 @@ on:
     branches:
     - main
 jobs:
-  build:
+  build-llvm-bootstrap:
+    name: Build bootstrap LLVM
     runs-on: ubuntu-latest
     steps:
     - uses: actions/checkout@main
     - name: Install dependencies
-      run: bash ci.sh deps
-    - name: Build LLVM
-      run: bash ci.sh llvm
+      run: bash ci.sh do_deps
+    - name: Build
+      run: bash ci.sh do_bootstrap
+    - name: Pack artifact
+      if: always()
+      run: bash ci.sh do_pack
+    - uses: actions/upload-artifact@main
+      if: always()
+      with:
+        name: llvm-bootstrap
+        path: artifact.tar
+
+  build-llvm-instrumented:
+    name: Build instrumented LLVM
+    needs: build-llvm-bootstrap
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@main
+    - uses: actions/download-artifact@main
+      with:
+        name: llvm-bootstrap
+    - name: Install dependencies and unpack artifact
+      run: bash ci.sh "do_deps && do_unpack"
+    - name: Build
+      run: bash ci.sh do_instrumented
+    - name: Pack artifact
+      if: always()
+      run: bash ci.sh do_pack
+    - uses: actions/upload-artifact@main
+      if: always()
+      with:
+        name: llvm-instrumented
+        path: artifact.tar
+
+  build-llvm-profiling:
+    name: Profile instrumented LLVM
+    needs: build-llvm-instrumented
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@main
+    - uses: actions/download-artifact@main
+      with:
+        name: llvm-instrumented
+    - name: Install dependencies and unpack artifact
+      run: bash ci.sh "do_deps && do_unpack"
+    - name: Profile
+      run: bash ci.sh do_profiling
+    - name: Pack artifact
+      if: always()
+      run: bash ci.sh do_pack
+    - uses: actions/upload-artifact@main
+      if: always()
+      with:
+        name: llvm-profiling
+        path: artifact.tar
+
+  build-llvm-final:
+    name: Build final LLVM
+    needs: build-llvm-profiling
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@main
+    - uses: actions/download-artifact@main
+      with:
+        name: llvm-profiling
+    - name: Install dependencies and unpack artifact
+      run: bash ci.sh "do_deps && do_unpack"
+    - name: Build
+      run: bash ci.sh do_final
+    - name: Pack artifact
+      if: always()
+      run: bash ci.sh do_pack
+    - uses: actions/upload-artifact@main
+      if: always()
+      with:
+        name: llvm-final
+        path: artifact.tar
+
+  build-llvm-dist:
+    name: Make distribution with binutils
+    needs: build-llvm-final
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@main
+    - uses: actions/download-artifact@main
+      with:
+        name: llvm-final
+    - name: Install dependencies and unpack artifact
+      run: bash ci.sh "do_deps && do_unpack"
     - name: Build binutils
-      run: bash ci.sh binutils
-    - name: Build kernel
-      run: bash ci.sh kernel
-    - name: Fix up build
-      run: bash ci.sh fixup
-    - name: Make dist
-      run: bash ci.sh dist
+      run: bash ci.sh do_binutils
+    - name: Add finishing touches
+      run: bash ci.sh do_fixup
+    - name: Make .tar.xz distribution
+      run: bash ci.sh do_dist
+    - uses: actions/upload-artifact@main
+      with:
+        name: distribution
+        path: toolchain-dist.tar.xz
+    - name: Generate revision
+      run: echo "REVISION=$(bash ci.sh do_revision)" >> $GITHUB_ENV
+    - uses: softprops/action-gh-release@master
+      with:
+        tag_name: ${{env.REVISION}}
+        body_path: revision-notes.md
+        files: toolchain-dist.tar.xz
+    - name: Pack artifact
+      if: always()
+      run: bash ci.sh do_pack
     - uses: actions/upload-artifact@main
+      if: always()
       with:
-        path: tc-build-install.tar.xz
+        name: llvm-dist
+        path: artifact.tar
diff --git a/.gitignore b/.gitignore
@@ -1,4 +1,5 @@
 /.idea/
 /build/
+/install/
 /venv/
 *.pyc
diff --git a/build-llvm.py b/build-llvm.py
@@ -4,6 +4,7 @@
 from argparse import ArgumentParser, RawTextHelpFormatter
 from pathlib import Path
 import platform
+import sys
 import textwrap
 import time
 
@@ -408,6 +409,10 @@
                     '''),
                     type=str,
                     default='ClangBuiltLinux')
+# Custom github-actions build stages
+parser.add_argument('--actions-stage',
+                    type=str,
+                    choices=['bootstrap', 'instrumented', 'profiling', 'final'])
 args = parser.parse_args()
 
 # Start tracking time that the script takes
@@ -534,7 +539,8 @@
 
 # Build bootstrap compiler if user did not request a single stage build
 if (use_bootstrap := not args.build_stage1_only):
-    tc_build.utils.print_header('Building LLVM (bootstrap)')
+    if not args.actions_stage or args.actions_stage == 'bootstrap':
+        tc_build.utils.print_header('Building LLVM (bootstrap)')
 
     bootstrap = LLVMBootstrapBuilder()
     bootstrap.build_targets = ['distribution']
@@ -551,8 +557,13 @@
         bootstrap.projects.append('compiler-rt')
 
     bootstrap.check_dependencies()
-    bootstrap.configure()
-    bootstrap.build()
+    if not args.actions_stage or args.actions_stage == 'bootstrap':
+        bootstrap.configure()
+        bootstrap.build()
+
+        if args.actions_stage:
+            tc_build.utils.print_info('Building LLVM (bootstrap) done.')
+            sys.exit(0)
 
 # If the user did not specify CMAKE_C_FLAGS or CMAKE_CXX_FLAGS, add them as empty
 # to paste stage 2 to ensure there are no environment issues (since CFLAGS and CXXFLAGS
@@ -582,11 +593,17 @@
     instrumented.targets = final.targets
     instrumented.tools = StageTools(Path(bootstrap.folders.build, 'bin'))
 
-    tc_build.utils.print_header('Building LLVM (instrumented)')
-    instrumented.configure()
-    instrumented.build()
+    if not args.actions_stage or args.actions_stage == 'instrumented':
+        tc_build.utils.print_header('Building LLVM (instrumented)')
+        instrumented.configure()
+        instrumented.build()
+
+        if args.actions_stage:
+            tc_build.utils.print_info('Building LLVM (instrumented) done.')
+            sys.exit(0)
 
-    tc_build.utils.print_header('Generating PGO profiles')
+    if not args.actions_stage or args.actions_stage == 'profiling':
+        tc_build.utils.print_header('Generating PGO profiles')
     pgo_builders = []
     if 'llvm' in args.pgo:
         llvm_builder = def_llvm_builder_cls()
@@ -661,12 +678,19 @@
         pgo_builders.append(kernel_builder)
 
     for pgo_builder in pgo_builders:
+        if args.actions_stage and args.actions_stage != 'profiling':
+            continue
         if hasattr(pgo_builder, 'configure') and callable(pgo_builder.configure):
             tc_build.utils.print_info('Building LLVM for profiling...')
             pgo_builder.configure()
         pgo_builder.build()
 
-    instrumented.generate_profdata()
+    if not args.actions_stage or args.actions_stage == 'profiling':
+        instrumented.generate_profdata()
+
+        if args.actions_stage:
+            tc_build.utils.print_info('Generating PGO profiles done.')
+            sys.exit(0)
 
 # Final build
 final.build_targets = args.build_targets
@@ -717,4 +741,8 @@
 final.build()
 final.show_install_info()
 
+if args.actions_stage:
+    tc_build.utils.print_info('Building LLVM (final) done.')
+    sys.exit(0)
+
 print(f"Script duration: {tc_build.utils.get_duration(script_start)}")
diff --git a/ci.sh b/ci.sh
@@ -6,23 +6,6 @@ src=$base/src
 
 set -eu
 
-function parse_parameters() {
-    while (($#)); do
-        case $1 in
-            all | binutils | deps | kernel | llvm | fixup | dist) action=$1 ;;
-            *) exit 33 ;;
-        esac
-        shift
-    done
-}
-
-function do_all() {
-    do_deps
-    do_llvm
-    do_binutils
-    do_kernel
-}
-
 function do_binutils() {
     "$base"/build-binutils.py \
         --install-folder "$install" \
@@ -31,9 +14,6 @@ function do_binutils() {
 }
 
 function do_deps() {
-    # We only run this when running on GitHub Actions
-    [[ -z ${GITHUB_ACTIONS:-} ]] && return 0
-
     sudo apt-get install -y --no-install-recommends \
         bc \
         bison \
@@ -90,24 +70,21 @@ EOF
 }
 
 function do_llvm() {
-    extra_args=()
-    [[ -n ${GITHUB_ACTIONS:-} ]] && extra_args+=(--no-ccache)
-
     "$base"/build-llvm.py \
-        --assertions \
-        --bolt \
-        --build-targets distribution \
-        --install-folder "$install" \
-        --install-targets distribution \
+        --projects clang lld \
+        --targets AArch64 ARM X86 \
         --lto thin \
         --pgo kernel-defconfig \
-        --projects clang lld \
+        --build-targets distribution \
+        --install-targets distribution \
+        --vendor-string usertam \
+        --install-folder "$install" \
         --quiet-cmake \
         --shallow-clone \
         --show-build-commands \
-        --targets AArch64 ARM X86 \
-        --vendor-string usertam \
-        "${extra_args[@]}"
+        --no-ccache \
+        --no-update \
+        "$@"
 }
 
 function do_fixup() {
@@ -126,13 +103,52 @@ function do_fixup() {
     done
 }
 
+function do_bootstrap() {
+    do_llvm --actions-stage bootstrap
+}
+
+function do_instrumented() {
+    do_llvm --actions-stage instrumented
+}
+
+function do_profiling() {
+    do_llvm --actions-stage profiling
+}
+
+function do_final() {
+    do_llvm --actions-stage final
+}
+
+function do_pack() {
+    tar -cf artifact.tar \
+        $(ls -d src build install)
+}
+
+function do_unpack() {
+    tar -xf artifact.tar
+    rm -rf artifact.tar
+}
+
 function do_dist() {
     tar --sort=name \
         --mtime='1970-01-01' \
         --owner=0 --group=0 --numeric-owner \
-        -I pixz -cf tc-build-install.tar.xz \
-        -C "$install" $(ls -A "$install")
+        -I pixz -cf toolchain-dist.tar.xz \
+        -C "$install" $(ls "$install")
+}
+
+function do_revision() {
+    hash=$(git -C src/llvm-project rev-parse HEAD | cut -c -7)
+    date=$(date +%y%m%d)
+    cat <<EOF > revision-notes.md
+Built against the main branch of [llvm-project](https://github.com/llvm/llvm-project) on commit \`$hash\`.
+- Projects: clang, lld
+- Targets: aarch64, arm, x86
+- Bolt: None
+- LTO: ThinLTO (thin)
+- PGO: kernel-defconfig
+EOF
+    echo "r$date.$hash"
 }
 
-parse_parameters "$@"
-do_"${action:=all}"
+eval "$@"
diff --git a/toolchain-dist.tar b/toolchain-dist.tar

-Original file line number
+Diff line change
@@ @@ -1,4 +1,5 @@ @@
 /.idea/
 /build/
 +/install/
 /venv/
 *.pyc