Build & Test the Emerge compiler #660
Workflow file for this run
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: build emerge compiler | |
| run-name: Build & Test the Emerge compiler | |
| on: | |
| push: | |
| branches: | |
| - '**' | |
| - '!release/**' | |
| workflow_call: | |
| inputs: | |
| kotlin-ref: | |
| description: "The ref of the Kotlin code to build. NOTE: other build dependencies, e.g. glibc, will use the version of the workflow, not this ref." | |
| type: string | |
| required: true | |
| default: "${{ github.sha }}" | |
| outputs: | |
| tarball-filename: | |
| description: "Name of the tarball inside the 'tarball' artifact." | |
| value: "${{ jobs.linux-distributables.outputs.tarball-filename }}" | |
| windows-zip-filename: | |
| description: "Name of the ZIP archive inside the 'windows-zip' artifact." | |
| value: "emerge-toolchain.zip" # is used throughout this workflow, search&replace to change | |
| windows-zip-sha256: | |
| description: "SHA256 checksum of the ZIP archive inside the 'windows-zip' artifact." | |
| value: "${{ jobs.windows-distributables.outputs.zip-sha256 }}" | |
| env: | |
| GLIBC_SOURCE_BALL_URL: "https://mirror.dogado.de/gnu/libc/glibc-2.41.tar.bz2" | |
| GCC_GIT_REF: "releases/gcc-14.1.0" | |
| LIBUNWIND_GIT_REF: "v1.8.1" | |
| LLVM_MAJOR_VERSION: "20" | |
| ZULU_JRE_URL: https://cdn.azul.com/zulu/bin/zulu21.36.17-ca-jre21.0.4-win_x64.zip | |
| jobs: | |
| build-glibc-x86_64: | |
| name: "build glibc for x86_64-pc-linux-gnu" | |
| runs-on: ubuntu-latest | |
| steps: | |
| - id: cache-glibc | |
| uses: actions/cache@v4.2.2 | |
| with: | |
| path: | | |
| ${{ github.workspace }}/glibc-build/csu | |
| ${{ github.workspace }}/glibc-build/libc.so | |
| key: glibc-x86_64-pc-linux-gnu-${{ env.GLIBC_SOURCE_BALL_URL }} | |
| - name: build glibc | |
| if: steps.cache-glibc.outputs.cache-hit != 'true' | |
| run: | | |
| sudo apt-get install bison | |
| curl "$GLIBC_SOURCE_BALL_URL" | tar --bzip2 --extract | |
| mv glibc-2.41 glibc-source | |
| mkdir glibc-build | |
| cd glibc-build | |
| $GITHUB_WORKSPACE/glibc-source/configure --host=x86_64-pc-linux-gnu --target=x86_64-pc-linux-gnu --prefix=$GITHUB_WORKSPACE/glibc-install | |
| N_PHYSICAL_CORES=$(cat /proc/cpuinfo | grep -E 'cpu cores\s*:' | head -1 | sed -E 's/^.*([0-9]+)$/\1/') | |
| make -j $N_PHYSICAL_CORES | |
| build-gcc-x86_64: | |
| name: "build gcc for x86_64-pc-linux-gnu" | |
| runs-on: ubuntu-latest | |
| steps: | |
| - id: cache-gcc | |
| uses: actions/cache@v4.2.2 | |
| with: | |
| path: | | |
| ${{ github.workspace }}/gcc-build/x86_64-pc-linux-gnu/libgcc | |
| key: gcc-x86_64-pc-linux-gnu-${{ env.GCC_GIT_REF }} | |
| - name: checkout GCC source | |
| if: steps.cache-gcc.outputs.cache-hit != 'true' | |
| uses: actions/checkout@v4 | |
| with: | |
| repository: gcc-mirror/gcc | |
| ref: ${{ env.GCC_GIT_REF }} | |
| path: gcc-source | |
| - name: build GCC | |
| if: steps.cache-gcc.outputs.cache-hit != 'true' | |
| run: | | |
| sudo apt-get install flex | |
| cd gcc-source | |
| contrib/download_prerequisites | |
| cd $GITHUB_WORKSPACE | |
| mkdir gcc-build | |
| cd gcc-build | |
| $GITHUB_WORKSPACE/gcc-source/configure --host=x86_64-pc-linux-gnu --target=x86_64-pc-linux-gnu --disable-multilib --enable-languages=c | |
| N_PHYSICAL_CORES=$(cat /proc/cpuinfo | grep -E 'cpu cores\s*:' | head -1 | sed -E 's/^.*([0-9]+)$/\1/') | |
| make -j $N_PHYSICAL_CORES | |
| build-libunwind-x86_64: | |
| name: "build nongnu libunwind for x86_64-pc-linux-gnu" | |
| runs-on: ubuntu-latest | |
| steps: | |
| - id: cache-libunwind | |
| uses: actions/cache@v4.2.2 | |
| with: | |
| path: | | |
| ${{ github.workspace }}/libunwind-install/lib | |
| key: libunwind-x86_64-pc-linux-gnu-${{ env.LIBUNWIND_GIT_REF }} | |
| - name: checkout libunwind source | |
| if: steps.cache-libunwind.outputs.cache-hit != 'true' | |
| uses: actions/checkout@v4 | |
| with: | |
| repository: libunwind/libunwind | |
| ref: ${{ env.LIBUNWIND_GIT_REF }} | |
| path: ${{ github.workspace }}/libunwind-source | |
| - name: build libunwind | |
| if: steps.cache-libunwind.outputs.cache-hit != 'true' | |
| run: | | |
| cd ${{ github.workspace }}/libunwind-source | |
| autoreconf -i | |
| ./configure "--prefix=${{ github.workspace }}/libunwind-install" --disable-minidebuginfo | |
| make | |
| make install | |
| emerge-toolchain-maven: | |
| runs-on: ubuntu-22.04 | |
| name: emerge compiler maven verify | |
| steps: | |
| - id: cache-emerge-toolchain-maven | |
| uses: actions/cache@v4.2.2 | |
| with: | |
| path: | | |
| ${{ github.workspace }}/toolchain/target/toolchain.jar | |
| key: "toolchain-runnable-jar-${{ inputs.kotlin-ref || github.sha }}" | |
| - name: checkout | |
| if: steps.cache-emerge-toolchain-maven.outputs.cache-hit != 'true' | |
| uses: actions/checkout@v4 | |
| with: | |
| ref: "${{ github.kotlin-ref || github.sha }}" | |
| - name: Set up JDK 21 | |
| if: steps.cache-emerge-toolchain-maven.outputs.cache-hit != 'true' | |
| uses: actions/setup-java@v4.7.1 | |
| with: | |
| java-version-file: .java-version | |
| distribution: zulu | |
| cache: maven | |
| - name: install llvm | |
| if: steps.cache-emerge-toolchain-maven.outputs.cache-hit != 'true' | |
| run: | | |
| wget https://apt.llvm.org/llvm.sh | |
| chmod +x llvm.sh | |
| sudo ./llvm.sh "${{ env.LLVM_MAJOR_VERSION }}" | |
| - name: mvn clean verify | |
| if: steps.cache-emerge-toolchain-maven.outputs.cache-hit != 'true' | |
| run: mvn "-Demerge.llvm-tests.llvm-install-dir=/usr/lib/llvm-20" -B clean verify | |
| linux-distributables: | |
| name: Linux distributable archives | |
| runs-on: ubuntu-22.04 | |
| outputs: | |
| tarball-filename: "${{ steps.tarball.outputs.tarball_name }}" | |
| needs: | |
| - build-gcc-x86_64 | |
| - build-glibc-x86_64 | |
| - build-libunwind-x86_64 | |
| - emerge-toolchain-maven | |
| steps: | |
| - name: checkout | |
| uses: actions/checkout@v4 | |
| with: | |
| ref: "${{ inputs.ref || github.sha }}" | |
| - name: get C-runtime binaries from glibc | |
| uses: actions/cache/restore@v4.2.2 | |
| with: | |
| path: | | |
| ${{ github.workspace }}/glibc-build/csu | |
| ${{ github.workspace }}/glibc-build/libc.so | |
| key: glibc-x86_64-pc-linux-gnu-${{ env.GLIBC_SOURCE_BALL_URL }} | |
| fail-on-cache-miss: true | |
| - name: get C-runtime binaries from gcc | |
| uses: actions/cache/restore@v4.2.2 | |
| with: | |
| path: | | |
| ${{ github.workspace }}/gcc-build/x86_64-pc-linux-gnu/libgcc | |
| key: gcc-x86_64-pc-linux-gnu-${{ env.GCC_GIT_REF }} | |
| fail-on-cache-miss: true | |
| - name: get libunwind binaries | |
| uses: actions/cache/restore@v4.2.2 | |
| with: | |
| path: | | |
| ${{ github.workspace }}/libunwind-install/lib | |
| key: libunwind-x86_64-pc-linux-gnu-${{ env.LIBUNWIND_GIT_REF }} | |
| fail-on-cache-miss: true | |
| - name: get runnable toolchain jar | |
| uses: actions/cache/restore@v4.2.2 | |
| with: | |
| path: | | |
| ${{ github.workspace }}/toolchain/target/toolchain.jar | |
| key: "toolchain-runnable-jar-${{ inputs.kotlin-ref || github.sha }}" | |
| fail-on-cache-miss: true | |
| enableCrossOsArchive: true | |
| - name: make tarball | |
| id: tarball | |
| run: | | |
| set -ex | |
| TARBALL_NAME="emerge-toolchain.tar.gz" | |
| mkdir -p "${{ github.workspace }}/tarball-root" && pushd $_ | |
| mkdir bin && pushd $_ | |
| cp "${{ github.workspace }}/toolchain/target/toolchain.jar" . | |
| popd | |
| cp -r "${{ github.workspace }}/stdlib" . | |
| cp -ar "${{ github.workspace }}/dist/linux/all/." . | |
| mkdir -p backends && pushd $_ | |
| mkdir -p noop && pushd $_ | |
| cp -r "${{ github.workspace }}/backend-api/src/main/emerge/noop-backend-platform" platform-sources | |
| popd | |
| mkdir -p native && pushd $_ | |
| cp -r "${{ github.workspace }}/llvm-backend/src/main/emerge-ffi-c" . | |
| popd | |
| mkdir -p linux-gnu && pushd $_ | |
| cp -r "${{ github.workspace }}/llvm-backend/src/main/emerge-linux-libc" . | |
| cp -r "${{ github.workspace }}/llvm-backend/src/main/emerge-platform-linux" . | |
| popd | |
| mkdir -p x86_64-pc-linux-gnu/lib && pushd $_ | |
| cp "${{ github.workspace }}/libunwind-install/lib/libunwind-x86_64.a" . | |
| cp "${{ github.workspace }}/gcc-build/x86_64-pc-linux-gnu/libgcc/crtbeginS.o" . | |
| cp "${{ github.workspace }}/gcc-build/x86_64-pc-linux-gnu/libgcc/crtendS.o" . | |
| cp "${{ github.workspace }}/glibc-build/csu/Scrt1.o" . | |
| cp "${{ github.workspace }}/glibc-build/libc.so" . | |
| popd | |
| popd | |
| popd | |
| tar --create \ | |
| --directory=tarball-root \ | |
| --owner=root \ | |
| --group=root \ | |
| --mode='ug=rw,a=rX' \ | |
| --preserve-permissions \ | |
| --gzip \ | |
| --format=gnu \ | |
| . \ | |
| > "$TARBALL_NAME" | |
| echo "tarball_name=$TARBALL_NAME" >> "$GITHUB_OUTPUT" | |
| echo "tarball_path=${{ github.workspace }}/$TARBALL_NAME" >> "$GITHUB_OUTPUT" | |
| - name: archive tarball | |
| uses: actions/upload-artifact@v4.4.3 | |
| with: | |
| name: tarball | |
| path: "${{ steps.tarball.outputs.tarball_path }}" | |
| llvm-build-for-windows: | |
| uses: ./.github/workflows/build-llvm-windows.yaml | |
| windows-distributables: | |
| name: Windows distributable archive | |
| runs-on: windows-2022 | |
| needs: | |
| - linux-distributables | |
| - llvm-build-for-windows | |
| outputs: | |
| zip-sha256: "${{ steps.pack-zip.outputs.sha256 }}" | |
| steps: | |
| - name: get tarball | |
| uses: actions/download-artifact@v4.1.8 | |
| with: | |
| run-id: ${{ needs.linux-distributables.run-id }} | |
| name: tarball | |
| - name: download zulu | |
| shell: pwsh | |
| run: | | |
| mkdir zulu-jre | |
| Invoke-WebRequest "${{ env.ZULU_JRE_URL }}" -OutFile zulu-archive.zip | |
| Expand-Archive -Path zulu-archive.zip -DestinationPath zulu-jre -Force -PassThru | |
| # the archive contains one directory with the full name of the JRE and version; this is noise not worth predicting | |
| # instead, we can rename this directory to JRE, which conveniently is also the name the directory and its contents | |
| # should have on users disks once installed | |
| Rename-Item -Path (Get-ChildItem zulu-jre)[0] -NewName jre | |
| - name: get LLVM binaries | |
| uses: actions/download-artifact@v4.1.8 | |
| with: | |
| run-id: "${{ needs.llvm-build-for-windows.run-id }}" | |
| name: llvm-windows-x86_64 | |
| path: ${{ github.workspace }}/llvm-install | |
| - name: prepare zip | |
| shell: pwsh | |
| run: | | |
| New-Item -Type Directory -Name "archive-root" | |
| # first extract the linux tarball. It doesn't come with much linux-specific stuff, so we can reuse all the | |
| # code for building it | |
| tar --directory archive-root -xf "${{ needs.linux-distributables.outputs.tarball-filename }}" | |
| Remove-Item "archive-root/configure.sh" # this one is linux specific | |
| New-Item -Type Directory -Path "archive-root/llvm" -Name "bin" | |
| # the LLVM distribution for windows builds THE WHOLE thing. This is needed for development. | |
| # in actual use, the emerge compiler doesn't use nearly all of the LLVM tools. So there is a lot of | |
| # disk and network transfer volume to be saved by only re-distributing the bare minimum | |
| Move-Item -Path "llvm-install/bin/ld.lld.exe" "archive-root/llvm/bin/ld.lld.exe" | |
| Move-Item -Path "llvm-install/bin/llc.exe" "archive-root/llvm/bin/llc.exe" | |
| Move-Item -Path "llvm-install/bin/LLVM-C.dll" "archive-root/llvm/bin/LLVM-C.dll" | |
| # fix the reference to the LLVM installation directiory | |
| (Get-Content archive-root/toolchain-config.yml) -replace '(?<=\s*llvm-installation-directory: ).+','llvm' | Set-Content archive-root/toolchain-config.yml | |
| # redistribute JRE | |
| Move-Item -Path "zulu-jre/jre" "archive-root/jre" | |
| - name: pack zip | |
| id: pack-zip | |
| shell: pwsh | |
| run: | | |
| Get-ChildItem -Path "archive-root" | Compress-Archive -DestinationPath "emerge-toolchain.zip" | |
| $hash = (Get-FileHash -Algorithm SHA256 "emerge-toolchain.zip").Hash | |
| Add-Content -Path "$Env:GITHUB_OUTPUT" -Value "sha256=$hash" | |
| - name: archive zip | |
| uses: actions/upload-artifact@v4.4.3 | |
| with: | |
| name: windows-zip | |
| path: "emerge-toolchain.zip" |