build-emerge.yaml

name: build emerge compiler
run-name: Build & Test the Emerge compiler
on:
  push:
    branches:
      - '**'
      - '!release/**'
  workflow_call:
    inputs:
      kotlin-ref:
        description: "The ref of the Kotlin code to build. NOTE: other build dependencies, e.g. glibc, will use the version of the workflow, not this ref."
        type: string
        required: true
        default: "${{ github.sha }}"
    outputs:
      tarball-filename:
        description: "Name of the tarball inside the 'tarball' artifact."
        value: "${{ jobs.linux-distributables.outputs.tarball-filename }}"
      windows-zip-filename:
        description: "Name of the ZIP archive inside the 'windows-zip' artifact."
        value: "emerge-toolchain.zip" # is used throughout this workflow, search&replace to change
      windows-zip-sha256:
        description: "SHA256 checksum of the ZIP archive inside the 'windows-zip' artifact."
        value: "${{ jobs.windows-distributables.outputs.zip-sha256 }}"

env:
  GLIBC_SOURCE_BALL_URL: "https://mirror.dogado.de/gnu/libc/glibc-2.41.tar.bz2"
  GCC_GIT_REF: "releases/gcc-14.1.0"
  LIBUNWIND_GIT_REF: "v1.8.1"
  LLVM_MAJOR_VERSION: "20"
  ZULU_JRE_URL: https://cdn.azul.com/zulu/bin/zulu21.36.17-ca-jre21.0.4-win_x64.zip

jobs:
  build-glibc-x86_64:
    name: "build glibc for x86_64-pc-linux-gnu"
    runs-on: ubuntu-latest
    steps:
      - id: cache-glibc
        uses: actions/cache@v4.2.2
        with:
          path: |
            ${{ github.workspace }}/glibc-build/csu
            ${{ github.workspace }}/glibc-build/libc.so
          key: glibc-x86_64-pc-linux-gnu-${{ env.GLIBC_SOURCE_BALL_URL }}
      - name: build glibc
        if: steps.cache-glibc.outputs.cache-hit != 'true'
        run: |
          sudo apt-get install bison
          curl "$GLIBC_SOURCE_BALL_URL" | tar --bzip2 --extract
          mv glibc-2.41 glibc-source
          mkdir glibc-build
          cd glibc-build
          $GITHUB_WORKSPACE/glibc-source/configure --host=x86_64-pc-linux-gnu --target=x86_64-pc-linux-gnu --prefix=$GITHUB_WORKSPACE/glibc-install
          N_PHYSICAL_CORES=$(cat /proc/cpuinfo | grep -E 'cpu cores\s*:' | head -1 | sed -E 's/^.*([0-9]+)$/\1/')
          make -j $N_PHYSICAL_CORES
  build-gcc-x86_64:
    name: "build gcc for x86_64-pc-linux-gnu"
    runs-on: ubuntu-latest
    steps:
      - id: cache-gcc
        uses: actions/cache@v4.2.2
        with:
          path: |
            ${{ github.workspace }}/gcc-build/x86_64-pc-linux-gnu/libgcc
          key: gcc-x86_64-pc-linux-gnu-${{ env.GCC_GIT_REF }}
      - name: checkout GCC source
        if: steps.cache-gcc.outputs.cache-hit != 'true'
        uses: actions/checkout@v4
        with:
          repository: gcc-mirror/gcc
          ref: ${{ env.GCC_GIT_REF }}
          path: gcc-source
      - name: build GCC
        if: steps.cache-gcc.outputs.cache-hit != 'true'
        run: |
          sudo apt-get install flex
          cd gcc-source
          contrib/download_prerequisites
          cd $GITHUB_WORKSPACE
          mkdir gcc-build
          cd gcc-build
          $GITHUB_WORKSPACE/gcc-source/configure --host=x86_64-pc-linux-gnu --target=x86_64-pc-linux-gnu --disable-multilib --enable-languages=c
          N_PHYSICAL_CORES=$(cat /proc/cpuinfo | grep -E 'cpu cores\s*:' | head -1 | sed -E 's/^.*([0-9]+)$/\1/')
          make -j $N_PHYSICAL_CORES
  build-libunwind-x86_64:
    name: "build nongnu libunwind for x86_64-pc-linux-gnu"
    runs-on: ubuntu-latest
    steps:
      - id: cache-libunwind
        uses: actions/cache@v4.2.2
        with:
          path: |
            ${{ github.workspace }}/libunwind-install/lib
          key: libunwind-x86_64-pc-linux-gnu-${{ env.LIBUNWIND_GIT_REF }}
      - name: checkout libunwind source
        if: steps.cache-libunwind.outputs.cache-hit != 'true'
        uses: actions/checkout@v4
        with:
          repository: libunwind/libunwind
          ref: ${{ env.LIBUNWIND_GIT_REF }}
          path: ${{ github.workspace }}/libunwind-source
      - name: build libunwind
        if: steps.cache-libunwind.outputs.cache-hit != 'true'
        run: |
          cd ${{ github.workspace }}/libunwind-source
          autoreconf -i
          ./configure "--prefix=${{ github.workspace }}/libunwind-install" --disable-minidebuginfo
          make
          make install
  emerge-toolchain-maven:
    runs-on: ubuntu-22.04
    name: emerge compiler maven verify
    steps:
      - id: cache-emerge-toolchain-maven
        uses: actions/cache@v4.2.2
        with:
          path: |
            ${{ github.workspace }}/toolchain/target/toolchain.jar
          key: "toolchain-runnable-jar-${{ inputs.kotlin-ref || github.sha }}"
      - name: checkout
        if: steps.cache-emerge-toolchain-maven.outputs.cache-hit != 'true'
        uses: actions/checkout@v4
        with:
          ref: "${{ github.kotlin-ref || github.sha }}"
      - name: Set up JDK 21
        if: steps.cache-emerge-toolchain-maven.outputs.cache-hit != 'true'
        uses: actions/setup-java@v4.7.1
        with:
          java-version-file: .java-version
          distribution: zulu
          cache: maven
      - name: install llvm
        if: steps.cache-emerge-toolchain-maven.outputs.cache-hit != 'true'
        run: |
          wget https://apt.llvm.org/llvm.sh
          chmod +x llvm.sh
          sudo ./llvm.sh "${{ env.LLVM_MAJOR_VERSION }}"
      - name: mvn clean verify
        if: steps.cache-emerge-toolchain-maven.outputs.cache-hit != 'true'
        run: mvn "-Demerge.llvm-tests.llvm-install-dir=/usr/lib/llvm-20" -B clean verify
  linux-distributables:
    name: Linux distributable archives
    runs-on: ubuntu-22.04
    outputs:
      tarball-filename: "${{ steps.tarball.outputs.tarball_name }}"
    needs:
      - build-gcc-x86_64
      - build-glibc-x86_64
      - build-libunwind-x86_64
      - emerge-toolchain-maven
    steps:
      - name: checkout
        uses: actions/checkout@v4
        with:
          ref: "${{ inputs.ref || github.sha }}"
      - name: get C-runtime binaries from glibc
        uses: actions/cache/restore@v4.2.2
        with:
          path: |
            ${{ github.workspace }}/glibc-build/csu
            ${{ github.workspace }}/glibc-build/libc.so
          key: glibc-x86_64-pc-linux-gnu-${{ env.GLIBC_SOURCE_BALL_URL }}
          fail-on-cache-miss: true
      - name: get C-runtime binaries from gcc
        uses: actions/cache/restore@v4.2.2
        with:
          path: |
            ${{ github.workspace }}/gcc-build/x86_64-pc-linux-gnu/libgcc
          key: gcc-x86_64-pc-linux-gnu-${{ env.GCC_GIT_REF }}
          fail-on-cache-miss: true
      - name: get libunwind binaries
        uses: actions/cache/restore@v4.2.2
        with:
          path: |
            ${{ github.workspace }}/libunwind-install/lib
          key: libunwind-x86_64-pc-linux-gnu-${{ env.LIBUNWIND_GIT_REF }}
          fail-on-cache-miss: true
      - name: get runnable toolchain jar
        uses: actions/cache/restore@v4.2.2
        with:
          path: |
            ${{ github.workspace }}/toolchain/target/toolchain.jar
          key: "toolchain-runnable-jar-${{ inputs.kotlin-ref || github.sha }}"
          fail-on-cache-miss: true
          enableCrossOsArchive: true
      - name: make tarball
        id: tarball
        run: |
          set -ex
          TARBALL_NAME="emerge-toolchain.tar.gz"
          mkdir -p "${{ github.workspace }}/tarball-root" && pushd $_
          
            mkdir bin && pushd $_
              cp "${{ github.workspace }}/toolchain/target/toolchain.jar" .
            popd
            cp -r "${{ github.workspace }}/stdlib" .
            cp -ar "${{ github.workspace }}/dist/linux/all/." .
            
            mkdir -p backends && pushd $_
              mkdir -p noop && pushd $_
                cp -r "${{ github.workspace }}/backend-api/src/main/emerge/noop-backend-platform" platform-sources
              popd
            
              mkdir -p native && pushd $_
                cp -r "${{ github.workspace }}/llvm-backend/src/main/emerge-ffi-c" .
              popd
            
              mkdir -p linux-gnu && pushd $_
                cp -r "${{ github.workspace }}/llvm-backend/src/main/emerge-linux-libc" .
                cp -r "${{ github.workspace }}/llvm-backend/src/main/emerge-platform-linux" .
              popd
            
              mkdir -p x86_64-pc-linux-gnu/lib && pushd $_
                cp "${{ github.workspace }}/libunwind-install/lib/libunwind-x86_64.a" .
                cp "${{ github.workspace }}/gcc-build/x86_64-pc-linux-gnu/libgcc/crtbeginS.o" .
                cp "${{ github.workspace }}/gcc-build/x86_64-pc-linux-gnu/libgcc/crtendS.o" .
                cp "${{ github.workspace }}/glibc-build/csu/Scrt1.o" .
                cp "${{ github.workspace }}/glibc-build/libc.so" .
              popd  
            popd
          popd
          tar --create \
            --directory=tarball-root \
            --owner=root \
            --group=root \
            --mode='ug=rw,a=rX' \
            --preserve-permissions \
            --gzip \
            --format=gnu \
            . \
            > "$TARBALL_NAME"
          echo "tarball_name=$TARBALL_NAME" >> "$GITHUB_OUTPUT"
          echo "tarball_path=${{ github.workspace }}/$TARBALL_NAME" >> "$GITHUB_OUTPUT"
      - name: archive tarball
        uses: actions/upload-artifact@v4.4.3
        with:
          name: tarball
          path: "${{ steps.tarball.outputs.tarball_path }}"
  llvm-build-for-windows:
    uses: ./.github/workflows/build-llvm-windows.yaml
  windows-distributables:
    name: Windows distributable archive
    runs-on: windows-2022
    needs:
      - linux-distributables
      - llvm-build-for-windows
    outputs:
      zip-sha256: "${{ steps.pack-zip.outputs.sha256 }}"
    steps:
      - name: get tarball
        uses: actions/download-artifact@v4.1.8
        with:
          run-id: ${{ needs.linux-distributables.run-id }}
          name: tarball
      - name: download zulu
        shell: pwsh
        run: |
          mkdir zulu-jre
          Invoke-WebRequest "${{ env.ZULU_JRE_URL }}" -OutFile zulu-archive.zip
          Expand-Archive -Path zulu-archive.zip -DestinationPath zulu-jre -Force -PassThru
          
          # the archive contains one directory with the full name of the JRE and version; this is noise not worth predicting
          # instead, we can rename this directory to JRE, which conveniently is also the name the directory and its contents
          # should have on users disks once installed
          Rename-Item -Path (Get-ChildItem zulu-jre)[0] -NewName jre
      - name: get LLVM binaries
        uses: actions/download-artifact@v4.1.8
        with:
          run-id: "${{ needs.llvm-build-for-windows.run-id }}"
          name: llvm-windows-x86_64
          path: ${{ github.workspace }}/llvm-install
      - name: prepare zip
        shell: pwsh
        run: |
          New-Item -Type Directory -Name "archive-root"
          
          # first extract the linux tarball. It doesn't come with much linux-specific stuff, so we can reuse all the
          # code for building it
          tar --directory archive-root -xf "${{ needs.linux-distributables.outputs.tarball-filename }}"
          Remove-Item "archive-root/configure.sh" # this one is linux specific
          
          New-Item -Type Directory -Path "archive-root/llvm" -Name "bin"
          # the LLVM distribution for windows builds THE WHOLE thing. This is needed for development.
          # in actual use, the emerge compiler doesn't use nearly all of the LLVM tools. So there is a lot of
          # disk and network transfer volume to be saved by only re-distributing the bare minimum
          Move-Item -Path "llvm-install/bin/ld.lld.exe" "archive-root/llvm/bin/ld.lld.exe"
          Move-Item -Path "llvm-install/bin/llc.exe" "archive-root/llvm/bin/llc.exe"
          Move-Item -Path "llvm-install/bin/LLVM-C.dll" "archive-root/llvm/bin/LLVM-C.dll"
          # fix the reference to the LLVM installation directiory
          (Get-Content archive-root/toolchain-config.yml) -replace '(?<=\s*llvm-installation-directory: ).+','llvm' | Set-Content archive-root/toolchain-config.yml

          # redistribute JRE
          Move-Item -Path "zulu-jre/jre" "archive-root/jre"
      - name: pack zip
        id: pack-zip
        shell: pwsh
        run: |
          Get-ChildItem -Path "archive-root" | Compress-Archive -DestinationPath "emerge-toolchain.zip"
          $hash = (Get-FileHash -Algorithm SHA256 "emerge-toolchain.zip").Hash
          Add-Content -Path "$Env:GITHUB_OUTPUT" -Value "sha256=$hash"
      - name: archive zip
        uses: actions/upload-artifact@v4.4.3
        with:
          name: windows-zip
          path: "emerge-toolchain.zip"