Merge pull request #677 from coldav/colin/build_installed_llvm

Change llvm over to a choice of install or cache, default PRs to install

Author: coldav

.github/actions/setup_build/action.yml

@@ -23,7 +23,10 @@ inputs:
   cross_arch:
     description: 'Cross compilation architecture from: x86, arm, aarch64, riscv64. Default: "none" (no cross compile), will auto fetch native arch'
     default: "none"
-  
+  llvm_source:
+    description: 'source to get llvm from - one of install or cache'
+    default: "cache"
+
 runs:
   # We don't want a new docker just a list of steps, so mark as composite
   using: "composite"
@@ -73,17 +76,31 @@ runs:
         cat $GITHUB_OUTPUT
 
     - name: load llvm native
-      if: inputs.cross_arch != 'none'
+      if: inputs.llvm_source == 'cache'
       uses: actions/cache/restore@v4
       with:
         path: llvm_install/**
         key: llvm-${{ inputs.os }}-${{ steps.set_llvm_key.outputs.key_version }}-${{ steps.set_llvm_key.outputs.key_native_arch }}-v${{ inputs.llvm_version }}-${{ inputs.llvm_build_type }}
         fail-on-cache-miss: true
+
+    - name: install llvm native
+      if: inputs.llvm_source == 'install'
+      shell: bash
+      run: |
+        set -x
+        sudo apt update
+
+        sudo apt install -y llvm-${{ inputs.llvm_version }}-dev liblld-${{ inputs.llvm_version }}-dev libclang-${{ inputs.llvm_version }}-dev
+        sudo apt install -y libpolly-${{ inputs.llvm_version }}-dev clang-${{ inputs.llvm_version }}
+
+        ln -s /usr/lib/llvm-${{ inputs.llvm_version }} llvm_install
+
     - shell: bash
       if: inputs.cross_arch != 'none'    
       run: mv llvm_install llvm_install_native
 
-    - name: load llvm
+    - name: load llvm cross
+      if: inputs.cross_arch != 'none' && inputs.llvm_source == 'cache'
       uses: actions/cache/restore@v4
       with:
         path: llvm_install/**

.github/workflows/pr_tests_cache.yml

@@ -48,6 +48,7 @@ jobs:
           llvm_version: ${{ env.llvm_previous }}
           llvm_build_type: RelAssert
           save: true
+          llvm_source: install
 
       - name: build host_x86_64
         uses:  ./.github/actions/do_build_pr/run_host_x86_64
@@ -82,6 +83,7 @@ jobs:
           llvm_version: ${{ env.llvm_current }}
           llvm_build_type: RelAssert
           save: true
+          llvm_source: install
 
       - name: build ubuntu_gcc_x86_64_riscv_fp16_cl3_0
         uses: ./.github/actions/do_build_pr/run_ubuntu_gcc_x86_64_riscv_fp16_cl3_0
@@ -119,6 +121,9 @@ jobs:
           llvm_build_type: RelAssert
           save: true
           cross_arch: x86
+          # Whilst we still have caches use this for 32 bit x86
+          # We may have to drop this or run just this part as part of the nightly.
+          llvm_source: cache
 
       - name: build ubuntu_clang_x86_llvm_latest_cl3_0_offline
         uses: ./.github/actions/do_build_pr/run_ubuntu_clang_x86_llvm_latest_cl3_0_offline
@@ -145,6 +150,7 @@ jobs:
           llvm_version: ${{ env.llvm_current }}
           llvm_build_type: RelAssert
           save: true
+          llvm_source: install
 
       - name: build ubuntu_gcc_aarch64_llvm_latest_cl3_0_fp16
         uses: ./.github/actions/do_build_pr/run_ubuntu_gcc_aarch64_llvm_latest_cl3_0_fp16
@@ -172,6 +178,7 @@ jobs:
           llvm_build_type: RelAssert
           save: true
           os: windows
+          llvm_source: cache
       - name: build windows_msvc_x86_64_llvm_latest_cl3_0_offline
         uses: ./.github/actions/do_build_pr/run_windows_msvc_x86_64_llvm_latest_cl3_0_offline
         with:

.github/workflows/run_pr_tests.yml

@@ -7,14 +7,23 @@ on:
         required: false
         type: boolean
         default: true
-
+      llvm_source:
+        required: false
+        description: 'method of sourcing llvm (install or cache), currently advisory until all converted'
+        type: string
+        default: 'install'
   # Allows you to run this workflow manually from the Actions tab
   workflow_dispatch:
     inputs:
       is_pull_request:
         required: false
         type: bool
         default: true
+      llvm_source:
+        required: false
+        type: string
+        description: 'method of sourcing llvm (install or cache), currently advisory until all converted'
+        default: 'install'
 
 permissions:
   packages: read
@@ -38,6 +47,7 @@ jobs:
         with:
           llvm_version: 18
           llvm_build_type: RelAssert
+          llvm_source: ${{ inputs.llvm_source}}
 
       - name: build
         uses:  ./.github/actions/do_build_pr/run_host_x86_64
@@ -72,6 +82,7 @@ jobs:
         with:
           llvm_version: 18
           llvm_build_type: RelAssert
+          llvm_source: ${{ inputs.llvm_source}}
 
       - name: build riscv M1
         uses: ./.github/actions/do_build_pr/run_riscv_m1
@@ -102,6 +113,7 @@ jobs:
         with:
           llvm_version: 19
           llvm_build_type: RelAssert
+          llvm_source: ${{ inputs.llvm_source}}
 
       - name: build initial config files
         uses: ./.github/actions/do_build_ock
@@ -173,6 +185,7 @@ jobs:
           llvm_version: 19
           llvm_build_type: RelAssert
           os: windows
+          llvm_source: cache
 
       - name: build and test ock
         uses: ./.github/actions/do_build_pr/run_windows_msvc_x86_64_llvm_latest_cl3_0_offline
@@ -195,11 +208,13 @@ jobs:
       with:
         llvm_version: '18'
         llvm_build_type: RelAssert
+        llvm_source: ${{ inputs.llvm_source}}
     - name: build ock
       uses: ./.github/actions/do_build_pr/run_ubuntu_gcc_x86_64_riscv_fp16_cl3_0_unitcl_vecz
 
   # Based on: mr-ubuntu-clang-x86-llvm-previous-cl3-0-offline
   run-ubuntu-clang-x86-llvm-latest-cl3-0-offline:
+    if: ${{ !inputs.is_pull_request }}  # do not run as PR job due to cross where we always install
     runs-on: ubuntu-22.04
     container:
       image: ghcr.io/uxlfoundation/ock_ubuntu_22.04-x86-64:latest
@@ -215,6 +230,7 @@ jobs:
         llvm_version: '19'
         llvm_build_type: RelAssert
         cross_arch: x86
+        llvm_source: ${{ inputs.llvm_source}}
     - name: build and run ock
       uses: ./.github/actions/do_build_pr/run_ubuntu_clang_x86_llvm_latest_cl3_0_offline
 
@@ -234,6 +250,7 @@ jobs:
       with:
         llvm_version: '19'
         llvm_build_type: RelAssert
+        llvm_source: ${{ inputs.llvm_source}}
     - name: build and run ock
       uses: ./.github/actions/do_build_pr/run_ubuntu_gcc_x86_64_riscv_fp16_cl3_0
 
@@ -253,6 +270,7 @@ jobs:
       with:
         llvm_version: '19'
         llvm_build_type: Release
+        llvm_source: ${{ inputs.llvm_source}}
     - name: build and run ock
       uses: ./.github/actions/do_build_pr/run_ubuntu_gcc_x86_llvm_latest_x86_64_images_cl3_0_release
 
@@ -272,7 +290,8 @@ jobs:
       with:
         llvm_version: '19'
         llvm_build_type: RelAssert
-    - name: build and run ock      
+        llvm_source:  ${{ inputs.llvm_source}}
+    - name: build and run ock
       uses: ./.github/actions/do_build_pr/run_ubuntu_gcc_aarch64_llvm_latest_cl3_0_fp16
 
   # Based on a combination of: mr-ubuntu-gcc-x86_64-clik
@@ -316,6 +335,7 @@ jobs:
       with:
         llvm_version: '19'
         llvm_build_type: RelAssert
+        llvm_source: ${{ inputs.llvm_source}}
     - name: build ock
       uses: ./.github/actions/do_build_pr/run_ubuntu_gcc_x86_64_refsi_g1_wi_cl3_0
 

.github/workflows/run_pr_tests_caller.yml

@@ -29,5 +29,6 @@ jobs:
     uses: ./.github/workflows/run_pr_tests.yml
     with:
       is_pull_request:  ${{ github.event_name != 'schedule' }}
+      llvm_source: ${{ github.event_name != 'schedule' && 'install' || 'cache'}}
 
 # additional ones here for cron and/or push to main - also can be in different file.