Better build system robustness (#87)

* add dev_tools.jl. Start to enforce proper cuda toolkit usage. In progress

* add CUDA_SDK_jll as a direct dependency allows for correct paths

* add a workspaces and ./dev for CUDA_SDK_jll

* update cuda tool pathing

* dev override functionality

* update to support cunumeric

* mimic cunumeric's ci runs into branches like develop

* modify set_jll_artifact_override

* guard docs on wrapper changes

* update version semantics to major.minor

Author: krasow

.github/workflows/ci.yml

@@ -16,6 +16,7 @@ on:
       - 'src/**'
       - 'scripts/**'
       - 'deps/build.jl'
+      - 'deps/buildtools/**'
       - 'Project.toml'
       - 'lib/LegatePreferences/src/**'
       - '.github/workflows/ci.yml'
@@ -29,17 +30,30 @@ on:
       - 'src/**'
       - 'scripts/**'
       - 'deps/build.jl'
+      - 'deps/buildtools/**'
       - 'Project.toml'
       - 'lib/LegatePreferences/src/**'
       - '.github/workflows/ci.yml'
       - 'Dockerfile' # container.yml depends on this
 jobs:
-  pkg_resolve:
-    uses: ./.github/workflows/pkg_resolve.yml
+  check_changes:
+    name: Check for wrapper changes
+    runs-on: ubuntu-latest
+    outputs:
+      wrapper_changed: ${{ steps.filter.outputs.wrapper }}
+    steps:
+      - uses: actions/checkout@v4
+      - uses: dorny/paths-filter@v3
+        id: filter
+        with:
+          filters: |
+            wrapper:
+              - 'lib/legate_jl_wrapper/**'
 
   test:
     name: Julia ${{ matrix.julia }} - ${{ matrix.os }}
-    needs: pkg_resolve
+    needs: check_changes
+    if: ${{ github.base_ref == 'main' || needs.check_changes.outputs.wrapper_changed != 'true' }}
     runs-on: ${{ matrix.os }}
     strategy:
       fail-fast: false

.github/workflows/developer.yml

@@ -18,6 +18,7 @@ on:
       - 'src/**'
       - 'scripts/**'
       - 'deps/build.jl'
+      - 'deps/buildtools/**'
       - 'Project.toml'
       - 'lib/legate_jl_wrapper/src/**'
       - 'lib/legate_jl_wrapper/include/**'
@@ -32,6 +33,7 @@ on:
       - 'src/**'
       - 'scripts/**'
       - 'deps/build.jl'
+      - 'deps/buildtools/**'
       - 'Project.toml'
       - 'lib/legate_jl_wrapper/src/**'
       - 'lib/legate_jl_wrapper/include/**'
@@ -59,7 +61,7 @@ jobs:
         shell: bash
     env:
         LEGATE_AUTO_CONFIG: 0
-        NO_CUDA: ON
+        LEGATE_WRAPPER_ENABLE_CUDA: OFF
     steps:
       - name: Check out the repo
         uses: actions/checkout@v4

.github/workflows/docs.yml

@@ -23,8 +23,24 @@ on:
     - 'README.md'
     - 'lib/LegatePreferences/src*'
 jobs:
+  check_changes:
+    name: Check for wrapper changes
+    runs-on: ubuntu-latest
+    outputs:
+      wrapper_changed: ${{ steps.filter.outputs.wrapper }}
+    steps:
+      - uses: actions/checkout@v4
+      - uses: dorny/paths-filter@v3
+        id: filter
+        with:
+          filters: |
+            wrapper:
+              - 'lib/legate_jl_wrapper/**'
+
   docs:
     name : Documentation
+    needs: check_changes
+    if: ${{ github.base_ref == 'main' || needs.check_changes.outputs.wrapper_changed != 'true' }}
     permissions:
       actions: write
       contents: write

.gitignore

@@ -25,6 +25,8 @@ LocalPreferences.toml
 # Files generated by the build process
 libcxxwrap-julia*
 build*
+!deps/buildtools/
+!deps/buildtools/**
 wrapper/build
 
 legate_wrapper_install
@@ -55,5 +57,8 @@ docs/site/
 # environment.
 Manifest.toml
 
+# Generated by build.jl — do not commit
+dev/Project.toml
+dev/Manifest.toml
 
 test_crash.sh

Project.toml

@@ -2,6 +2,9 @@ name = "Legate"
 uuid = "1238f2cf-6593-4d60-9aca-2f5364e49909"
 version = "0.1.2"
 
+[workspace]
+projects = ["test", "dev"]
+
 [deps]
 CxxWrap = "1f15a43c-97ca-5a2a-ae31-89f07a497df4"
 FunctionWrappers = "069b7b12-0de2-55c6-9aab-29f3d0a68a2e"
@@ -26,7 +29,7 @@ CxxWrap = "0.17"
 FunctionWrappers = "1.1.3"
 JuliaFormatter = "2.2.0"
 LegatePreferences = "0.1.6"
+Preferences = "1"
 julia = "1.10"
 legate_jl_wrapper_jll = "25.10.4"
 legate_jll = "25.10.2"
-Preferences = "1"

deps/build.jl

@@ -16,138 +16,52 @@
  * Author(s): David Krasowska <krasow@u.northwestern.edu>
  *            Ethan Meitz <emeitz@andrew.cmu.edu>
 =#
+using Pkg
 using Preferences
 using LegatePreferences
 
+include("buildtools/dev_tools.jl")
 include("version.jl")
 
-# Automatically pipes errors to new file
-# and appends stdout to build.log
-function run_sh(cmd::Cmd, filename::String)
-    println(cmd)
-
-    build_log = joinpath(@__DIR__, "build.log")
-    tmp_build_log = joinpath(@__DIR__, "$(filename).log")
-    err_log = joinpath(@__DIR__, "$(filename).err")
-
-    if isfile(err_log)
-        rm(err_log)
-    end
-
-    if isfile(tmp_build_log)
-        rm(tmp_build_log)
-    end
-
-    try
-        run(pipeline(cmd; stdout=tmp_build_log, stderr=err_log, append=false))
-        contents = read(tmp_build_log, String)
-        open(build_log, "a") do io
-            println(contents)
-        end
-    catch e
-        println("stderr log generated: ", err_log, '\n')
-        contents = read(err_log, String)
-        if !isempty(strip(contents))
-            println("---- Begin stderr log ----")
-            println(contents)
-            println("---- End stderr log ----")
-        end
-    end
-end
-
 # patch legion. The readme below talks about our compilation error
 # https://github.com/ejmeitz/cuNumeric.jl/blob/main/scripts/README.md
 function patch_legion(repo_root::String, legate_root::String)
     if !check_if_patch(legate_root)
         legion_patch = joinpath(repo_root, "scripts/patch_legion.sh")
         @info "Legate.jl: Running legion patch script: $legion_patch"
-        run_sh(`bash $legion_patch $repo_root $legate_root`, "legion_patch")
-    end
-end
-
-function build_jlcxxwrap(repo_root, legate_root)
-    build_libcxxwrap = joinpath(repo_root, "scripts/install_cxxwrap.sh")
-    version_path = joinpath(DEPOT_PATH[1], "dev/libcxxwrap_julia_jll/override/LEGATE_INSTALL.txt")
-    if isfile(version_path)
-        version = VersionNumber(strip(read(version_path, String)))
-        @info "libcxxwrap: Found Legate $version"
-        if is_supported_version(version)
-            @info "libcxxwrap: Found supported version built with Legate.jl: $version"
-            return nothing
-        else
-            @info "libcxxwrap: Unsupported version found: $version. Rebuilding..."
-        end
-    else
-        @info "libcxxwrap: No version file found. Starting build..."
-    end
-
-    @info "libcxxwrap: Running build script: $build_libcxxwrap"
-    run_sh(`bash $build_libcxxwrap $repo_root`, "libcxxwrap")
-    open(version_path, "w") do io
-        write(io, string(get_legate_version(legate_root)))
+        BuildTools.run_sh(
+            `bash $legion_patch $repo_root $legate_root`, "legion_patch"; log_dir=@__DIR__
+        )
     end
 end
 
-function build_cpp_wrapper(repo_root, legate_root, install_root)
+function build_cpp_wrapper(
+    repo_root, legate_root, install_root; cuda_root=nothing, cuda_enabled=true
+)
     @info "liblegatewrapper: Building C++ Wrapper Library"
-    if isdir(install_root)
-        rm(install_root; recursive=true)
-        mkdir(install_root)
-    end
-
-    build_cpp_wrapper = joinpath(repo_root, "scripts/build_cpp_wrapper.sh")
-    nthreads = Threads.nthreads()
-
-    bld_command = `$build_cpp_wrapper $repo_root $legate_root $install_root $nthreads`
-
-    # write out a bash script for debugging
-    cmd_str = join(bld_command.exec, " ")
-    wrapper_path = joinpath(repo_root, "build_wrapper.sh")
-    open(wrapper_path, "w") do io
-        println(io, "#!/bin/bash")
-        println(io, "set -xe")
-        println(io, cmd_str)
-    end
-    chmod(wrapper_path, 0o755)
-
-    @info "Running build command: $bld_command"
-    run_sh(`bash $bld_command`, "cpp_wrapper")
-end
-
-function _find_jll_artifact_dir(jll)
-    eval(:(using $(jll)))
-    jll_mod = getfield(Main, jll)
-    root = jll_mod.artifact_dir
-    return root
+    isdir(install_root) && (rm(install_root; recursive=true); mkdir(install_root))
+    bld_command = `$(joinpath(repo_root, "scripts/build_cpp_wrapper.sh")) $repo_root $legate_root $install_root $(Threads.nthreads())`
+    BuildTools.run_build_wrapper_script(
+        repo_root, bld_command; cuda_root, cuda_enabled, log_dir=@__DIR__
+    )
 end
 
-function _start_build()
-    pkg_root = up_dir(@__DIR__)
-    deps_dir = joinpath(@__DIR__)
-
-    build_log = joinpath(deps_dir, "build.log")
-    open(build_log, "w") do io
-        println(io, "=== Build started ===")
-    end
-
-    @info "Legate.jl: Parsed Package Dir as: $(pkg_root)"
-    return pkg_root
-end
-
-"""
-    build CxxWrap and legate_jl_wrapper
-"""
-function build_deps(pkg_root, legate_root)
+function build_deps(pkg_root, legate_root; cuda_root=nothing, cuda_enabled=true)
+    BuildTools.check_cmake_version(MIN_CMAKE_VERSION)
     install_dir = joinpath(pkg_root, "lib", "legate_jl_wrapper", "build")
     if !legate_valid(legate_root)
         error(
             "Legate.jl: Unsupported Legate version at $(legate_root). " *
-            "Installed version: $(installed_version) not in range supported: " *
+            "Installed version: $(get_legate_version(legate_root)) not in range supported: " *
             "$(MIN_LEGATE_VERSION)-$(MAX_LEGATE_VERSION).",
         )
     end
-    build_jlcxxwrap(pkg_root, legate_root) # $pkg_root/lib/libcxxwrap-julia 
-    build_cpp_wrapper(pkg_root, legate_root, install_dir) # $pkg_root/lib/legate_jl_wrapper
+    BuildTools.build_jlcxxwrap(
+        pkg_root, get_legate_version(legate_root);
+        log_dir=@__DIR__, is_compatible=is_supported_version,
+    )
+    build_cpp_wrapper(pkg_root, legate_root, install_dir; cuda_root, cuda_enabled)
+    BuildTools.set_jll_artifact_override(:legate_jl_wrapper_jll, install_dir)
 end
 
 function build(::LegatePreferences.JLL)
@@ -157,7 +71,7 @@ end
 
 function build(::LegatePreferences.Conda)
     @warn "Conda Build does not currently pass our CI. Proceed with caution."
-    pkg_root = _start_build()
+    pkg_root = BuildTools.start_build("Legate.jl", @__DIR__)
 
     legate_root = load_preference(LegatePreferences, "legate_conda_env", nothing)
     if isnothing(legate_root)
@@ -170,19 +84,20 @@ function build(::LegatePreferences.Conda)
 end
 
 function build(::LegatePreferences.Developer)
-    pkg_root = _start_build()
+    pkg_root = BuildTools.start_build("Legate.jl", @__DIR__)
 
-    # can be nothing so this errors if not set
     legate_root = load_preference(LegatePreferences, "legate_path", nothing)
     if isnothing(legate_root)
-        # we are using legate_jll for legate
-        legate_root = _find_jll_artifact_dir(:legate_jll)
+        legate_root, cuda_root = BuildTools.setup_jll_build_env(pkg_root, BuildTools.LEGATE_JLL_DEP)
+        cuda_enabled = !isnothing(cuda_root) # cuda_root resolving to nothing means there is no cuda
     else
-        # this means we have a custom path set
         is_legate_installed(legate_root; throw_errors=true)
         patch_legion(pkg_root, legate_root)
+        cuda_enabled, cuda_root = BuildTools.resolve_custom_cuda("legate") # cuda_root is nothing.
     end
-    build_deps(pkg_root, legate_root)
+
+    build_deps(pkg_root, legate_root; cuda_root, cuda_enabled)
+    set_preferences!(LegatePreferences, "LEGATE_LIBDIR" => joinpath(legate_root, "lib"); force=true)
 end
 
 const mode_str = load_preference(LegatePreferences, "legate_mode", LegatePreferences.MODE_JLL)

deps/buildtools/cuda_tools.jl

@@ -0,0 +1,43 @@
+function resolve_custom_cuda(pkg_name::String)
+    cuda_enabled = Sys.which("nvcc") !== nothing
+    if !cuda_enabled
+        @warn "nvcc not found on PATH — building without CUDA. " *
+            "If your $(pkg_name) was built with CUDA, add nvcc to PATH and rebuild."
+    end
+    return cuda_enabled, nothing
+end
+
+function detect_jll_cuda_enabled(jll_mod)
+    cuda_val = get(jll_mod.host_platform.tags, "cuda", nothing)
+    return cuda_val !== nothing && cuda_val != "none"
+end
+
+# try/catch because CUDA_SDK_jll is a weakdep — may not be in the environment.
+function try_get_cuda_sdk_jll_dir()
+    try
+        Core.eval(Main, :(using CUDA_SDK_jll))
+        return joinpath(getfield(Main, :CUDA_SDK_jll).artifact_dir, "cuda")
+    catch
+        return nothing
+    end
+end
+
+function build_cuda_env(cuda_enabled::Bool, cuda_root)
+    # user-set env vars override auto-detection
+    if haskey(ENV, "LEGATE_WRAPPER_ENABLE_CUDA")
+        user_enable = uppercase(ENV["LEGATE_WRAPPER_ENABLE_CUDA"]) == "ON"
+        if user_enable && !cuda_enabled
+            @warn "LEGATE_WRAPPER_ENABLE_CUDA=ON overrides auto-detected cuda_enabled=false. " *
+                "The wrapper will link CUDA, but the underlying legate build may not — expect link/runtime errors if mismatched."
+        end
+        cuda_enabled = user_enable
+    end
+    if haskey(ENV, "CUDA_TOOLKIT_ROOT")
+        cuda_root = ENV["CUDA_TOOLKIT_ROOT"]
+    end
+
+    env = Dict{String,String}()
+    !cuda_enabled && (env["LEGATE_WRAPPER_ENABLE_CUDA"] = "OFF")
+    cuda_root !== nothing && (env["CUDA_TOOLKIT_ROOT"] = cuda_root)
+    return env
+end