test-rapids-build-times.yml

name: Test RAPIDS build times

on:
  workflow_dispatch:
    inputs:
      branch:
        type: string
        required: false
        default: main
      node_type:
        type: string
        required: false
        default: cpu32

jobs:
  check-event:
    name: Check GH Event
    runs-on: ubuntu-latest
    outputs:
      ok: ${{ steps.check_gh_event.outputs.ok }}
    steps:
      - id: check_gh_event
        name: Check GH Event
        shell: bash
        run: |
          [[ '${{ github.event_name }}' == 'workflow_dispatch' && '${{ github.repository }}' == 'rapidsai/devcontainers' ]] \
          && echo "ok=true"  | tee -a "$GITHUB_OUTPUT" \
          || echo "ok=false" | tee -a "$GITHUB_OUTPUT";

  test-rapids-build-times:
    name: ${{ matrix.name }}
    if: needs.check-event.outputs.ok == 'true'
    needs: check-event
    secrets: inherit
    uses: rapidsai/shared-workflows/.github/workflows/build-in-devcontainer.yaml@main
    permissions:
      actions: read
      packages: read
      id-token: write
      contents: read
      pull-requests: read
    strategy:
      fail-fast: false
      matrix:
        include:
          - name: 'build cluster: no'
            env: |
              SCCACHE_NO_DIST_COMPILE=1
              MAX_DEVICE_OBJ_TO_COMPILE_IN_PARALLEL=2
          - name: 'build cluster: yes'
            env: ""

    with:
      arch: '["amd64", "arm64"]'
      cuda: '["12.9", "13.1"]'
      node_type: ${{ inputs.node_type }}
      rapids-aux-secret-1: GIST_REPO_READ_ORG_GITHUB_TOKEN
      timeout-minutes: 720
      # 1. Prohibit sccache from shutting down automatically
      # 2. Infinitely retry transient errors
      # 3. Enable debug logging to track cache misses
      # 4. Never fallback to locally compiling
      # 5. Use RAPIDS_AUX_SECRET_1 as the sccache-dist auth token
      env: |
        CONDA_ENV_CREATE_QUIET=1
        PARALLEL_LEVEL=0
        SCCACHE_IDLE_TIMEOUT=0
        SCCACHE_SERVER_LOG=sccache=debug
        SCCACHE_DIST_MAX_RETRIES=inf
        SCCACHE_DIST_FALLBACK_TO_LOCAL_COMPILE=false
        SCCACHE_DIST_AUTH_TOKEN_VAR=RAPIDS_AUX_SECRET_1
        ${{ matrix.env }}
      build_command: |
        function begin_group() {
          local blue="34"
          echo -e "::group::\e[${blue}m${1:-}\e[0m"
        }

        function end_group() {
          local name="${1:-}"
          local build_status="${2:-0}"
          local red="31"

          echo "::endgroup::"
          if [ "$build_status" -ne 0 ]; then
            echo -e "::error::\e[${red}m ${name} - Failed (⬆️ click above for full log ⬆️)\e[0m"
          fi
        }

        function run_command() {
          local -;
          set -euo pipefail;

          local group="${1:-}";
          shift;
          local command=("$@");
          local exit_code="0";

          begin_group "$group";

          echo "Working directory: $(pwd)";
          echo "Running command: ${command[*]}";
          "${command[@]}" || exit_code=$?;

          end_group "$group" "$exit_code"

          return "$exit_code"
        }

        # Clone all the repos
        time run_command "Clone RAPIDS repositories" bash -c "\
          clone-all \
            -j$(nproc) \
            -b ${{ inputs.branch }} \
            -q \
            -v \
            --clone-upstream \
            --depth 1 \
            --single-branch \
            --shallow-submodules \
            --no-update-env \
          >/dev/null 2>&1
          ";

        time run_command "Create RAPIDS python environment" \
          rapids-post-start-command;

        # Configure all the C++ libs
        time run_command "Configure C++ libraries" bash -c "\
          configure-all \
            -j${PARALLEL_LEVEL} \
            -GNinja \
            -Wno-dev \
            -DBUILD_TESTS=ON \
            -DBUILD_BENCHMARKS=ON \
            -DBUILD_PRIMS_BENCH=ON \
            -DBUILD_SHARED_LIBS=ON \
            -DRAFT_COMPILE_LIBRARY=ON \
            -DBUILD_CUGRAPH_MG_TESTS=ON \
          >/dev/null"

        # for ENVVAR in "SCCACHE_RECACHE=1" \
        for ENVVAR in "SCCACHE_S3_USE_PREPROCESSOR_CACHE_MODE=1" \
                      "SCCACHE_S3_USE_PREPROCESSOR_CACHE_MODE=0" ; do

          # Restart sccache
          run_command "Start sccache (${ENVVAR})" bash -c "\
            env ${ENVVAR} devcontainer-utils-start-sccache --kill-all >/dev/null 2>&1"

          # Build all the C++ libs
          time run_command "Build C++ libraries (${ENVVAR})" bash -c "\
            ${ENVVAR} build-all-cpp -j${PARALLEL_LEVEL}"

          # Print cache and dist stats
          run_command "sccache stats (${ENVVAR})" \
            sccache --show-adv-stats

          # Print build times
          run_command "Build times (${ENVVAR})" bash -c "\
            find /var/log/devcontainer-utils/ -type f -name 'build-*-time.log' -print0 \
          | xargs -0 -n1 grep -H real | sed 's/real\t/ /g' || : # Nonfatal if not found"

          # Clean
          run_command "Clean (${ENVVAR})" bash -c "\
            find /var/log/devcontainer-utils/ -type f -name 'build-*-time.log' -delete >/dev/null 2>&1 || : # Nonfatal if not found
            find ~/ -maxdepth 4 -type l -path '*/cpp/build/latest' -print0 | xargs -P$(nproc) -0 -n1 ninja clean -C >/dev/null 2>&1 || : # Nonfatal if not found"
        done