Test RAPIDS build times #24
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Test RAPIDS build times | |
| on: | |
| workflow_dispatch: | |
| inputs: | |
| branch: | |
| type: string | |
| required: false | |
| default: main | |
| node_type: | |
| type: string | |
| required: false | |
| default: cpu32 | |
| jobs: | |
| check-event: | |
| name: Check GH Event | |
| runs-on: ubuntu-latest | |
| outputs: | |
| ok: ${{ steps.check_gh_event.outputs.ok }} | |
| steps: | |
| - id: check_gh_event | |
| name: Check GH Event | |
| shell: bash | |
| run: | | |
| [[ '${{ github.event_name }}' == 'workflow_dispatch' && '${{ github.repository }}' == 'rapidsai/devcontainers' ]] \ | |
| && echo "ok=true" | tee -a "$GITHUB_OUTPUT" \ | |
| || echo "ok=false" | tee -a "$GITHUB_OUTPUT"; | |
| test-rapids-build-times: | |
| name: ${{ matrix.name }} | |
| if: needs.check-event.outputs.ok == 'true' | |
| needs: check-event | |
| secrets: inherit | |
| uses: rapidsai/shared-workflows/.github/workflows/build-in-devcontainer.yaml@main | |
| permissions: | |
| actions: read | |
| packages: read | |
| id-token: write | |
| contents: read | |
| pull-requests: read | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| include: | |
| - name: 'build cluster: no' | |
| env: | | |
| SCCACHE_NO_DIST_COMPILE=1 | |
| MAX_DEVICE_OBJ_TO_COMPILE_IN_PARALLEL=2 | |
| - name: 'build cluster: yes' | |
| env: "" | |
| with: | |
| arch: '["amd64", "arm64"]' | |
| cuda: '["12.9", "13.1"]' | |
| node_type: ${{ inputs.node_type }} | |
| rapids-aux-secret-1: GIST_REPO_READ_ORG_GITHUB_TOKEN | |
| timeout-minutes: 720 | |
| # 1. Prohibit sccache from shutting down automatically | |
| # 2. Infinitely retry transient errors | |
| # 3. Enable debug logging to track cache misses | |
| # 4. Never fallback to locally compiling | |
| # 5. Use RAPIDS_AUX_SECRET_1 as the sccache-dist auth token | |
| env: | | |
| CONDA_ENV_CREATE_QUIET=1 | |
| PARALLEL_LEVEL=0 | |
| SCCACHE_IDLE_TIMEOUT=0 | |
| SCCACHE_SERVER_LOG=sccache=debug | |
| SCCACHE_DIST_MAX_RETRIES=inf | |
| SCCACHE_DIST_FALLBACK_TO_LOCAL_COMPILE=false | |
| SCCACHE_DIST_AUTH_TOKEN_VAR=RAPIDS_AUX_SECRET_1 | |
| TIME=real:\t%E | |
| ${{ matrix.env }} | |
| build_command: | | |
| function begin_group() { | |
| local blue="34" | |
| echo -e "::group::\e[${blue}m${1:-}\e[0m" | |
| } | |
| function end_group() { | |
| local name="${1:-}" | |
| local build_status="${2:-0}" | |
| local red="31" | |
| echo "::endgroup::" | |
| if [ "$build_status" -ne 0 ]; then | |
| echo -e "::error::\e[${red}m ${name} - Failed (⬆️ click above for full log ⬆️)\e[0m" | |
| fi | |
| } | |
| function run_command() { | |
| local -; | |
| set -euo pipefail; | |
| local group="${1:-}"; | |
| shift; | |
| local command=("$@"); | |
| local exit_code="0"; | |
| begin_group "$group"; | |
| echo "Working directory: $(pwd)"; | |
| echo "Running command: ${command[*]@Q}"; | |
| "${command[@]}" || exit_code=$?; | |
| end_group "$group" "$exit_code" | |
| return "$exit_code" | |
| } | |
| # convert ucx branch names | |
| convert_ucx_branch() { | |
| local -; | |
| set -euo pipefail; | |
| local repo="$1" | |
| local custom_branch="$2" | |
| local normalized_branch="${custom_branch}" | |
| if [[ "${repo}" == "ucx"* ]]; then | |
| # Only convert branches that match the pattern release/YY.MM | |
| if [[ "${custom_branch}" =~ ^release/[0-9]{2}\.[0-9]{2}$ ]]; then | |
| RAPIDS_VERSION=$(echo "${custom_branch}" | awk '{split($0, a, "/"); print a[2]}') | |
| # Get UCX version associated w/ RAPIDS version | |
| UCX_VERSION="$(curl -sL https://version.gpuci.io/rapids/${RAPIDS_VERSION})" | |
| normalized_branch="release/${UCX_VERSION}" | |
| fi | |
| fi | |
| echo "${normalized_branch}" | |
| } | |
| # Clone all the repos | |
| time run_command "Clone RAPIDS repositories" bash -c "\ | |
| RAPIDS_TO_UCXX_BRANCH=\"$(convert_ucx_branch ucxx '${{ inputs.branch }}')\"; | |
| CLONE_ARGS=(-j$(nproc) -q -v --clone-upstream --depth 1 --single-branch --shallow-submodules --no-update-env); | |
| clone-all -b \"${{ inputs.branch }}\" \${CLONE_ARGS[*]} >/dev/null 2>&1; | |
| clone-ucxx -b \"\$RAPIDS_TO_UCXX_BRANCH\" \${CLONE_ARGS[*]} >/dev/null 2>&1; | |
| " 2>&1 | |
| sleep 1 | |
| time run_command "Create RAPIDS python environment" bash -c "\ | |
| rapids-post-start-command >/dev/null" 2>&1 | |
| sleep 1 | |
| # Configure all the C++ libs | |
| time run_command "Configure C++ libraries" bash -c "\ | |
| configure-all \ | |
| -j${PARALLEL_LEVEL} \ | |
| -GNinja \ | |
| -Wno-dev \ | |
| -DBUILD_TESTS=ON \ | |
| -DBUILD_BENCHMARKS=ON \ | |
| -DBUILD_PRIMS_BENCH=ON \ | |
| -DBUILD_SHARED_LIBS=ON \ | |
| -DRAFT_COMPILE_LIBRARY=ON \ | |
| -DBUILD_CUGRAPH_MG_TESTS=ON \ | |
| >/dev/null" 2>&1 | |
| sleep 1 | |
| for ENVVAR in "SCCACHE_RECACHE=1" \ | |
| "SCCACHE_S3_USE_PREPROCESSOR_CACHE_MODE=0" \ | |
| "SCCACHE_S3_USE_PREPROCESSOR_CACHE_MODE=1" ; do | |
| # Restart sccache | |
| run_command "Start sccache (${ENVVAR})" bash -c "\ | |
| ${ENVVAR} devcontainer-utils-start-sccache --kill-all >/dev/null 2>&1" | |
| # Build all the C++ libs | |
| time run_command "Build C++ libraries (${ENVVAR})" bash -c "\ | |
| ${ENVVAR} build-all-cpp -j${PARALLEL_LEVEL} >/dev/null" 2>&1 | |
| sleep 1 | |
| # Print cache and dist stats | |
| run_command "sccache stats (${ENVVAR})" \ | |
| sccache --show-adv-stats | |
| # Print build times | |
| run_command "Build times (${ENVVAR})" bash -c "\ | |
| find /var/log/devcontainer-utils/ -type f -name 'build-*-time.log' -print0 \ | |
| | xargs -0 -n1 grep -H real | sed 's/real\t/ /g' || : # Nonfatal if not found" | |
| # Clean | |
| run_command "Clean (${ENVVAR})" bash -c "\ | |
| find /var/log/devcontainer-utils/ -type f -name 'build-*-time.log' -delete >/dev/null 2>&1 || : # Nonfatal if not found | |
| find ~/ -maxdepth 4 -type l -path '*/cpp/build/latest' -print0 | xargs -P$(nproc) -0 -n1 ninja clean -C >/dev/null 2>&1 || : # Nonfatal if not found" | |
| done |