Skip to content

Commit 0067d3e

Browse files
committed
Share GPU diagnostics across CI tests
1 parent c763e1e commit 0067d3e

6 files changed

Lines changed: 22 additions & 23 deletions

File tree

ci/gpu_info.sh

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
#!/bin/bash
2+
# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
3+
# SPDX-License-Identifier: Apache-2.0
4+
5+
set -euo pipefail
6+
7+
rapids-logger "Check GPU status"
8+
nvidia-smi
9+
10+
rapids-logger "Check memory configuration"
11+
nvidia-smi -q | grep "Addressing Mode" || echo "Addressing Mode not reported"
12+
grep Coherent /proc/driver/nvidia/params || echo "Coherent GPU memory mode not reported"
13+
grep -o 'init_on_alloc=[^ ]*' /proc/cmdline || echo "init_on_alloc: kernel default"

ci/test_cpp.sh

Lines changed: 1 addition & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -33,13 +33,7 @@ set -u
3333

3434
rapids-print-env
3535

36-
rapids-logger "Check GPU status"
37-
nvidia-smi
38-
39-
rapids-logger "Check memory configuration"
40-
nvidia-smi -q | grep "Addressing Mode" || echo "Addressing Mode not reported"
41-
grep Coherent /proc/driver/nvidia/params || echo "Coherent GPU memory mode not reported"
42-
grep -o 'init_on_alloc=[^ ]*' /proc/cmdline || echo "init_on_alloc: kernel default"
36+
./ci/gpu_info.sh
4337

4438
# Run librmm gtests from librmm-tests package
4539
rapids-logger "Run gtests"

ci/test_cpp_debug.sh

Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
#!/bin/bash
2-
# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION.
2+
# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
33
# SPDX-License-Identifier: Apache-2.0
44

55
set -euo pipefail
@@ -33,9 +33,7 @@ set -u
3333

3434
rapids-print-env
3535

36-
rapids-logger "Check GPU status"
37-
nvidia-smi
38-
nvidia-smi -q | grep "Addressing Mode" || echo "Addressing Mode not reported"
36+
./ci/gpu_info.sh
3937

4038
rapids-logger "Building librmm in Debug mode"
4139
cmake -S cpp -B cpp/build \

ci/test_python.sh

Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
#!/bin/bash
2-
# SPDX-FileCopyrightText: Copyright (c) 2020-2026, NVIDIA CORPORATION.
2+
# SPDX-FileCopyrightText: Copyright (c) 2020-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
33
# SPDX-License-Identifier: Apache-2.0
44

55
set -euo pipefail
@@ -39,9 +39,7 @@ RAPIDS_TESTS_DIR=${RAPIDS_TESTS_DIR:-"${PWD}/test-results"}
3939
RAPIDS_COVERAGE_DIR=${RAPIDS_COVERAGE_DIR:-"${PWD}/coverage-results"}
4040
mkdir -p "${RAPIDS_TESTS_DIR}" "${RAPIDS_COVERAGE_DIR}"
4141

42-
rapids-logger "Check GPU status"
43-
nvidia-smi
44-
nvidia-smi -q | grep "Addressing Mode" || echo "Addressing Mode not reported"
42+
./ci/gpu_info.sh
4543

4644
rapids-logger "pytest rmm"
4745

ci/test_python_integrations.sh

Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
#!/bin/bash
2-
# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION.
2+
# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
33
# SPDX-License-Identifier: Apache-2.0
44

55
set -euo pipefail
@@ -22,9 +22,7 @@ RAPIDS_TESTS_DIR=${RAPIDS_TESTS_DIR:-"${PWD}/test-results"}
2222
RAPIDS_COVERAGE_DIR=${RAPIDS_COVERAGE_DIR:-"${PWD}/coverage-results"}
2323
mkdir -p "${RAPIDS_TESTS_DIR}" "${RAPIDS_COVERAGE_DIR}"
2424

25-
rapids-logger "Check GPU status"
26-
nvidia-smi
27-
nvidia-smi -q | grep "Addressing Mode" || echo "Addressing Mode not reported"
25+
./ci/gpu_info.sh
2826

2927
EXITCODE=0
3028

ci/test_wheel_integrations.sh

Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
#!/bin/bash
2-
# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION.
2+
# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
33
# SPDX-License-Identifier: Apache-2.0
44

55
set -eou pipefail
@@ -28,9 +28,7 @@ PIP_INSTALL_SHARED_ARGS=(
2828

2929
EXITCODE=0
3030

31-
rapids-logger "Check GPU status"
32-
nvidia-smi
33-
nvidia-smi -q | grep "Addressing Mode" || echo "Addressing Mode not reported"
31+
./ci/gpu_info.sh
3432

3533
echo "::group::PyTorch Tests"
3634

0 commit comments

Comments
 (0)