Skip to content

Commit 401efb1

Browse files
committed
feat(ci): Add test failure reporting with PR comments and error annotations
When tests fail in CI, capture the failed test names from ctest output and surface them in two ways: - GitHub Actions ::error:: annotations visible in the job summary - Automatic PR comment listing the specific failed tests with a link to the full logs Applied to both Linux adapters release and Ubuntu debug test jobs.
1 parent 1210ef6 commit 401efb1

File tree

1 file changed

+97
-2
lines changed

1 file changed

+97
-2
lines changed

.github/workflows/linux-build-base.yml

Lines changed: 97 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -96,6 +96,7 @@ jobs:
9696
cudf-changes: ${{ steps.changes.outputs.cudf }}
9797
build-outcome: ${{ steps.build.outcome }}
9898
test-outcome: ${{ steps.tests.outcome }}
99+
failed-tests: ${{ steps.tests.outputs.failed-tests }}
99100
steps:
100101
- uses: actions/checkout@v5
101102
with:
@@ -196,7 +197,22 @@ jobs:
196197
run: |
197198
source /setup-classpath.sh
198199
ulimit -n 65536
199-
ctest -j 24 --timeout 900 --label-exclude cuda_driver --output-on-failure --no-tests=error
200+
set +e
201+
ctest -j 24 --timeout 900 --label-exclude cuda_driver --output-on-failure --no-tests=error 2>&1 | tee /tmp/ctest-output.log
202+
CTEST_EXIT=${PIPESTATUS[0]}
203+
set -e
204+
if [[ $CTEST_EXIT -ne 0 ]]; then
205+
FAILED_TESTS=$(grep -A 1000 'The following tests FAILED:' /tmp/ctest-output.log | grep -E '^\s+[0-9]+ - ' | sed 's/.*- \(.*\) (.*/\1/' || true)
206+
if [[ -n "$FAILED_TESTS" ]]; then
207+
echo "::error::Failed tests in Linux adapters release: $(echo "$FAILED_TESTS" | tr '\n' ', ' | sed 's/, $//')"
208+
{
209+
echo 'failed-tests<<EOF'
210+
echo "$FAILED_TESTS"
211+
echo 'EOF'
212+
} >> "$GITHUB_OUTPUT"
213+
fi
214+
exit 1
215+
fi
200216
201217
# Clang-tidy needs a complete build because some files are only generated during the build
202218
# that clang tidy will not find and report errors otherwise.
@@ -315,13 +331,44 @@ jobs:
315331
exit 1
316332
fi
317333
if [[ "$TEST_OUTCOME" != "success" ]]; then
334+
if [[ -n "$FAILED_TESTS" ]]; then
335+
echo "::error::The following tests failed in Linux adapters release:"
336+
echo "$FAILED_TESTS" | while read -r test; do
337+
echo "::error:: - $test"
338+
done
339+
fi
318340
echo "Tests failed."
319341
exit 1
320342
fi
321343
echo "Tests passed."
322344
env:
323345
BUILD_OUTCOME: ${{ needs.adapters.outputs.build-outcome }}
324346
TEST_OUTCOME: ${{ needs.adapters.outputs.test-outcome }}
347+
FAILED_TESTS: ${{ needs.adapters.outputs.failed-tests }}
348+
349+
- name: Comment on PR with test failures
350+
if: failure() && needs.adapters.outputs.failed-tests != ''
351+
env:
352+
GH_TOKEN: ${{ github.token }}
353+
FAILED_TESTS: ${{ needs.adapters.outputs.failed-tests }}
354+
RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
355+
run: |
356+
PR_NUMBER="${{ github.event.pull_request.number }}"
357+
if [[ -z "$PR_NUMBER" ]]; then
358+
echo "Not a pull request — skipping comment."
359+
exit 0
360+
fi
361+
FORMATTED=$(echo "$FAILED_TESTS" | sed 's/^/- /')
362+
BODY="### :x: Test Failures: Linux adapters release
363+
364+
The following tests failed:
365+
366+
${FORMATTED}
367+
368+
:link: [View full test logs](${RUN_URL})"
369+
gh pr comment "$PR_NUMBER" \
370+
--repo "${{ github.repository }}" \
371+
--body "$BODY"
325372
326373
cudf-tests:
327374
runs-on: 4-core-ubuntu-gpu-t4
@@ -396,6 +443,7 @@ jobs:
396443
outputs:
397444
build-outcome: ${{ steps.build.outcome }}
398445
test-outcome: ${{ steps.tests.outcome }}
446+
failed-tests: ${{ steps.tests.outputs.failed-tests }}
399447
defaults:
400448
run:
401449
shell: bash
@@ -493,7 +541,23 @@ jobs:
493541
if: steps.build.outcome == 'success'
494542
run: |
495543
ulimit -n 65536
496-
cd _build/debug && ctest -j 24 --timeout 1800 --output-on-failure --no-tests=error
544+
cd _build/debug
545+
set +e
546+
ctest -j 24 --timeout 1800 --output-on-failure --no-tests=error 2>&1 | tee /tmp/ctest-output.log
547+
CTEST_EXIT=${PIPESTATUS[0]}
548+
set -e
549+
if [[ $CTEST_EXIT -ne 0 ]]; then
550+
FAILED_TESTS=$(grep -A 1000 'The following tests FAILED:' /tmp/ctest-output.log | grep -E '^\s+[0-9]+ - ' | sed 's/.*- \(.*\) (.*/\1/' || true)
551+
if [[ -n "$FAILED_TESTS" ]]; then
552+
echo "::error::Failed tests in Ubuntu debug: $(echo "$FAILED_TESTS" | tr '\n' ', ' | sed 's/, $//')"
553+
{
554+
echo 'failed-tests<<EOF'
555+
echo "$FAILED_TESTS"
556+
echo 'EOF'
557+
} >> "$GITHUB_OUTPUT"
558+
fi
559+
exit 1
560+
fi
497561
498562
ubuntu-debug-build-status:
499563
if: always()
@@ -522,13 +586,44 @@ jobs:
522586
exit 1
523587
fi
524588
if [[ "$TEST_OUTCOME" != "success" ]]; then
589+
if [[ -n "$FAILED_TESTS" ]]; then
590+
echo "::error::The following tests failed in Ubuntu debug:"
591+
echo "$FAILED_TESTS" | while read -r test; do
592+
echo "::error:: - $test"
593+
done
594+
fi
525595
echo "Tests failed."
526596
exit 1
527597
fi
528598
echo "Tests passed."
529599
env:
530600
BUILD_OUTCOME: ${{ needs.ubuntu-debug.outputs.build-outcome }}
531601
TEST_OUTCOME: ${{ needs.ubuntu-debug.outputs.test-outcome }}
602+
FAILED_TESTS: ${{ needs.ubuntu-debug.outputs.failed-tests }}
603+
604+
- name: Comment on PR with test failures
605+
if: failure() && needs.ubuntu-debug.outputs.failed-tests != ''
606+
env:
607+
GH_TOKEN: ${{ github.token }}
608+
FAILED_TESTS: ${{ needs.ubuntu-debug.outputs.failed-tests }}
609+
RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
610+
run: |
611+
PR_NUMBER="${{ github.event.pull_request.number }}"
612+
if [[ -z "$PR_NUMBER" ]]; then
613+
echo "Not a pull request — skipping comment."
614+
exit 0
615+
fi
616+
FORMATTED=$(echo "$FAILED_TESTS" | sed 's/^/- /')
617+
BODY="### :x: Test Failures: Ubuntu debug
618+
619+
The following tests failed:
620+
621+
${FORMATTED}
622+
623+
:link: [View full test logs](${RUN_URL})"
624+
gh pr comment "$PR_NUMBER" \
625+
--repo "${{ github.repository }}" \
626+
--body "$BODY"
532627
533628
fedora-debug:
534629
runs-on: 32-core-ubuntu

0 commit comments

Comments
 (0)