Skip to content

Commit d48195c

Browse files
kgpaikgpai
committed
feat(ci): Report specific failed test cases and improve navigation guidance
- Extract individual gtest case names (e.g., ValuesTest.empty) from ctest output in addition to ctest group names (e.g., velox_exec_test_group0) - Prefer showing specific test case names when available - Update error annotations and PR comments to guide users to the specific job and 'Run Tests' step for full failure details
1 parent ccf8f85 commit d48195c

File tree

1 file changed

+76
-20
lines changed

1 file changed

+76
-20
lines changed

‎.github/workflows/linux-build-base.yml‎

Lines changed: 76 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -98,6 +98,7 @@ jobs:
9898
test-outcome: ${{ steps.tests.outcome }}
9999
flaky: ${{ steps.tests.outputs.flaky }}
100100
failed-tests: ${{ steps.tests.outputs.failed-tests }}
101+
failed-cases: ${{ steps.tests.outputs.failed-cases }}
101102
steps:
102103
- uses: actions/checkout@v5
103104
with:
@@ -211,13 +212,21 @@ jobs:
211212
else
212213
echo "::error::Tests failed consistently on retry."
213214
FAILED_TESTS=$(grep -A 1000 'The following tests FAILED:' /tmp/ctest-output.log | grep -E '^\s+[0-9]+ - ' | sed 's/.*- \(.*\) (.*/\1/' || true)
215+
FAILED_CASES=$(grep -E '^\[ FAILED \]' /tmp/ctest-output.log | sed 's/\[ FAILED \] //' | sed 's/ (.*//' || true)
214216
if [[ -n "$FAILED_TESTS" ]]; then
215217
{
216218
echo 'failed-tests<<EOF'
217219
echo "$FAILED_TESTS"
218220
echo 'EOF'
219221
} >> "$GITHUB_OUTPUT"
220222
fi
223+
if [[ -n "$FAILED_CASES" ]]; then
224+
{
225+
echo 'failed-cases<<EOF'
226+
echo "$FAILED_CASES"
227+
echo 'EOF'
228+
} >> "$GITHUB_OUTPUT"
229+
fi
221230
exit 1
222231
fi
223232
fi
@@ -362,18 +371,25 @@ jobs:
362371
exit 1
363372
fi
364373
if [[ "$TEST_OUTCOME" != "success" ]]; then
365-
if [[ -n "$FAILED_TESTS" ]]; then
374+
if [[ -n "$FAILED_CASES" ]]; then
375+
CASE_LIST=$(echo "$FAILED_CASES" | sed 's/^/ - /')
376+
CASE_COUNT=$(echo "$FAILED_CASES" | wc -l | tr -d ' ')
377+
echo "::error::${CASE_COUNT} test case(s) failed in the Linux adapters release configuration. Failed: $(echo "$FAILED_CASES" | tr '\n' ', ' | sed 's/,$//'). To see the full test output with failure details, click the 'Linux release with adapters' job in the workflow run, then expand the 'Run Tests' step."
378+
echo ""
379+
echo "Failed test cases:"
380+
echo "$CASE_LIST"
381+
elif [[ -n "$FAILED_TESTS" ]]; then
366382
TEST_LIST=$(echo "$FAILED_TESTS" | sed 's/^/ - /')
367383
TEST_COUNT=$(echo "$FAILED_TESTS" | wc -l | tr -d ' ')
368-
echo "::error::${TEST_COUNT} test(s) failed in the Linux adapters release configuration. Failed tests: $(echo "$FAILED_TESTS" | tr '\n' ', ' | sed 's/,$//'). To see the full test output with failure details, check the 'Run Tests' step in the 'Linux release with adapters' job."
384+
echo "::error::${TEST_COUNT} test(s) failed in the Linux adapters release configuration. Failed tests: $(echo "$FAILED_TESTS" | tr '\n' ', ' | sed 's/,$//'). To see the full test output with failure details, click the 'Linux release with adapters' job in the workflow run, then expand the 'Run Tests' step."
369385
echo ""
370386
echo "Failed tests:"
371387
echo "$TEST_LIST"
372-
echo ""
373-
echo "To investigate, look at the 'Run Tests' step in the 'Linux release with adapters' job for the full ctest output, which includes the specific assertion failures and stack traces for each test."
374388
else
375389
echo "::error::Tests failed in the Linux adapters release configuration but no specific test names were captured. Check the 'Run Tests' step in the 'Linux release with adapters' job for details."
376390
fi
391+
echo ""
392+
echo "To investigate, click the 'Linux release with adapters' job in the workflow run, then expand the 'Run Tests' step for the full ctest output with assertion failures and stack traces."
377393
exit 1
378394
fi
379395
if [[ "$FLAKY" == "true" ]]; then
@@ -385,34 +401,46 @@ jobs:
385401
TEST_OUTCOME: ${{ needs.adapters.outputs.test-outcome }}
386402
FLAKY: ${{ needs.adapters.outputs.flaky }}
387403
FAILED_TESTS: ${{ needs.adapters.outputs.failed-tests }}
404+
FAILED_CASES: ${{ needs.adapters.outputs.failed-cases }}
388405
389406
- name: Upload test failure artifact
390-
if: failure() && needs.adapters.outputs.failed-tests != ''
407+
if: failure()
391408
env:
392409
FAILED_TESTS: ${{ needs.adapters.outputs.failed-tests }}
410+
FAILED_CASES: ${{ needs.adapters.outputs.failed-cases }}
393411
RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
394412
run: |
395413
mkdir -p /tmp/ci-failure
396-
FORMATTED=$(echo "$FAILED_TESTS" | sed 's/^/- /')
414+
if [[ -n "$FAILED_CASES" ]]; then
415+
FORMATTED=$(echo "$FAILED_CASES" | sed 's/^/- /')
416+
LABEL="test cases"
417+
elif [[ -n "$FAILED_TESTS" ]]; then
418+
FORMATTED=$(echo "$FAILED_TESTS" | sed 's/^/- /')
419+
LABEL="tests"
420+
else
421+
echo "No failure details to upload."
422+
exit 0
423+
fi
397424
cat > /tmp/ci-failure/comment.md <<EOF
398425
### :x: Test Failures: Linux adapters release
399426
400-
The following tests failed:
427+
The following ${LABEL} failed:
401428
402429
${FORMATTED}
403430
404-
Check the **Linux release with adapters** workflow for detailed logs.
431+
To investigate, click the **Linux release with adapters** job in the workflow run below, then expand the **Run Tests** step for full ctest output with assertion failures and stack traces.
405432
406-
:link: [View full test logs](${RUN_URL})
433+
:link: [View workflow run](${RUN_URL})
407434
EOF
408435
409436
- name: Upload failure comment artifact
410-
if: failure() && needs.adapters.outputs.failed-tests != ''
437+
if: failure()
411438
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
412439
with:
413440
name: ci-failure-adapters-test
414441
path: /tmp/ci-failure/comment.md
415442
retention-days: 1
443+
if-no-files-found: ignore
416444

417445
cudf-tests:
418446
runs-on: 4-core-ubuntu-gpu-t4
@@ -489,6 +517,7 @@ jobs:
489517
test-outcome: ${{ steps.tests.outcome }}
490518
flaky: ${{ steps.tests.outputs.flaky }}
491519
failed-tests: ${{ steps.tests.outputs.failed-tests }}
520+
failed-cases: ${{ steps.tests.outputs.failed-cases }}
492521
defaults:
493522
run:
494523
shell: bash
@@ -599,13 +628,21 @@ jobs:
599628
else
600629
echo "::error::Tests failed consistently on retry."
601630
FAILED_TESTS=$(grep -A 1000 'The following tests FAILED:' /tmp/ctest-output.log | grep -E '^\s+[0-9]+ - ' | sed 's/.*- \(.*\) (.*/\1/' || true)
631+
FAILED_CASES=$(grep -E '^\[ FAILED \]' /tmp/ctest-output.log | sed 's/\[ FAILED \] //' | sed 's/ (.*//' || true)
602632
if [[ -n "$FAILED_TESTS" ]]; then
603633
{
604634
echo 'failed-tests<<EOF'
605635
echo "$FAILED_TESTS"
606636
echo 'EOF'
607637
} >> "$GITHUB_OUTPUT"
608638
fi
639+
if [[ -n "$FAILED_CASES" ]]; then
640+
{
641+
echo 'failed-cases<<EOF'
642+
echo "$FAILED_CASES"
643+
echo 'EOF'
644+
} >> "$GITHUB_OUTPUT"
645+
fi
609646
exit 1
610647
fi
611648
fi
@@ -660,18 +697,25 @@ jobs:
660697
exit 1
661698
fi
662699
if [[ "$TEST_OUTCOME" != "success" ]]; then
663-
if [[ -n "$FAILED_TESTS" ]]; then
700+
if [[ -n "$FAILED_CASES" ]]; then
701+
CASE_LIST=$(echo "$FAILED_CASES" | sed 's/^/ - /')
702+
CASE_COUNT=$(echo "$FAILED_CASES" | wc -l | tr -d ' ')
703+
echo "::error::${CASE_COUNT} test case(s) failed in the Ubuntu debug configuration. Failed: $(echo "$FAILED_CASES" | tr '\n' ', ' | sed 's/,$//'). To see the full test output with failure details, click the 'Ubuntu debug with system dependencies' job in the workflow run, then expand the 'Run Tests' step."
704+
echo ""
705+
echo "Failed test cases:"
706+
echo "$CASE_LIST"
707+
elif [[ -n "$FAILED_TESTS" ]]; then
664708
TEST_LIST=$(echo "$FAILED_TESTS" | sed 's/^/ - /')
665709
TEST_COUNT=$(echo "$FAILED_TESTS" | wc -l | tr -d ' ')
666-
echo "::error::${TEST_COUNT} test(s) failed in the Ubuntu debug configuration. Failed tests: $(echo "$FAILED_TESTS" | tr '\n' ', ' | sed 's/,$//'). To see the full test output with failure details, check the 'Run Tests' step in the 'Ubuntu debug with system dependencies' job."
710+
echo "::error::${TEST_COUNT} test(s) failed in the Ubuntu debug configuration. Failed tests: $(echo "$FAILED_TESTS" | tr '\n' ', ' | sed 's/,$//'). To see the full test output with failure details, click the 'Ubuntu debug with system dependencies' job in the workflow run, then expand the 'Run Tests' step."
667711
echo ""
668712
echo "Failed tests:"
669713
echo "$TEST_LIST"
670-
echo ""
671-
echo "To investigate, look at the 'Run Tests' step in the 'Ubuntu debug with system dependencies' job for the full ctest output, which includes the specific assertion failures and stack traces for each test."
672714
else
673715
echo "::error::Tests failed in the Ubuntu debug configuration but no specific test names were captured. Check the 'Run Tests' step in the 'Ubuntu debug with system dependencies' job for details."
674716
fi
717+
echo ""
718+
echo "To investigate, click the 'Ubuntu debug with system dependencies' job in the workflow run, then expand the 'Run Tests' step for the full ctest output with assertion failures and stack traces."
675719
exit 1
676720
fi
677721
if [[ "$FLAKY" == "true" ]]; then
@@ -683,34 +727,46 @@ jobs:
683727
TEST_OUTCOME: ${{ needs.ubuntu-debug.outputs.test-outcome }}
684728
FLAKY: ${{ needs.ubuntu-debug.outputs.flaky }}
685729
FAILED_TESTS: ${{ needs.ubuntu-debug.outputs.failed-tests }}
730+
FAILED_CASES: ${{ needs.ubuntu-debug.outputs.failed-cases }}
686731
687732
- name: Upload test failure artifact
688-
if: failure() && needs.ubuntu-debug.outputs.failed-tests != ''
733+
if: failure()
689734
env:
690735
FAILED_TESTS: ${{ needs.ubuntu-debug.outputs.failed-tests }}
736+
FAILED_CASES: ${{ needs.ubuntu-debug.outputs.failed-cases }}
691737
RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
692738
run: |
693739
mkdir -p /tmp/ci-failure
694-
FORMATTED=$(echo "$FAILED_TESTS" | sed 's/^/- /')
740+
if [[ -n "$FAILED_CASES" ]]; then
741+
FORMATTED=$(echo "$FAILED_CASES" | sed 's/^/- /')
742+
LABEL="test cases"
743+
elif [[ -n "$FAILED_TESTS" ]]; then
744+
FORMATTED=$(echo "$FAILED_TESTS" | sed 's/^/- /')
745+
LABEL="tests"
746+
else
747+
echo "No failure details to upload."
748+
exit 0
749+
fi
695750
cat > /tmp/ci-failure/comment.md <<EOF
696751
### :x: Test Failures: Ubuntu debug
697752
698-
The following tests failed:
753+
The following ${LABEL} failed:
699754
700755
${FORMATTED}
701756
702-
Check the **Ubuntu debug with system dependencies** workflow for detailed logs.
757+
To investigate, click the **Ubuntu debug with system dependencies** job in the workflow run below, then expand the **Run Tests** step for full ctest output with assertion failures and stack traces.
703758
704-
:link: [View full test logs](${RUN_URL})
759+
:link: [View workflow run](${RUN_URL})
705760
EOF
706761
707762
- name: Upload failure comment artifact
708-
if: failure() && needs.ubuntu-debug.outputs.failed-tests != ''
763+
if: failure()
709764
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
710765
with:
711766
name: ci-failure-ubuntu-debug-test
712767
path: /tmp/ci-failure/comment.md
713768
retention-days: 1
769+
if-no-files-found: ignore
714770

715771
fedora-debug:
716772
runs-on: 32-core-ubuntu

0 commit comments

Comments
 (0)