Skip to content

Report unit test files with no result #3105

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
yongwww merged 5 commits into from
Apr 23, 2026
+132 −52
Merged

Report unit test files with no result #3105

Changes from 4 commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
5685596
Report unit test files with no result
dierksen Apr 16, 2026
49dd90d
Fix unit test no-result summary parsing
dierksen Apr 17, 2026
30789df
Address no-result PR review feedback
dierksen Apr 17, 2026
651dd97
Merge branch 'main' into jdierksen/report-missing-junit
dierksen Apr 22, 2026
5f3b154
Treat no-result tests as warnings
dierksen Apr 23, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
186 changes: 134 additions & 52 deletions scripts/test_utils.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -40,8 +40,10 @@ PYTEST_FLAGS="--continue-on-collection-errors"

# Global variables for test execution
FAILED_TESTS=""
NO_RESULT_TESTS=""
TOTAL_TESTS=0
PASSED_TESTS=0
NO_RESULT_COUNT=0
TOTAL_TEST_CASES=0
SAMPLED_TEST_CASES=0
# shellcheck disable=SC2034 # EXIT_CODE is used by calling scripts
Expand Down Expand Up @@ -187,6 +189,49 @@ collect_tests() {
ALL_NODE_IDS=$(echo "$COLLECTION_OUTPUT" | grep "::" || true)
}

# Return the expected JUnit XML path for a test file
junit_file_for_test() {
local test_file=$1
local flattened_test_file=${test_file//\//_}
local test_hash
test_hash=$(printf '%s' "$test_file" | cksum | awk '{print $1}')
echo "${JUNIT_DIR}/${flattened_test_file}.${test_hash}.xml"
}

# Record a failed test file in the execution summary
record_failed_test() {
local test_file=$1
FAILED_TESTS="$FAILED_TESTS\n - $test_file"
# shellcheck disable=SC2034 # EXIT_CODE is used by calling scripts
EXIT_CODE=1
}

# Record a test file that produced no result artifacts
record_no_result_test() {
local test_file=$1
NO_RESULT_TESTS="$NO_RESULT_TESTS\n - $test_file"
NO_RESULT_COUNT=$((NO_RESULT_COUNT + 1))
# shellcheck disable=SC2034 # EXIT_CODE is used by calling scripts
EXIT_CODE=1
}
Comment thread
coderabbitai[bot] marked this conversation as resolved.

# Describe which execution artifacts are missing for a test file
describe_missing_artifacts() {
local result_file=$1
local junit_file=$2
local -a missing=()

if [ -n "$result_file" ] && [ ! -f "$result_file" ]; then
missing+=("result marker")
fi
if [ ! -f "$junit_file" ]; then
missing+=("JUnit XML: $junit_file")
fi

local IFS=', '
echo "${missing[*]}"
Comment thread
dierksen marked this conversation as resolved.
}

# Sample tests based on SAMPLE_RATE and SAMPLE_OFFSET
sample_tests() {
local all_node_ids=$1
Expand Down Expand Up @@ -240,8 +285,9 @@ dry_run_full_file() {
local test_file=$1

TOTAL_TESTS=$((TOTAL_TESTS + 1))
JUNIT_FILENAME="${test_file//\//_}.xml"
JUNIT_FLAG="--junitxml=${JUNIT_DIR}/${JUNIT_FILENAME}"
local junit_file
junit_file=$(junit_file_for_test "$test_file")
JUNIT_FLAG="--junitxml=${junit_file}"
# shellcheck disable=SC2086 # PYTEST_COMMAND_PREFIX needs word splitting
echo "$TOTAL_TESTS. ${PYTEST_COMMAND_PREFIX} pytest $PYTEST_FLAGS ${JUNIT_FLAG} \"${test_file}\""
}
Expand Down Expand Up @@ -289,6 +335,8 @@ print_dry_run_summary() {
run_sanity_test_file() {
local test_file=$1
local file_count=$2
local junit_file
junit_file=$(junit_file_for_test "$test_file")

echo "=========================================="
echo "[$file_count] Processing: $test_file"
Expand Down Expand Up @@ -328,21 +376,29 @@ run_sanity_test_file() {
# Create a bash array with the node IDs
mapfile -t SAMPLED_NODE_IDS_ARRAY <<< "$SAMPLED_NODE_IDS"

JUNIT_FILENAME="${test_file//\//_}.xml"
JUNIT_FLAG="--junitxml=${JUNIT_DIR}/${JUNIT_FILENAME}"
JUNIT_FLAG="--junitxml=${junit_file}"

# Run pytest with the sampled node IDs
TOTAL_TESTS=$((TOTAL_TESTS + 1))
rm -f "$junit_file"

# shellcheck disable=SC2086 # PYTEST_COMMAND_PREFIX and PYTEST_FLAGS need word splitting
if ${PYTEST_COMMAND_PREFIX} pytest $PYTEST_FLAGS "${JUNIT_FLAG}" "${SAMPLED_NODE_IDS_ARRAY[@]}"; then
echo "✅ PASSED: $test_file ($SAMPLED_IN_FILE/$TOTAL_IN_FILE tests)"
PASSED_TESTS=$((PASSED_TESTS + 1))
if [ -f "$junit_file" ]; then
echo "✅ PASSED: $test_file ($SAMPLED_IN_FILE/$TOTAL_IN_FILE tests)"
PASSED_TESTS=$((PASSED_TESTS + 1))
else
echo "⚠️ NO RESULT: $test_file ($SAMPLED_IN_FILE/$TOTAL_IN_FILE tests, missing JUnit XML: $junit_file)"
record_no_result_test "$test_file"
fi
else
echo "❌ FAILED: $test_file ($SAMPLED_IN_FILE/$TOTAL_IN_FILE tests)"
FAILED_TESTS="$FAILED_TESTS\n - $test_file"
# shellcheck disable=SC2034 # EXIT_CODE is used by calling scripts
EXIT_CODE=1
if [ -f "$junit_file" ]; then
echo "❌ FAILED: $test_file ($SAMPLED_IN_FILE/$TOTAL_IN_FILE tests)"
record_failed_test "$test_file"
else
echo "⚠️ NO RESULT: $test_file ($SAMPLED_IN_FILE/$TOTAL_IN_FILE tests, missing JUnit XML: $junit_file)"
record_no_result_test "$test_file"
fi
fi

echo ""
Expand All @@ -351,25 +407,35 @@ run_sanity_test_file() {
# Run a single test file in full mode
run_full_test_file() {
local test_file=$1
local junit_file
junit_file=$(junit_file_for_test "$test_file")

echo "=========================================="
JUNIT_FILENAME="${test_file//\//_}.xml"
JUNIT_FLAG="--junitxml=${JUNIT_DIR}/${JUNIT_FILENAME}"
JUNIT_FLAG="--junitxml=${junit_file}"
# shellcheck disable=SC2086 # PYTEST_COMMAND_PREFIX needs word splitting
echo "Running: ${PYTEST_COMMAND_PREFIX} pytest $PYTEST_FLAGS ${JUNIT_FLAG} \"${test_file}\""
echo "=========================================="

TOTAL_TESTS=$((TOTAL_TESTS + 1))
rm -f "$junit_file"

# shellcheck disable=SC2086 # PYTEST_COMMAND_PREFIX and PYTEST_FLAGS need word splitting
if ${PYTEST_COMMAND_PREFIX} pytest $PYTEST_FLAGS "${JUNIT_FLAG}" "${test_file}"; then
echo "✅ PASSED: $test_file"
PASSED_TESTS=$((PASSED_TESTS + 1))
if [ -f "$junit_file" ]; then
echo "✅ PASSED: $test_file"
PASSED_TESTS=$((PASSED_TESTS + 1))
else
echo "⚠️ NO RESULT: $test_file (missing JUnit XML: $junit_file)"
record_no_result_test "$test_file"
fi
else
echo "❌ FAILED: $test_file"
FAILED_TESTS="$FAILED_TESTS\n - $test_file"
# shellcheck disable=SC2034 # EXIT_CODE is used by calling scripts
EXIT_CODE=1
if [ -f "$junit_file" ]; then
echo "❌ FAILED: $test_file"
record_failed_test "$test_file"
else
echo "⚠️ NO RESULT: $test_file (missing JUnit XML: $junit_file)"
record_no_result_test "$test_file"
fi
fi

echo ""
Expand Down Expand Up @@ -475,13 +541,16 @@ run_tests_parallel() {
local file_index=$3
local result_file="$PARALLEL_TMP_DIR/result_${file_index}"
local log_file="$PARALLEL_TMP_DIR/log_${file_index}"
local junit_file
junit_file=$(junit_file_for_test "$test_file")

(
# Set GPU for this test
export CUDA_VISIBLE_DEVICES=$gpu_id

# Redirect output to log file
exec > "$log_file" 2>&1
rm -f "$junit_file"

echo "=========================================="
echo "[$file_index/$total_files] Processing: $test_file"
Expand Down Expand Up @@ -513,8 +582,7 @@ run_tests_parallel() {
fi

mapfile -t SAMPLED_NODE_IDS_ARRAY <<< "$SAMPLED_NODE_IDS"
JUNIT_FILENAME="${test_file//\//_}.xml"
JUNIT_FLAG="--junitxml=${JUNIT_DIR}/${JUNIT_FILENAME}"
JUNIT_FLAG="--junitxml=${junit_file}"

# shellcheck disable=SC2086
if ${PYTEST_COMMAND_PREFIX} pytest $PYTEST_FLAGS "${JUNIT_FLAG}" "${SAMPLED_NODE_IDS_ARRAY[@]}"; then
Expand All @@ -526,8 +594,7 @@ run_tests_parallel() {
fi
else
# Run full test
JUNIT_FILENAME="${test_file//\//_}.xml"
JUNIT_FLAG="--junitxml=${JUNIT_DIR}/${JUNIT_FILENAME}"
JUNIT_FLAG="--junitxml=${junit_file}"

# shellcheck disable=SC2086
if ${PYTEST_COMMAND_PREFIX} pytest $PYTEST_FLAGS "${JUNIT_FLAG}" "${test_file}"; then
Expand All @@ -541,7 +608,7 @@ run_tests_parallel() {
) &

local pid=$!
echo "$pid:$test_file:$result_file:$log_file:$file_index"
echo "$pid:$test_file:$result_file:$log_file:$file_index:$junit_file"
}

# Launch tests in parallel with GPU queue
Expand Down Expand Up @@ -582,9 +649,9 @@ run_tests_parallel() {
job_info=$(run_single_test_background "$test_file" "$gpu_id" "$file_index")

# Parse job info
local pid result_file log_file
IFS=':' read -r pid test_file result_file log_file file_index <<< "$job_info"
test_result_files[$pid]="$result_file:$test_file:$log_file:$file_index"
local pid result_file log_file junit_file
IFS=':' read -r pid test_file result_file log_file file_index junit_file <<< "$job_info"
test_result_files[$pid]="$result_file:$test_file:$log_file:$file_index:$junit_file"
test_pid_map[$pid]="$test_file"
test_gpu_map[$pid]="$gpu_id"

Expand All @@ -609,8 +676,8 @@ run_tests_parallel() {
# Sort results by file_index for deterministic output
local -a sorted_pids=()
for pid in "${!test_result_files[@]}"; do
local result_file test_file log_file file_index
IFS=':' read -r result_file test_file log_file file_index <<< "${test_result_files[$pid]}"
local result_file test_file log_file file_index junit_file
IFS=':' read -r result_file test_file log_file file_index junit_file <<< "${test_result_files[$pid]}"
sorted_pids+=("$file_index:$pid")
done
local sorted_list
Expand All @@ -620,8 +687,8 @@ run_tests_parallel() {
# Process results in order
for entry in "${sorted_pids[@]}"; do
local pid="${entry#*:}"
local result_file test_file log_file file_index
IFS=':' read -r result_file test_file log_file file_index <<< "${test_result_files[$pid]}"
local result_file test_file log_file file_index junit_file
IFS=':' read -r result_file test_file log_file file_index junit_file <<< "${test_result_files[$pid]}"

# Show log output
if [ -f "$log_file" ]; then
Expand All @@ -633,45 +700,52 @@ run_tests_parallel() {
if [ -f "$result_file" ]; then
local result
result=$(cat "$result_file")
if [[ "$result" == SKIPPED* ]]; then
continue
fi

TOTAL_TESTS=$((TOTAL_TESTS + 1))

if [ "$mode" = "sanity" ] && [[ "$result" == PASSED:* || "$result" == FAILED:* ]]; then
local total_in_file sampled_in_file
# shellcheck disable=SC2034 # status is part of the read but unused
IFS=':' read -r _ total_in_file sampled_in_file <<< "$result"
TOTAL_TEST_CASES=$((TOTAL_TEST_CASES + total_in_file))
SAMPLED_TEST_CASES=$((SAMPLED_TEST_CASES + sampled_in_file))
fi

if [ ! -f "$junit_file" ]; then
echo "⚠️ NO RESULT: $test_file (missing JUnit XML: $junit_file)"
record_no_result_test "$test_file"
continue
fi

if [[ "$result" == PASSED* ]]; then
PASSED_TESTS=$((PASSED_TESTS + 1))
if [ "$mode" = "sanity" ]; then
local total_in_file sampled_in_file
# shellcheck disable=SC2034 # status is part of the read but unused
IFS=':' read -r _ total_in_file sampled_in_file <<< "$result"
TOTAL_TEST_CASES=$((TOTAL_TEST_CASES + total_in_file))
SAMPLED_TEST_CASES=$((SAMPLED_TEST_CASES + sampled_in_file))
fi
elif [[ "$result" == FAILED* ]]; then
FAILED_TESTS="$FAILED_TESTS\n - $test_file"
# shellcheck disable=SC2034 # EXIT_CODE is used by calling scripts
EXIT_CODE=1
if [ "$mode" = "sanity" ]; then
local total_in_file sampled_in_file
# shellcheck disable=SC2034 # status is part of the read but unused
IFS=':' read -r _ total_in_file sampled_in_file <<< "$result"
TOTAL_TEST_CASES=$((TOTAL_TEST_CASES + total_in_file))
SAMPLED_TEST_CASES=$((SAMPLED_TEST_CASES + sampled_in_file))
fi
elif [[ "$result" == SKIPPED* ]]; then
# Don't count skipped tests as passed
TOTAL_TESTS=$((TOTAL_TESTS - 1))
record_failed_test "$test_file"
fi
else
TOTAL_TESTS=$((TOTAL_TESTS + 1))
local missing_artifacts
missing_artifacts=$(describe_missing_artifacts "$result_file" "$junit_file")
echo "⚠️ NO RESULT: $test_file (missing ${missing_artifacts})"
record_no_result_test "$test_file"
fi
done
}

# Print execution summary
print_execution_summary() {
if [ "$SANITY_TEST" == "true" ]; then
local failed_count=$((TOTAL_TESTS - PASSED_TESTS - NO_RESULT_COUNT))
Comment thread
dierksen marked this conversation as resolved.
echo "=========================================="
echo "SANITY TEST SUMMARY"
echo "=========================================="
echo "Total test files executed: $TOTAL_TESTS"
echo "Test files passed: $PASSED_TESTS"
echo "Test files failed: $((TOTAL_TESTS - PASSED_TESTS))"
echo "Test files failed: $failed_count"
echo "Test files with no result: $NO_RESULT_COUNT"
echo ""
echo "Total test cases (full suite): $TOTAL_TEST_CASES"
echo "Sampled test cases (executed): $SAMPLED_TEST_CASES"
Expand All @@ -685,19 +759,27 @@ print_execution_summary() {
echo "To reproduce this exact run:"
echo " SAMPLE_RATE=${SAMPLE_RATE} SAMPLE_OFFSET=${SAMPLE_OFFSET} $0 --sanity-test"
else
local failed_count=$((TOTAL_TESTS - PASSED_TESTS - NO_RESULT_COUNT))
echo "=========================================="
echo "TEST SUMMARY"
echo "=========================================="
echo "Total test files executed: $TOTAL_TESTS"
echo "Passed: $PASSED_TESTS"
echo "Failed: $((TOTAL_TESTS - PASSED_TESTS))"
echo "Failed: $failed_count"
echo "No result: $NO_RESULT_COUNT"
Comment thread
dierksen marked this conversation as resolved.
fi

if [ -n "$FAILED_TESTS" ]; then
echo ""
echo "Failed test files:"
echo -e "$FAILED_TESTS"
fi

if [ -n "$NO_RESULT_TESTS" ]; then
echo ""
echo "Test files with no result:"
echo -e "$NO_RESULT_TESTS"
fi
}

# Main execution function for dry run mode
Expand Down
Loading