Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 5 additions & 4 deletions .github/actions/test-failure-detection/action.yml
Original file line number Diff line number Diff line change
Expand Up @@ -122,20 +122,21 @@ runs:
# Fast single-pass JSON extraction for test failures (< 1s for 10K lines)
# Filter JSON lines and parse in one pass - eliminates 2-minute hang
# Note: Line numbers aren't available in Go test JSON output
# IMPORTANT: Only detect actual test function failures, not package/suite completion events
grep '^{' "$json_file" 2>/dev/null | \
jq -r 'select(.Action == "fail") |
"--- FAIL: \(if (.Test and .Test != null and .Test != "null") then .Test else "unknown" end) (\(.Package))"' \
jq -r 'select(.Action == "fail" and .Test != null and .Test != "" and .Test != "null" and (.Test | test("^Test[A-Za-z]"))) |
"--- FAIL: \(.Test) (\(.Package))"' \
2>/dev/null > "$failures_file"

# Create structured test failure entries with unique signatures
if grep '^{' "$json_file" 2>/dev/null | jq -r 'select(.Action == "fail")' 2>/dev/null | head -1 | grep -q .; then
if grep '^{' "$json_file" 2>/dev/null | jq -r 'select(.Action == "fail" and .Test != null and .Test != "" and .Test != "null" and (.Test | test("^Test[A-Za-z]")))' 2>/dev/null | head -1 | grep -q .; then
echo "📝 Creating structured test failure entries with enhanced output..."

# First pass: Extract test failure basic info from Action == "fail" entries
local temp_failures
temp_failures=$(mktemp)
grep '^{' "$json_file" 2>/dev/null | \
jq -r 'select(.Action == "fail") | {
jq -r 'select(.Action == "fail" and .Test != null and .Test != "" and .Test != "null" and (.Test | test("^Test[A-Za-z]"))) | {
type: "test",
package: .Package,
test: (if (.Test and .Test != null and .Test != "null") then .Test else "unknown" end),
Expand Down
15 changes: 9 additions & 6 deletions .github/workflows/fortress-test-matrix.yml
Original file line number Diff line number Diff line change
Expand Up @@ -254,21 +254,24 @@ jobs:
source test-failure-functions.sh 2>/dev/null || true
detect_failures_from_json "test-output.log" "test-failures.txt" || true

# JSON failure override: Check if JSON contains failures even when exit code is 0
# JSON failure override: Check if JSON contains ACTUAL test failures when exit code is 0
if [[ $TEST_EXIT_CODE -eq 0 ]] && [[ -f test-output.log ]]; then
echo "🔍 Checking JSON output for failures despite exit code 0..."
echo "🔍 Checking JSON output for actual test failures despite exit code 0..."

# Check for JSON test failure entries or error patterns
JSON_FAILURES=0
if grep -q '^{' test-output.log 2>/dev/null; then
# Count JSON failure entries (only actual test failures, not log messages)
# Count ONLY failed test events, not package/suite failures which are expected
# Look for Action="fail" AND Package field AND Test field (indicating actual test failure)
JSON_FAILURES=$(grep '^{' test-output.log 2>/dev/null | \
jq -r 'select(.Action == "fail") | .Test' 2>/dev/null | wc -l | xargs || echo "0")
jq -r 'select(.Action == "fail" and .Test != null and .Test != "" and (.Package // "") != "" and (.Test | test("^Test[A-Z]"))) | .Test' 2>/dev/null | wc -l | xargs || echo "0")

if [[ $JSON_FAILURES -gt 0 ]]; then
echo "⚠️ Found $JSON_FAILURES actual test failures in JSON output"
echo "🔧 Overriding exit code from 0 to 1 due to detected failures"
echo "⚠️ Found $JSON_FAILURES actual failing test functions in JSON output"
echo "🔧 Overriding exit code from 0 to 1 due to detected test failures"
TEST_EXIT_CODE=1
else
echo "✅ JSON output contains no actual test failures (exit code 0 is correct)"
fi
fi
fi
Expand Down
6 changes: 3 additions & 3 deletions .github/workflows/fortress-test-validation.yml
Original file line number Diff line number Diff line change
Expand Up @@ -297,8 +297,8 @@ jobs:
# Count test failures - handle both regular test format and fuzz test format
LEGACY_TEST_COUNT=$(jq '[.[] | select(.failures) | .failures[]] | length' "$summary_file" 2>/dev/null || echo "0")

# Also count entries that are fuzz tests (Type == "test" and no .failures field)
FUZZ_TEST_COUNT=$(jq '[.[] | select(.Type == "test" and (.failures | length) > 0)] | length' "$summary_file" 2>/dev/null || echo "0")
# Also count entries that are fuzz tests (Type == "test" and has .failures field with content)
FUZZ_TEST_COUNT=$(jq '[.[] | select(.Type == "test" and (.failures // [] | length > 0))] | length' "$summary_file" 2>/dev/null || echo "0")

# Count build failures
BUILD_COUNT=$(jq '[.[] | select(.Type == "build")] | length' "$summary_file" 2>/dev/null || echo "0")
Expand Down Expand Up @@ -370,7 +370,7 @@ jobs:

# Count all failure types
RECOUNT_TEST=$(jq '[.[] | select(.failures) | .failures[]] | length' "$summary_file" 2>/dev/null || echo "0")
RECOUNT_FUZZ=$(jq '[.[] | select(.Type == "test" and (.failures | length) > 0)] | length' "$summary_file" 2>/dev/null || echo "0")
RECOUNT_FUZZ=$(jq '[.[] | select(.Type == "test" and (.failures // [] | length > 0))] | length' "$summary_file" 2>/dev/null || echo "0")
RECOUNT_BUILD=$(jq '[.[] | select(.Type == "build")] | length' "$summary_file" 2>/dev/null || echo "0")

FILE_FAILURES=$((RECOUNT_TEST + RECOUNT_FUZZ + RECOUNT_BUILD))
Expand Down