Refactor: Coverage Script Cleanup & Auto-Ratchet

.coverage-threshold

@@ -0,0 +1 @@
+64.2

.github/workflows/unit-test-and-coverage-gate.yml

@@ -1,227 +1,200 @@
-name: Unit and Coverage 
+name: Unit Tests and Coverage
 
 on:
   pull_request:
-    branches: [ main ]          # Gate PRs into main
+    branches: [main]
   push:
-    branches: [ main ]          # also run on direct pushes
-  workflow_dispatch:             # Manual runs
+    branches: [main]  # Only on main to avoid duplicate runs with pull_request
+  workflow_dispatch:
     inputs:
       ref:
-        description: "Branch or SHA to test (e.g. feature/x or a1b2c3)"
+        description: "Branch or SHA to test"
         required: false
       cov_threshold:
-        description: "Override threshold (percent) for this manual run only"
+        description: "Override coverage threshold (%)"
         required: false
 
 concurrency:
-  group: earthly-tests-${{ github.ref }}
+  group: unit-tests-${{ github.ref }}
   cancel-in-progress: true
 
 permissions:
-  contents: read
+  contents: write  # Needed to push threshold updates
 
 jobs:
-  run-earthly-tests:
-    name: Run earthly +test (coverage gate)
+  test:
+    name: Unit Tests & Coverage Gate
     runs-on: ubuntu-latest
     timeout-minutes: 30
 
     steps:
-      - name: Checkout repository
+      - name: Checkout
         uses: actions/checkout@v4
         with:
           fetch-depth: 0
           persist-credentials: false
-          # Use input ref if provided; else PR head SHA; else current SHA
-          ref: ${{ inputs.ref != '' && inputs.ref || (github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha) }}
+          ref: ${{ inputs.ref || github.event.pull_request.head.sha || github.sha }}
 
       - name: Setup Earthly
         uses: earthly/actions/setup-earthly@v1
         with:
           version: "latest"
 
-      - name: Show Earthly version
-        run: earthly --version
-
-      - name: Resolve coverage threshold
-        id: threshold
+      - name: Configure test parameters
+        id: config
         env:
-          MANUAL_COV: ${{ inputs.cov_threshold }}
+          INPUT_COV_THRESHOLD: ${{ inputs.cov_threshold }}
         run: |
-          set -euo pipefail
-
-          # Default threshold (matches script default)
-          DEFAULT="12.3"
-
-          # Use manual override if provided, otherwise use default
-          if [[ -n "${MANUAL_COV:-}" ]]; then
-            HEAD_VAL="${MANUAL_COV}"
-            echo "Using manual threshold override: ${HEAD_VAL}%"
+          # Read threshold from file, allow manual override
+          FILE_THRESHOLD=$(cat .coverage-threshold 2>/dev/null || echo "65.0")
+          COV_THRESHOLD="${INPUT_COV_THRESHOLD:-$FILE_THRESHOLD}"
+          echo "cov_threshold=${COV_THRESHOLD}" >> "$GITHUB_OUTPUT"
+          echo "build_id=${GITHUB_RUN_ID}" >> "$GITHUB_OUTPUT"
+          if [[ -n "${INPUT_COV_THRESHOLD}" ]]; then
+            echo "::notice::Coverage threshold: ${COV_THRESHOLD}% (from manual override)"
           else
-            HEAD_VAL="$DEFAULT"
-            echo "Using default threshold: ${HEAD_VAL}%"
+            echo "::notice::Coverage threshold: ${COV_THRESHOLD}% (from .coverage-threshold file)"
           fi
 
-          # Set environment variables for subsequent steps
-          echo "COV_THRESHOLD=${HEAD_VAL}" >> "$GITHUB_ENV"
-          echo "PRINT_TS=${GITHUB_RUN_ID}" >> "$GITHUB_ENV"
-          echo "FAIL_ON_NO_TESTS=false" >> "$GITHUB_ENV"
-
-          echo "Resolved threshold: ${HEAD_VAL}%"
-          echo "Build ID: ${GITHUB_RUN_ID}"
-
-      # Run standard tests with coverage
-      - name: Run Earthly +test with coverage threshold
+      - name: Run tests with coverage
+        id: test
+        env:
+          COV_THRESHOLD: ${{ steps.config.outputs.cov_threshold }}
+          BUILD_ID: ${{ steps.config.outputs.build_id }}
         run: |
-          earthly +test --COV_THRESHOLD="${COV_THRESHOLD}" --PRINT_TS="${PRINT_TS}" --FAIL_ON_NO_TESTS="${FAIL_ON_NO_TESTS}"
+          earthly +test \
+            --COV_THRESHOLD="${COV_THRESHOLD}" \
+            --PRINT_TS="${BUILD_ID}" \
+            --FAIL_ON_NO_TESTS="false"
 
-      # Upload main coverage artifacts (always generated by script)
-      - name: Upload main coverage artifacts
+      - name: Upload coverage artifacts
         if: always()
         uses: actions/upload-artifact@v4
         with:
-          name: coverage-reports
+          name: coverage-${{ github.run_id }}
           path: |
             coverage.out
-            coverage_total.txt
-            coverage_packages.txt
-            test_raw.log
-
-      # Upload detailed per-directory artifacts for debugging
-      - name: Upload per-directory coverage artifacts
-        if: always()
-        uses: actions/upload-artifact@v4
-        with:
-          name: per-directory-coverage
-          path: |
-            *_coverage.out
-            *_test.log
-            overall_coverage.out
-            combined_coverage.out
-            overall_test.log
-            overall_test_with_failures.log
-          if-no-files-found: ignore
-
-      # Legacy individual uploads for backward compatibility
-      - name: Upload coverage.out (legacy)
-        if: always()
-        uses: actions/upload-artifact@v4
-        with:
-          name: coverage.out
-          path: coverage.out
-          if-no-files-found: warn
-
-      - name: Upload coverage_total.txt (legacy)
-        if: always()
-        uses: actions/upload-artifact@v4
-        with:
-          name: coverage_total.txt
-          path: coverage_total.txt
-          if-no-files-found: warn
+            coverage_report.txt
+          retention-days: 30
 
-      - name: Upload coverage_packages.txt (legacy)
-        if: always()
-        uses: actions/upload-artifact@v4
-        with:
-          name: coverage_packages.txt
-          path: coverage_packages.txt
-          if-no-files-found: warn
-
-      - name: Upload test_raw.log (legacy)
-        if: always()
-        uses: actions/upload-artifact@v4
-        with:
-          name: test_raw.log
-          path: test_raw.log
-          if-no-files-found: warn
-
-      - name: Publish coverage summary
+      - name: Generate coverage summary
         if: always()
         run: |
           {
-            echo "## Coverage Summary"
-            if [[ -f coverage_total.txt ]]; then
-              echo ""
-              echo '```'
-              cat coverage_total.txt
-              echo '```'
-            fi
-            if [[ -f coverage_packages.txt ]]; then
-              echo ""
-              echo "Packages by coverage (lowest first):"
-              echo '```'
-              head -n 50 coverage_packages.txt || true
-              echo '```'
-            fi
+            echo "## 📊 Test Coverage Report"
             echo ""
-            echo "**Threshold used:** ${COV_THRESHOLD}%"
-            echo "**Build ID:** ${PRINT_TS}"
-            echo "**Test method:** Per-directory coverage with script-based execution"
 
-            # Add directory-level summary if available
-            if [[ -f coverage_total.txt ]] && grep -q "| Directory" coverage_total.txt; then
+            # Extract overall coverage
+            if [[ -f coverage_report.txt ]]; then
+              # Extract numeric values using simpler patterns
+              OVERALL=$(grep "Overall Coverage:" coverage_report.txt | sed 's/[^0-9.]*\([0-9.]\+\)%.*/\1/')%
+              THRESHOLD=$(grep "Threshold:" coverage_report.txt | sed 's/[^0-9.]*\([0-9.]\+\)%.*/\1/')%
+              STATUS=$(grep "Status:" coverage_report.txt | sed 's/[^A-Z]*\([A-Z]\+\).*/\1/')
+
+              # Status badge
+              if [[ "$STATUS" == "PASSED" ]]; then
+                echo "| Metric | Value | Status |"
+                echo "|--------|-------|--------|"
+                echo "| **Overall Coverage** | $OVERALL | ✅ PASSED |"
+                echo "| **Threshold** | $THRESHOLD | - |"
+                echo "| **Build** | #${{ github.run_id }} | - |"
+              else
+                echo "| Metric | Value | Status |"
+                echo "|--------|-------|--------|"
+                echo "| **Overall Coverage** | $OVERALL | ❌ FAILED |"
+                echo "| **Threshold** | $THRESHOLD | - |"
+                echo "| **Build** | #${{ github.run_id }} | - |"
+              fi
               echo ""
-              echo "### Directory Test Results"
-              echo '```'
-              grep -A 100 "| Directory" coverage_total.txt | head -n 50 || true
-              echo '```'
+
+              # Failed tests section
+              if grep -q "Failed Tests:" coverage_report.txt; then
+                echo "### ❌ Failed Tests"
+                echo ""
+                sed -n '/\*\*Failed Tests:\*\*/,/^\*\*Note/p' coverage_report.txt | grep "•" | head -20
+                echo ""
+              fi
+
+              # Directory results table
+              if grep -q "| Directory" coverage_report.txt; then
+                echo "### 📁 Directory Results"
+                echo ""
+                sed -n '/| Directory/,/^$/p' coverage_report.txt | head -50
+                echo ""
+              fi
+            else
+              echo "⚠️ Coverage report not generated"
+              echo ""
+              echo "Check the workflow logs for details."
             fi
           } >> "$GITHUB_STEP_SUMMARY"
 
-  # Debug job for troubleshooting (runs on manual trigger with failures)
-  run-earthly-tests-debug:
-    name: Run earthly +test-debug (enhanced debugging)
-    runs-on: ubuntu-latest
-    timeout-minutes: 45
-    if: failure() && github.event_name == 'workflow_dispatch'
-    needs: run-earthly-tests
-
-    steps:
-      - name: Checkout repository
-        uses: actions/checkout@v4
-        with:
-          fetch-depth: 0
-          persist-credentials: false
-          ref: ${{ inputs.ref != '' && inputs.ref || (github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha) }}
-
-      - name: Setup Earthly
-        uses: earthly/actions/setup-earthly@v1
-        with:
-          version: "latest"
-
-      - name: Run Earthly +test-debug with enhanced output
+      - name: Auto-update coverage threshold (ratchet)
+        if: success() && github.ref != 'refs/heads/main'
         env:
-          COV_THRESHOLD: ${{ inputs.cov_threshold || '12.3' }}
-          PRINT_TS: ${{ github.run_id }}
-          FAIL_ON_NO_TESTS: "false"
-        run: |
-          earthly +test-debug --COV_THRESHOLD="${COV_THRESHOLD}" --PRINT_TS="${PRINT_TS}" --FAIL_ON_NO_TESTS="${FAIL_ON_NO_TESTS}"
-
-      - name: Upload all debug artifacts
-        if: always()
-        uses: actions/upload-artifact@v4
-        with:
-          name: debug-coverage-artifacts
-          path: |
-            coverage.out
-            coverage_total.txt
-            coverage_packages.txt
-            test_raw.log
-            *_coverage.out
-            *_test.log
-            overall_coverage.out
-            combined_coverage.out
-            overall_test.log
-            overall_test_with_failures.log
-          if-no-files-found: ignore
-
-      - name: Show debug summary
-        if: always()
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          EVENT_NAME: ${{ github.event_name }}
+          HEAD_REF: ${{ github.head_ref }}
+          REF_NAME: ${{ github.ref_name }}
         run: |
-          echo "=== Debug Coverage Analysis ==="
-          echo "Files generated:"
-          ls -la *.out *.txt *.log || true
-          echo ""
-          echo "Directory structure:"
-          find . -name "*coverage*" -o -name "*test*.log" | head -20 || true
+          set -x  # Debug: show commands
+
+          # Determine branch name based on event type
+          if [[ "${EVENT_NAME}" == "pull_request" ]]; then
+            BRANCH="${HEAD_REF}"
+          else
+            # For workflow_dispatch or other events, use ref_name
+            BRANCH="${REF_NAME}"
+          fi
+
+          echo "Target branch: ${BRANCH}"
+
+          # Skip if on main branch (shouldn't happen due to condition, but safety check)
+          if [[ "$BRANCH" == "main" ]]; then
+            echo "Skipping ratchet on main branch"
+            exit 0
+          fi
+
+          # Get current coverage from report using simpler pattern
+          CURRENT=$(grep "Overall Coverage:" coverage_report.txt | sed 's/[^0-9.]*\([0-9.]\+\)%.*/\1/')
+          OLD_THRESHOLD=$(cat .coverage-threshold 2>/dev/null || echo "0")
+
+          # Validate CURRENT is a valid number before proceeding
+          if [[ -z "$CURRENT" ]] || ! [[ "$CURRENT" =~ ^[0-9]+\.?[0-9]*$ ]]; then
+            echo "::error::Failed to parse coverage value from report (got: '$CURRENT')"
+            exit 1
+          fi
+
+          echo "Current coverage: ${CURRENT}%"
+          echo "Current threshold: ${OLD_THRESHOLD}%"
+
+          # Calculate new threshold (0.5% buffer below actual coverage)
+          # Use printf to ensure consistent one-decimal-place formatting
+          NEW_THRESHOLD=$(printf '%.1f' "$(echo "$CURRENT - 0.5" | bc -l)")
+
+          echo "Proposed new threshold: ${NEW_THRESHOLD}%"
+
+          # Only update if new threshold is higher than old threshold
+          if (( $(echo "$NEW_THRESHOLD > $OLD_THRESHOLD" | bc -l) )); then
+            echo "Updating threshold: ${OLD_THRESHOLD}% -> ${NEW_THRESHOLD}%"
+            echo "${NEW_THRESHOLD}" > .coverage-threshold
+
+            # Configure git
+            git config user.name "github-actions[bot]"
+            git config user.email "github-actions[bot]@users.noreply.github.com"
+
+            # Fetch and checkout the branch
+            git fetch origin "${BRANCH}"
+            git checkout "${BRANCH}"
+            # Ensure branch is up to date in case it changed after the workflow started
+            git pull --rebase origin "${BRANCH}"
+
+            # Stage and commit
+            git add .coverage-threshold
+            git commit -m "chore: auto-update coverage threshold to ${NEW_THRESHOLD}% (was ${OLD_THRESHOLD}%)"
+            git push origin "${BRANCH}"
+
+            echo "::notice::Coverage threshold updated to ${NEW_THRESHOLD}% on branch ${BRANCH}"
+          else
+            echo "New threshold (${NEW_THRESHOLD}%) is not higher than current (${OLD_THRESHOLD}%), no update needed"
+          fi