feat(ci): add coverage regression detection (#244)

* Initial plan

* feat(ci): add coverage regression detection

Co-authored-by: Mossaka <5447827+Mossaka@users.noreply.github.com>

---------

Co-authored-by: copilot-swe-agent[bot] <198982749+Copilot@users.noreply.github.com>
Co-authored-by: Mossaka <5447827+Mossaka@users.noreply.github.com>

Author: Copilot

.github/workflows/test-coverage.yml

@@ -17,11 +17,13 @@ jobs:
   coverage:
     name: Test Coverage Report
     runs-on: ubuntu-latest
-    timeout-minutes: 10
+    timeout-minutes: 15
 
     steps:
       - name: Checkout repository
         uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4
+        with:
+          fetch-depth: 0
 
       - name: Setup Node.js
         uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4
@@ -35,21 +37,65 @@ jobs:
       - name: Build project
         run: npm run build
 
-      - name: Run tests with coverage
+      - name: Run tests with coverage (PR branch)
         run: npm run test:coverage
 
-      - name: Generate coverage summary
+      - name: Save PR coverage
+        run: cp coverage/coverage-summary.json /tmp/pr-coverage-summary.json
+
+      - name: Get base branch coverage (PR only)
+        if: github.event_name == 'pull_request'
+        id: base_coverage
+        run: |
+          # Save the current commit
+          PR_COMMIT=$(git rev-parse HEAD)
+
+          # Checkout base branch
+          git checkout ${{ github.event.pull_request.base.sha }}
+
+          # Install dependencies and build for base branch
+          npm ci
+          npm run build
+
+          # Run coverage on base branch
+          npm run test:coverage || true
+
+          # Save base coverage
+          if [ -f coverage/coverage-summary.json ]; then
+            cp coverage/coverage-summary.json /tmp/base-coverage-summary.json
+            echo "base_coverage_exists=true" >> $GITHUB_OUTPUT
+          else
+            echo "base_coverage_exists=false" >> $GITHUB_OUTPUT
+          fi
+
+          # Checkout back to PR commit
+          git checkout $PR_COMMIT
+
+          # Reinstall PR dependencies
+          npm ci
+
+      - name: Compare coverage (PR only)
+        if: github.event_name == 'pull_request' && steps.base_coverage.outputs.base_coverage_exists == 'true'
+        id: compare
+        run: |
+          npx tsx scripts/ci/compare-coverage.ts \
+            /tmp/pr-coverage-summary.json \
+            /tmp/base-coverage-summary.json
+        continue-on-error: true
+
+      - name: Generate coverage summary (push to main)
+        if: github.event_name == 'push'
         id: coverage
         run: |
           # Read the coverage summary
           COVERAGE_JSON=$(cat coverage/coverage-summary.json)
-          
+
           # Extract metrics using jq
           LINES_PCT=$(echo "$COVERAGE_JSON" | jq -r '.total.lines.pct')
           STATEMENTS_PCT=$(echo "$COVERAGE_JSON" | jq -r '.total.statements.pct')
           FUNCTIONS_PCT=$(echo "$COVERAGE_JSON" | jq -r '.total.functions.pct')
           BRANCHES_PCT=$(echo "$COVERAGE_JSON" | jq -r '.total.branches.pct')
-          
+
           LINES_COVERED=$(echo "$COVERAGE_JSON" | jq -r '.total.lines.covered')
           LINES_TOTAL=$(echo "$COVERAGE_JSON" | jq -r '.total.lines.total')
           STATEMENTS_COVERED=$(echo "$COVERAGE_JSON" | jq -r '.total.statements.covered')
@@ -58,7 +104,7 @@ jobs:
           FUNCTIONS_TOTAL=$(echo "$COVERAGE_JSON" | jq -r '.total.functions.total')
           BRANCHES_COVERED=$(echo "$COVERAGE_JSON" | jq -r '.total.branches.covered')
           BRANCHES_TOTAL=$(echo "$COVERAGE_JSON" | jq -r '.total.branches.total')
-          
+
           # Create summary for GitHub Actions Summary
           echo "## Test Coverage Report" >> $GITHUB_STEP_SUMMARY
           echo "" >> $GITHUB_STEP_SUMMARY
@@ -69,62 +115,56 @@ jobs:
           echo "| **Functions** | ${FUNCTIONS_PCT}% | ${FUNCTIONS_COVERED}/${FUNCTIONS_TOTAL} |" >> $GITHUB_STEP_SUMMARY
           echo "| **Branches** | ${BRANCHES_PCT}% | ${BRANCHES_COVERED}/${BRANCHES_TOTAL} |" >> $GITHUB_STEP_SUMMARY
           echo "" >> $GITHUB_STEP_SUMMARY
-          
-          # Create PR comment body
-          COMMENT_BODY="## Test Coverage Report
-
-          | Metric | Coverage | Covered/Total |
-          |--------|----------|---------------|
-          | **Lines** | ${LINES_PCT}% | ${LINES_COVERED}/${LINES_TOTAL} |
-          | **Statements** | ${STATEMENTS_PCT}% | ${STATEMENTS_COVERED}/${STATEMENTS_TOTAL} |
-          | **Functions** | ${FUNCTIONS_PCT}% | ${FUNCTIONS_COVERED}/${FUNCTIONS_TOTAL} |
-          | **Branches** | ${BRANCHES_PCT}% | ${BRANCHES_COVERED}/${BRANCHES_TOTAL} |
-
-          <details>
-          <summary>Coverage Thresholds</summary>
-
-          The project has the following coverage thresholds configured:
-          - Lines: 38%
-          - Statements: 38%
-          - Functions: 35%
-          - Branches: 30%
-
-          </details>
-
-          ---
-          *Coverage report generated by \\\`npm run test:coverage\\\`*"
-          
-          # Save for next step (escape newlines for GitHub Actions)
-          echo "COMMENT_BODY<<EOF" >> $GITHUB_ENV
-          echo "$COMMENT_BODY" >> $GITHUB_ENV
-          echo "EOF" >> $GITHUB_ENV
-          
+
           # Also save individual metrics as outputs
           echo "lines_pct=${LINES_PCT}" >> $GITHUB_OUTPUT
           echo "statements_pct=${STATEMENTS_PCT}" >> $GITHUB_OUTPUT
           echo "functions_pct=${FUNCTIONS_PCT}" >> $GITHUB_OUTPUT
           echo "branches_pct=${BRANCHES_PCT}" >> $GITHUB_OUTPUT
 
-      - name: Comment PR with coverage report
+      - name: Comment PR with coverage comparison
         if: github.event_name == 'pull_request'
         uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7
         with:
           github-token: ${{ secrets.GITHUB_TOKEN }}
           script: |
-            const commentBody = process.env.COMMENT_BODY;
-            
+            const fs = require('fs');
+
+            // Try to read the coverage report from compare step
+            let commentBody = process.env.COVERAGE_REPORT;
+
+            // If no comparison report, generate a simple report
+            if (!commentBody) {
+              const prCoverage = JSON.parse(fs.readFileSync('/tmp/pr-coverage-summary.json', 'utf8'));
+              const total = prCoverage.total;
+
+              commentBody = `## 📊 Test Coverage Report
+
+            | Metric | Coverage |
+            |--------|----------|
+            | Lines | ${total.lines.pct.toFixed(2)}% |
+            | Statements | ${total.statements.pct.toFixed(2)}% |
+            | Functions | ${total.functions.pct.toFixed(2)}% |
+            | Branches | ${total.branches.pct.toFixed(2)}% |
+
+            > ℹ️ Base branch coverage not available for comparison.
+
+            ---
+            *Coverage report generated by \`npm run test:coverage\`*`;
+            }
+
             // Find existing coverage comment
             const { data: comments } = await github.rest.issues.listComments({
               owner: context.repo.owner,
               repo: context.repo.repo,
               issue_number: context.issue.number,
             });
-            
-            const botComment = comments.find(comment => 
-              comment.user.type === 'Bot' && 
-              comment.body.includes('Test Coverage Report')
+
+            const botComment = comments.find(comment =>
+              comment.user.type === 'Bot' &&
+              (comment.body.includes('Test Coverage Report') || comment.body.includes('Coverage Check'))
             );
-            
+
             if (botComment) {
               // Update existing comment
               await github.rest.issues.updateComment({
@@ -151,8 +191,11 @@ jobs:
             coverage/
           retention-days: 30
 
-      - name: Check coverage thresholds
+      - name: Fail on coverage regression
+        if: github.event_name == 'pull_request' && steps.compare.outcome == 'failure'
         run: |
-          echo "Checking if coverage meets minimum thresholds..."
-          # Jest will fail if coverage is below thresholds defined in jest.config.js
-          # This step is informational since the test:coverage command already checks
+          echo "❌ Coverage regression detected!"
+          echo "This PR decreases overall test coverage. Please add tests to maintain coverage levels."
+          echo ""
+          echo "See the PR comment above for detailed coverage comparison."
+          exit 1