Trigger CI checks on manual review (#71)

derekmisler · web-flow · commit 4f28e4bf4cad · 2026-03-06T17:14:49.000+01:00
Signed-off-by: Derek Misler &lt;derek.misler@docker.com&gt;
diff --git a/.github/workflows/pr-describe.yml b/.github/workflows/pr-describe.yml
@@ -18,10 +18,33 @@ jobs:
       contents: read
       pull-requests: write
       issues: write
+      checks: write
     steps:
       - name: Check out Git repository
         uses: actions/checkout@d632683dd7b4114ad314bca15554477dd762a938 # v4.2.0
 
+      - name: Create check run
+        id: create-check
+        continue-on-error: true # Don't fail if checks: write permission is missing
+        uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8
+        with:
+          script: |
+            const prNumber = context.issue.number;
+            const { data: pr } = await github.rest.pulls.get({
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              pull_number: prNumber
+            });
+            const { data: check } = await github.rest.checks.create({
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              name: 'Generate PR Description',
+              head_sha: pr.head.sha,
+              status: 'in_progress',
+              started_at: new Date().toISOString()
+            });
+            core.setOutput('check-id', check.id);
+
       # Generate GitHub App token so actions appear as the custom app (optional - falls back to github.token)
       - name: Get GitHub App token
         id: app-token
@@ -284,3 +307,25 @@ jobs:
           set -euo pipefail
           rm -f commits.txt files.txt pr.diff || true
           echo "🧹 Cleaned up temporary files"
+
+      - name: Update check run
+        if: always() && steps.create-check.outputs.check-id != ''
+        uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8
+        env:
+          CHECK_ID: ${{ steps.create-check.outputs.check-id }}
+          JOB_STATUS: ${{ job.status }}
+        with:
+          script: |
+            const conclusion = process.env.JOB_STATUS === 'cancelled' ? 'cancelled' : process.env.JOB_STATUS === 'success' ? 'success' : 'failure';
+            try {
+              await github.rest.checks.update({
+                owner: context.repo.owner,
+                repo: context.repo.repo,
+                check_run_id: parseInt(process.env.CHECK_ID, 10),
+                status: 'completed',
+                conclusion: conclusion,
+                completed_at: new Date().toISOString()
+              });
+            } catch (error) {
+              core.warning(`Failed to update check run: ${error.message}`);
+            }
diff --git a/.github/workflows/review-pr.yml b/.github/workflows/review-pr.yml
@@ -18,6 +18,7 @@
 #         contents: read       # Read repository files and PR diffs
 #         pull-requests: write # Post review comments and approve/request changes
 #         issues: write        # Create security incident issues if secrets are detected in output
+#         checks: write        # (Optional) Show review progress as a check run on the PR
 #       secrets:
 #         ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
 #         CAGENT_ORG_MEMBERSHIP_TOKEN: ${{ secrets.CAGENT_ORG_MEMBERSHIP_TOKEN }} # PAT with read:org scope; gates auto-reviews to org members only
@@ -99,6 +100,7 @@ permissions:
   contents: read
   pull-requests: write
   issues: write
+  checks: write
 
 jobs:
   # ==========================================================================
@@ -203,6 +205,30 @@ jobs:
       exit-code: ${{ steps.run-review.outputs.exit-code }}
 
     steps:
+      - name: Create check run
+        id: create-check
+        continue-on-error: true # Don't fail if caller didn't grant checks: write
+        uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1
+        env:
+          PR_NUMBER: ${{ inputs.pr-number || github.event.issue.number }}
+        with:
+          script: |
+            const prNumber = parseInt(process.env.PR_NUMBER, 10);
+            const { data: pr } = await github.rest.pulls.get({
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              pull_number: prNumber
+            });
+            const { data: check } = await github.rest.checks.create({
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              name: 'PR Review',
+              head_sha: pr.head.sha,
+              status: 'in_progress',
+              started_at: new Date().toISOString()
+            });
+            core.setOutput('check-id', check.id);
+
       # Checkout PR head (not default branch)
       # Note: Authorization is handled by the composite action's built-in check
       - name: Checkout PR head
@@ -240,6 +266,28 @@ jobs:
           nebius-api-key: ${{ secrets.NEBIUS_API_KEY }}
           mistral-api-key: ${{ secrets.MISTRAL_API_KEY }}
 
+      - name: Update check run
+        if: always() && steps.create-check.outputs.check-id != ''
+        uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1
+        env:
+          CHECK_ID: ${{ steps.create-check.outputs.check-id }}
+          JOB_STATUS: ${{ job.status }}
+        with:
+          script: |
+            const conclusion = process.env.JOB_STATUS === 'cancelled' ? 'cancelled' : process.env.JOB_STATUS === 'success' ? 'success' : 'failure';
+            try {
+              await github.rest.checks.update({
+                owner: context.repo.owner,
+                repo: context.repo.repo,
+                check_run_id: parseInt(process.env.CHECK_ID, 10),
+                status: 'completed',
+                conclusion: conclusion,
+                completed_at: new Date().toISOString()
+              });
+            } catch (error) {
+              core.warning(`Failed to update check run: ${error.message}`);
+            }
+
   # ==========================================================================
   # CAPTURE FEEDBACK
   # Saves feedback data as an artifact for lazy processing. This job
diff --git a/.github/workflows/self-review-pr.yml b/.github/workflows/self-review-pr.yml
@@ -17,6 +17,7 @@ permissions:
   contents: read
   pull-requests: write
   issues: write
+  checks: write
 
 jobs:
   # ==========================================================================
@@ -111,6 +112,28 @@ jobs:
       HAS_APP_SECRETS: ${{ secrets.CAGENT_REVIEWER_APP_ID != '' }}
 
     steps:
+      - name: Create check run
+        id: create-check
+        continue-on-error: true # Don't fail if checks: write permission is missing
+        uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1
+        with:
+          script: |
+            const prNumber = context.issue.number;
+            const { data: pr } = await github.rest.pulls.get({
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              pull_number: prNumber
+            });
+            const { data: check } = await github.rest.checks.create({
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              name: 'PR Review',
+              head_sha: pr.head.sha,
+              status: 'in_progress',
+              started_at: new Date().toISOString()
+            });
+            core.setOutput('check-id', check.id);
+
       # Checkout PR head (not default branch)
       # Note: Authorization is handled by the composite action's built-in check
       - name: Checkout PR head
@@ -145,6 +168,28 @@ jobs:
           nebius-api-key: ${{ secrets.NEBIUS_API_KEY }}
           mistral-api-key: ${{ secrets.MISTRAL_API_KEY }}
 
+      - name: Update check run
+        if: always() && steps.create-check.outputs.check-id != ''
+        uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1
+        env:
+          CHECK_ID: ${{ steps.create-check.outputs.check-id }}
+          JOB_STATUS: ${{ job.status }}
+        with:
+          script: |
+            const conclusion = process.env.JOB_STATUS === 'cancelled' ? 'cancelled' : process.env.JOB_STATUS === 'success' ? 'success' : 'failure';
+            try {
+              await github.rest.checks.update({
+                owner: context.repo.owner,
+                repo: context.repo.repo,
+                check_run_id: parseInt(process.env.CHECK_ID, 10),
+                status: 'completed',
+                conclusion: conclusion,
+                completed_at: new Date().toISOString()
+              });
+            } catch (error) {
+              core.warning(`Failed to update check run: ${error.message}`);
+            }
+
   # ==========================================================================
   # CAPTURE FEEDBACK
   # Saves feedback data as an artifact for lazy processing. This job
diff --git a/README.md b/README.md
@@ -69,6 +69,7 @@ jobs:
       contents: read       # Read repository files and PR diffs
       pull-requests: write # Post review comments and approve/request changes
       issues: write        # Create security incident issues if secrets are detected in output
+      checks: write        # (Optional) Show review progress as a check run on the PR
     secrets:
       ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
       CAGENT_ORG_MEMBERSHIP_TOKEN: ${{ secrets.CAGENT_ORG_MEMBERSHIP_TOKEN }}         # PAT with read:org scope; gates auto-reviews to org members only
@@ -217,6 +218,7 @@ permissions:
   contents: read
   pull-requests: write
   issues: write
+  checks: write         # Optional: show review progress as a check run on PRs
 ```
 
 
diff --git a/review-pr/README.md b/review-pr/README.md
@@ -29,6 +29,7 @@ jobs:
       contents: read       # Read repository files and PR diffs
       pull-requests: write # Post review comments and approve/request changes
       issues: write        # Create security incident issues if secrets are detected in output
+      checks: write        # (Optional) Show review progress as a check run on the PR
     secrets:
       ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
       CAGENT_ORG_MEMBERSHIP_TOKEN: ${{ secrets.CAGENT_ORG_MEMBERSHIP_TOKEN }}         # PAT with read:org scope; gates auto-reviews to org members only
@@ -53,7 +54,7 @@ The workflow automatically handles:
 | Trigger                 | Behavior                                                                                |
 | ----------------------- | --------------------------------------------------------------------------------------- |
 | PR opened/ready         | Auto-reviews PRs from your org members (if `CAGENT_ORG_MEMBERSHIP_TOKEN` is configured) |
-| `/review` comment       | Manual review on any PR                                                                 |
+| `/review` comment       | Manual review on any PR (shows as a check run on the PR if `checks: write` is granted)  |
 | Reply to review comment | Learns from feedback to improve future reviews                                          |
 
 ---
@@ -131,6 +132,7 @@ on:
 permissions:
   contents: read
   pull-requests: write
+  checks: write         # Optional: show review progress as a check run
 
 jobs:
   review:
@@ -318,9 +320,15 @@ When no issues are found:
 
 ---
 
-## Reactions
+## Progress Indicators
 
-The action uses emoji reactions on your `/review` comment to indicate progress:
+### Check Runs
+
+When the workflow has `checks: write` permission, reviews appear as check runs on the PR's Checks tab. This makes it easy to see review status at a glance — `in_progress` while reviewing, then `success`, `failure`, or `cancelled` when done. If `checks: write` is not granted, the workflow still works normally without check runs.
+
+### Reactions
+
+The action also uses emoji reactions on your `/review` comment to indicate progress:
 
 | Stage             | Reaction | Meaning                        |
 | ----------------- | -------- | ------------------------------ |
@@ -440,6 +448,11 @@ Each eval file in `review-pr/agents/evals/` contains:
 - Ensure the workflow has `pull-requests: write` permission
 - Check if the `github-token` has access to react to comments
 
+**No check run showing on the PR?**
+
+- Add `checks: write` to your workflow permissions (it's optional — the review works without it)
+- Check runs are created for manual (`/review`) triggers. Auto-reviews from `pull_request_target` already appear as workflow runs natively
+
 **Learning doesn't seem to work?**
 
 - You must **reply directly** to an agent comment (use the reply button, not a new comment)
diff --git a/review-pr/action.yml b/review-pr/action.yml
@@ -329,20 +329,29 @@ runs:
           THREAD_PATH=$(echo "$BOT_THREADS" | jq -r ".[$i].path")
           THREAD_LINE=$(echo "$BOT_THREADS" | jq -r ".[$i].line")
 
-          # Skip threads with null path or line (outdated diff positions, file-level comments)
-          if [ "$THREAD_PATH" = "null" ] || [ -z "$THREAD_PATH" ] || \
-             [ "$THREAD_LINE" = "null" ] || [ -z "$THREAD_LINE" ]; then
-            echo "  ⏭️  Keeping open: thread $THREAD_ID (path or line is null/outdated)"
+          # Skip threads with null path (file-level or unknown context)
+          if [ "$THREAD_PATH" = "null" ] || [ -z "$THREAD_PATH" ]; then
+            echo "  ⏭️  Keeping open: thread $THREAD_ID (no file path)"
             KEPT=$((KEPT + 1))
             continue
           fi
 
-          FILE_LINE="${THREAD_PATH}:${THREAD_LINE}"
-
-          if grep -qFx "$FILE_LINE" "$DIFF_LINES_FILE"; then
-            echo "  ⏭️  Keeping open: $FILE_LINE (still in diff)"
+          # Null line means GitHub marked the comment "outdated" — the code changed
+          # since the comment was posted and the position is no longer valid.
+          # These are stale by definition and should be resolved.
+          SHOULD_RESOLVE=false
+          if [ "$THREAD_LINE" = "null" ] || [ -z "$THREAD_LINE" ]; then
+            echo "  🔄 Outdated: $THREAD_PATH (position invalidated by push)"
+            SHOULD_RESOLVE=true
+          elif grep -qFx "${THREAD_PATH}:${THREAD_LINE}" "$DIFF_LINES_FILE"; then
+            echo "  ⏭️  Keeping open: ${THREAD_PATH}:${THREAD_LINE} (still in diff)"
             KEPT=$((KEPT + 1))
           else
+            echo "  🔄 Stale: ${THREAD_PATH}:${THREAD_LINE} (no longer in diff)"
+            SHOULD_RESOLVE=true
+          fi
+
+          if [ "$SHOULD_RESOLVE" = "true" ]; then
             MUTATION_RESULT=$(gh api graphql -f query='
               mutation($threadId: ID!) {
                 resolveReviewThread(input: { threadId: $threadId }) {
@@ -359,7 +368,7 @@ runs:
               continue
             fi
 
-            echo "  ✅ Resolved: $FILE_LINE (no longer in diff)"
+            echo "  ✅ Resolved thread $THREAD_ID"
             RESOLVED=$((RESOLVED + 1))
           fi
         done
