Reconcile exact-review leases #465640
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Reconcile exact-review leases | |
| on: | |
| workflow_run: | |
| workflows: [ClawSweeper] | |
| types: [completed] | |
| schedule: | |
| - cron: "*/15 * * * *" | |
| workflow_dispatch: | |
| permissions: {} | |
| env: | |
| CLAWSWEEPER_APP_CLIENT_ID: Iv23liOECG0slfuhz093 | |
| concurrency: | |
| # This workflow owns both run telemetry and exact-lease repair. Keep terminal | |
| # tuples distinct because coalescing workflow_run events would drop telemetry. | |
| group: exact-review-reconcile-${{ github.event_name == 'workflow_run' && format('{0}-{1}', github.event.workflow_run.id, github.event.workflow_run.run_attempt) || 'sweep' }} | |
| cancel-in-progress: false | |
| jobs: | |
| reconcile: | |
| name: Observe and reconcile terminal review run | |
| # Schedule a runner only for titles the observer can classify. Queue-backed | |
| # "Review exact item" runs and support runs otherwise start an ubuntu runner | |
| # only to print "skipped", starving the Blacksmith jobs that must claim their | |
| # six-minute queue reservations. The test suite keeps this allowlist aligned | |
| # with REVIEW_RUN_OBSERVER_TITLE_LANES. | |
| if: >- | |
| ${{ | |
| github.event_name == 'workflow_run' && | |
| ( | |
| startsWith(github.event.workflow_run.display_title, 'Review scheduled hot item ') || | |
| startsWith(github.event.workflow_run.display_title, 'Review scheduled normal item ') || | |
| startsWith(github.event.workflow_run.display_title, 'Review event item') || | |
| startsWith(github.event.workflow_run.display_title, 'Review hot ClawSweeper items') || | |
| startsWith(github.event.workflow_run.display_title, 'Review hot target repo ') || | |
| startsWith(github.event.workflow_run.display_title, 'Retry failed Codex reviews') || | |
| startsWith(github.event.workflow_run.display_title, 'Review target repo ') || | |
| startsWith(github.event.workflow_run.display_title, 'Review ClawSweeper items') | |
| ) | |
| }} | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 5 | |
| permissions: | |
| actions: read | |
| contents: read | |
| steps: | |
| - name: Reconcile terminal run | |
| if: ${{ github.event.workflow_run.event == 'repository_dispatch' && startsWith(github.event.workflow_run.display_title, 'Review event item ') }} | |
| env: | |
| CLAWSWEEPER_WEBHOOK_SECRET: ${{ secrets.CLAWSWEEPER_WEBHOOK_SECRET }} | |
| QUEUE_URL: ${{ vars.CLAWSWEEPER_EXACT_REVIEW_QUEUE_URL || 'https://clawsweeper.openclaw.ai' }} | |
| SOURCE_RUN_ATTEMPT: ${{ github.event.workflow_run.run_attempt }} | |
| SOURCE_RUN_ID: ${{ github.event.workflow_run.id }} | |
| run: | | |
| set -euo pipefail | |
| test -n "$CLAWSWEEPER_WEBHOOK_SECRET" | |
| queue_url="${QUEUE_URL%/}" | |
| payload="$(node -e ' | |
| const runAttempt = Number(process.env.SOURCE_RUN_ATTEMPT); | |
| if (!Number.isInteger(runAttempt) || runAttempt < 1) process.exit(1); | |
| process.stdout.write(JSON.stringify({ | |
| runs: [{ | |
| run_id: process.env.SOURCE_RUN_ID, | |
| run_attempt: runAttempt, | |
| }], | |
| include_all_claimed: true, | |
| })); | |
| ')" | |
| signature="$(PAYLOAD="$payload" node -e 'const crypto=require("node:crypto"); process.stdout.write(`sha256=${crypto.createHmac("sha256", process.env.CLAWSWEEPER_WEBHOOK_SECRET).update(process.env.PAYLOAD).digest("hex")}`)')" | |
| for attempt in 1 2 3; do | |
| if curl --fail --silent --show-error --connect-timeout 5 --max-time 120 \ | |
| --request POST \ | |
| --header "content-type: application/json" \ | |
| --header "x-clawsweeper-exact-review-signature: $signature" \ | |
| --data-binary "$payload" \ | |
| "$queue_url/internal/exact-review/reconcile" >/dev/null; then | |
| exit 0 | |
| fi | |
| if [ "$attempt" -lt 3 ]; then | |
| sleep "$((attempt * 5))" | |
| fi | |
| done | |
| exit 1 | |
| - uses: actions/checkout@v7 | |
| if: ${{ always() }} | |
| # workflow_run may expose secrets, so execute observer code only from the trusted default branch. | |
| with: | |
| ref: ${{ github.event.repository.default_branch }} | |
| persist-credentials: false | |
| - name: Record terminal review wave | |
| if: ${{ always() }} | |
| env: | |
| CLAWSWEEPER_WEBHOOK_SECRET: ${{ secrets.CLAWSWEEPER_WEBHOOK_SECRET }} | |
| GH_TOKEN: ${{ github.token }} | |
| QUEUE_URL: ${{ vars.CLAWSWEEPER_EXACT_REVIEW_QUEUE_URL || 'https://clawsweeper.openclaw.ai' }} | |
| run: node scripts/review-run-observer.mjs --event-file "$GITHUB_EVENT_PATH" | |
| sweep: | |
| name: Sweep terminal exact-review runs | |
| if: ${{ github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' }} | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 10 | |
| permissions: | |
| actions: read | |
| contents: read | |
| issues: read | |
| pull-requests: read | |
| steps: | |
| - name: Reconcile claimed terminal runs | |
| env: | |
| CLAWSWEEPER_WEBHOOK_SECRET: ${{ secrets.CLAWSWEEPER_WEBHOOK_SECRET }} | |
| GH_TOKEN: ${{ github.token }} | |
| QUEUE_URL: ${{ vars.CLAWSWEEPER_EXACT_REVIEW_QUEUE_URL || 'https://clawsweeper.openclaw.ai' }} | |
| run: | | |
| node --input-type=module <<'NODE' | |
| import { createHmac } from "node:crypto"; | |
| const secret = process.env.CLAWSWEEPER_WEBHOOK_SECRET || ""; | |
| const token = process.env.GH_TOKEN || ""; | |
| const repository = process.env.GITHUB_REPOSITORY || ""; | |
| const queueUrl = (process.env.QUEUE_URL || "").replace(/\/$/, ""); | |
| if (!secret || !token || !repository || !queueUrl) throw new Error("missing reconciler configuration"); | |
| async function signedPost(path, value) { | |
| const body = JSON.stringify(value); | |
| const signature = `sha256=${createHmac("sha256", secret).update(body).digest("hex")}`; | |
| const response = await fetch(`${queueUrl}${path}`, { | |
| method: "POST", | |
| headers: { | |
| "content-type": "application/json", | |
| "x-clawsweeper-exact-review-signature": signature, | |
| }, | |
| body, | |
| }); | |
| if (!response.ok) throw new Error(`${path} returned ${response.status}: ${await response.text()}`); | |
| return response.json(); | |
| } | |
| const claimed = await signedPost("/internal/exact-review/claimed-runs", { | |
| runs: [], | |
| include_all_claimed: true, | |
| }); | |
| const runs = Array.isArray(claimed.runs) ? claimed.runs : []; | |
| const terminalRuns = []; | |
| const unavailable = []; | |
| for (let offset = 0; offset < runs.length; offset += 8) { | |
| await Promise.all(runs.slice(offset, offset + 8).map(async (claim) => { | |
| const runId = String(claim.run_id || ""); | |
| const claimedAttempt = claim.run_attempt == null ? null : Number(claim.run_attempt); | |
| const claimGeneration = Number(claim.claim_generation); | |
| if (!/^\d+$/.test(runId) || (claimedAttempt !== null && (!Number.isInteger(claimedAttempt) || claimedAttempt < 1)) || !Number.isInteger(claimGeneration) || claimGeneration < 0) { | |
| throw new Error("invalid claimed run tuple"); | |
| } | |
| const suffix = claimedAttempt === null ? "" : `/attempts/${claimedAttempt}`; | |
| const response = await fetch(`${process.env.GITHUB_API_URL || "https://api.github.com"}/repos/${repository}/actions/runs/${runId}${suffix}`, { | |
| headers: { authorization: `Bearer ${token}`, accept: "application/vnd.github+json" }, | |
| }); | |
| if (!response.ok) { | |
| unavailable.push(`${runId}:${response.status}`); | |
| return; | |
| } | |
| const run = await response.json(); | |
| const runAttempt = Number(run.run_attempt); | |
| if (String(run.id || "") !== runId || !Number.isInteger(runAttempt) || runAttempt < 1 || (claimedAttempt !== null && runAttempt !== claimedAttempt)) { | |
| unavailable.push(`${runId}:mismatch`); | |
| return; | |
| } | |
| if (run.status !== "completed") return; | |
| if (!run.conclusion) { | |
| unavailable.push(`${runId}:missing-conclusion`); | |
| return; | |
| } | |
| terminalRuns.push({ | |
| run_id: runId, | |
| run_attempt: runAttempt, | |
| claimed_run_attempt: claimedAttempt, | |
| claim_generation: claimGeneration, | |
| outcome: run.conclusion === "success" ? "success" : run.conclusion === "cancelled" ? "cancelled" : "failure", | |
| }); | |
| })); | |
| } | |
| let reconciled = 0; | |
| if (terminalRuns.length) { | |
| const result = await signedPost("/internal/exact-review/reconcile", { terminal_runs: terminalRuns }); | |
| reconciled = Number(result.reconciled) || 0; | |
| } | |
| console.log(`checked=${runs.length} terminal=${terminalRuns.length} reconciled=${reconciled} unavailable=${unavailable.length}`); | |
| if (unavailable.length) console.log(`unavailable runs (next sweep retries): ${unavailable.join(",")}`); | |
| // Partial lookup failures are routine (rate limits, expired run retention); the | |
| // 15-minute cadence and lease expiry are the backstop. Fail only on total blindness. | |
| if (runs.length && unavailable.length === runs.length) throw new Error("all claimed run lookups failed"); | |
| NODE | |
| - uses: actions/checkout@v7 | |
| - name: Create state token | |
| id: state-token | |
| continue-on-error: true | |
| uses: ./.github/actions/create-state-token | |
| with: | |
| client-id: ${{ env.CLAWSWEEPER_APP_CLIENT_ID }} | |
| private-key: ${{ secrets.CLAWSWEEPER_APP_PRIVATE_KEY }} | |
| - uses: ./.github/actions/setup-pnpm | |
| with: | |
| build-script: build:all | |
| # The workflow-scoped GITHUB_TOKEN cannot mutate the target repo; stuck | |
| # escalation labels need an app installation token for the target owner. | |
| # The label endpoint is served under /issues but still requires | |
| # pull-requests write when the item is a pull request; issues write alone | |
| # escalates issues and 403s every pull-request escalation. | |
| - name: Create target write token | |
| id: target-write-token | |
| continue-on-error: true | |
| uses: actions/create-github-app-token@bcd2ba49218906704ab6c1aa796996da409d3eb1 # v3.2.0 | |
| with: | |
| client-id: ${{ env.CLAWSWEEPER_APP_CLIENT_ID }} | |
| private-key: ${{ secrets.CLAWSWEEPER_APP_PRIVATE_KEY }} | |
| owner: openclaw | |
| repositories: openclaw | |
| permission-issues: write | |
| permission-pull-requests: write | |
| - name: Recover orphaned review placeholders | |
| env: | |
| CLAWSWEEPER_WEBHOOK_SECRET: ${{ secrets.CLAWSWEEPER_WEBHOOK_SECRET }} | |
| GH_TOKEN: ${{ github.token }} | |
| TARGET_WRITE_TOKEN: ${{ steps.target-write-token.outputs.token }} | |
| QUEUE_URL: ${{ vars.CLAWSWEEPER_EXACT_REVIEW_QUEUE_URL || 'https://clawsweeper.openclaw.ai' }} | |
| REVIEW_PLACEHOLDER_CURSOR_STORE_URL: ${{ vars.CLAWSWEEPER_EXACT_REVIEW_QUEUE_URL || 'https://clawsweeper.openclaw.ai' }} | |
| REVIEW_PLACEHOLDER_LOOKBACK_HOURS: ${{ vars.REVIEW_PLACEHOLDER_LOOKBACK_HOURS || '48' }} | |
| REVIEW_PLACEHOLDER_MAX_CHECKS: ${{ vars.REVIEW_PLACEHOLDER_MAX_CHECKS || '20' }} | |
| REVIEW_PLACEHOLDER_MAX_RECOVERIES: ${{ vars.REVIEW_PLACEHOLDER_MAX_RECOVERIES || '5' }} | |
| REVIEW_PLACEHOLDER_MIN_AGE_HOURS: ${{ vars.REVIEW_PLACEHOLDER_MIN_AGE_HOURS || '2' }} | |
| TARGET_BRANCH: main | |
| TARGET_REPO: openclaw/openclaw | |
| run: node dist/review-placeholder-recovery.js | |
| - name: Report remaining state repository size | |
| continue-on-error: true | |
| env: | |
| CLAWSWEEPER_STATE_REPO_TOKEN: ${{ steps.state-token.outputs.token }} | |
| STATE_REPOSITORY: openclaw/clawsweeper-state | |
| STATE_REPO_SIZE_WARN_GB: ${{ vars.STATE_REPO_SIZE_WARN_GB || '5' }} | |
| run: node dist/repair/state-repo-size.js |