GitHub Models PR Review #1
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # ============================================================ | |
| # GitHub Models PR Review Workflow | |
| # ============================================================ | |
| # Integration approach: GitHub Models inference endpoint (OpenAI-compatible) | |
| # Rationale: Uses the free GitHub Models allowance included with GitHub/Copilot | |
| # accounts. No separate API key required — authentication uses the auto-provided | |
| # GITHUB_TOKEN. We fetch only PR metadata and the diff via the GitHub API | |
| # (no untrusted code is executed, no checkout of PR code). | |
| # | |
| # Models used (both free-tier on GitHub Models): | |
| # Primary : openai/gpt-4o — strong review quality, low-tier rate limit | |
| # Fallback : openai/gpt-4o-mini — lighter model, higher rate limit | |
| # | |
| # Auto-provided by GitHub (no setup needed): | |
| # GITHUB_TOKEN — used for gh cli calls, posting comments, and Models API auth | |
| # (requires `models: read` permission — see permissions block) | |
| # ============================================================ | |
| name: GitHub Models PR Review | |
| on: | |
| # Manual trigger: backlog sweep or single PR review | |
| workflow_dispatch: | |
| inputs: | |
| mode: | |
| description: "Review mode" | |
| required: true | |
| type: choice | |
| options: | |
| - latest | |
| - single | |
| - backlog | |
| default: latest | |
| pr_number: | |
| description: "PR number (required when mode=single)" | |
| required: false | |
| type: string | |
| # Auto-trigger on new/updated PRs | |
| pull_request_target: | |
| types: [opened, reopened, synchronize] | |
| permissions: | |
| pull-requests: write | |
| contents: read | |
| issues: write | |
| models: read | |
| jobs: | |
| # ────────────────────────────────────────────────────────── | |
| # AUTO: review a single PR when it is opened/updated | |
| # ────────────────────────────────────────────────────────── | |
| review-pr-auto: | |
| name: Review PR (auto) | |
| if: github.event_name == 'pull_request_target' | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Review PR ${{ github.event.pull_request.number }} | |
| env: | |
| GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| PR_NUMBER: ${{ github.event.pull_request.number }} | |
| REPO: ${{ github.repository }} | |
| run: | | |
| set -euo pipefail | |
| # ── helpers ────────────────────────────────────── | |
| MARKER="<!-- pr-review -->" | |
| PRIMARY_MODEL="openai/gpt-4o" | |
| FALLBACK_MODEL="openai/gpt-4o-mini" | |
| MODELS_ENDPOINT="https://models.github.ai/inference/chat/completions" | |
| already_reviewed() { | |
| local pr="$1" | |
| local comments | |
| if ! comments=$(gh api "repos/${REPO}/issues/${pr}/comments" --jq '.[].body' 2>/dev/null); then | |
| # API failed — conservative: assume already reviewed to avoid spam | |
| echo "yes" | |
| return | |
| fi | |
| if printf '%s' "$comments" | grep -qF "${MARKER}"; then | |
| echo "yes" | |
| else | |
| echo "no" | |
| fi | |
| } | |
| post_review() { | |
| local pr="$1" | |
| local body="$2" | |
| gh api "repos/${REPO}/issues/${pr}/comments" \ | |
| -X POST \ | |
| -f body="${MARKER}"$'\n'"${body}" | |
| } | |
| # call_model <model> <prompt> | |
| # Returns 0 and sets REVIEW_BODY on success. | |
| # Returns 1 on all retries exhausted. | |
| call_model() { | |
| local model="$1" | |
| local prompt="$2" | |
| local attempt delay http_code response body | |
| for attempt in 1 2 3; do | |
| delay=$(( 2 ** attempt )) # 2s, 4s, 8s | |
| # Capture body + HTTP status in one call | |
| response=$(curl -sS -w "\n%{http_code}" \ | |
| -X POST "${MODELS_ENDPOINT}" \ | |
| -H "Authorization: Bearer ${GH_TOKEN}" \ | |
| -H "Content-Type: application/json" \ | |
| -d "$(jq -n \ | |
| --arg model "${model}" \ | |
| --arg content "${prompt}" \ | |
| '{ | |
| model: $model, | |
| max_tokens: 1024, | |
| temperature: 0.3, | |
| messages: [{ role: "user", content: $content }] | |
| }')" 2>/dev/null) || true | |
| http_code=$(printf '%s' "${response}" | tail -n1) | |
| body=$(printf '%s' "${response}" | head -n -1) | |
| echo " [attempt ${attempt}] model=${model} http=${http_code}" | |
| # Check for API-level error in body | |
| if printf '%s' "${body}" | jq -e '.error' >/dev/null 2>&1; then | |
| echo " API error: $(printf '%s' "${body}" | jq -r '.error.message // .error')" | |
| if [ "${attempt}" -lt 3 ]; then | |
| echo " Retrying in ${delay}s..." | |
| sleep "${delay}" | |
| fi | |
| continue | |
| fi | |
| if [ "${http_code}" = "200" ]; then | |
| local text | |
| text=$(printf '%s' "${body}" | jq -r '.choices[0].message.content // empty' 2>/dev/null || true) | |
| if [ -n "${text}" ]; then | |
| REVIEW_BODY="${text}" | |
| USED_MODEL="${model}" | |
| return 0 | |
| fi | |
| echo " Empty content in response; retrying..." | |
| elif [ "${http_code}" = "429" ] || [ "${http_code}" -ge 500 ]; then | |
| local retry_after | |
| # respect retry-after if the header was surfaced in the body | |
| retry_after=$(printf '%s' "${body}" | jq -r '.error.retry_after // empty' 2>/dev/null || true) | |
| if [ -n "${retry_after}" ]; then | |
| echo " Rate limited; waiting ${retry_after}s (retry-after)..." | |
| sleep "${retry_after}" | |
| else | |
| echo " Retryable error ${http_code}; waiting ${delay}s..." | |
| sleep "${delay}" | |
| fi | |
| else | |
| echo " Non-retryable error ${http_code} from ${model}" | |
| break | |
| fi | |
| done | |
| return 1 | |
| } | |
| # ── skip if already reviewed ────────────────────── | |
| if [ "$(already_reviewed "${PR_NUMBER}")" = "yes" ]; then | |
| echo "PR #${PR_NUMBER} already has a review — skipping." | |
| exit 0 | |
| fi | |
| # ── fetch PR metadata (no checkout of PR code) ─── | |
| PR_JSON=$(gh api "repos/${REPO}/pulls/${PR_NUMBER}") | |
| PR_TITLE=$(echo "${PR_JSON}" | jq -r '.title') | |
| PR_BODY=$(echo "${PR_JSON}" | jq -r '.body // "(no description)"') | |
| PR_AUTHOR=$(echo "${PR_JSON}" | jq -r '.user.login') | |
| # changed files list | |
| FILES=$(gh api "repos/${REPO}/pulls/${PR_NUMBER}/files" \ | |
| --jq '[.[] | .filename] | join("\n")' 2>/dev/null || echo "(unavailable)") | |
| # diff (capped at 50 KB to stay well inside the context window) | |
| DIFF=$(gh pr diff "${PR_NUMBER}" --repo "${REPO}" 2>/dev/null | head -c 51200 | iconv -c -f UTF-8 -t UTF-8 || echo "(diff unavailable)") | |
| # ── build prompt ────────────────────────────────── | |
| # Note: indented here to satisfy YAML block scalar; sed strips the indent. | |
| PROMPT="You are a security-aware code reviewer for an open-source repository of n8n workflow JSON templates. Your job is to review pull requests submitted by contributors and flag issues specific to n8n template repositories. | |
| ## PR Details | |
| - **PR number**: #${PR_NUMBER} | |
| - **Author**: @${PR_AUTHOR} | |
| - **Title**: ${PR_TITLE} | |
| - **Description**: ${PR_BODY} | |
| ## Changed Files | |
| \`\`\` | |
| ${FILES} | |
| \`\`\` | |
| ## Diff (may be truncated at 50 KB) | |
| \`\`\`diff | |
| ${DIFF} | |
| \`\`\` | |
| ## Your task | |
| Respond with a structured markdown review containing EXACTLY these three sections: | |
| ### Summary | |
| 2-4 bullet points describing what this PR does. | |
| ### Safety Assessment | |
| One of: **SAFE TO MERGE** | **NEEDS CHANGES** | **DO NOT MERGE** | |
| Followed by a brief rationale (1-3 sentences). | |
| ### Action Items | |
| If the assessment is NEEDS CHANGES or DO NOT MERGE, list specific actionable steps the maintainer should request from the contributor. If SAFE TO MERGE, write \"None — looks good.\". | |
| ## Review checklist (n8n template repo specific — flag any of these) | |
| - Hardcoded credentials, API keys, tokens, passwords, or secrets in JSON | |
| - Suspicious or malicious URLs (phishing, C2, data exfiltration patterns) | |
| - Malicious or obfuscated code/expressions inside n8n nodes | |
| - Duplicate templates (the same workflow already exists in the repo) | |
| - Broken or invalid JSON that would fail to import into n8n | |
| - Missing README entry for the new template (contributors should add one) | |
| - Overly broad permissions requested by the workflow (e.g. admin-level API scopes) | |
| Be concise. This review will be posted as a GitHub comment." | |
| PROMPT=$(printf '%s' "${PROMPT}" | sed 's/^ //') | |
| # ── call GitHub Models with fallback ───────────── | |
| REVIEW_BODY="" | |
| USED_MODEL="" | |
| echo "Trying primary model: ${PRIMARY_MODEL}" | |
| if ! call_model "${PRIMARY_MODEL}" "${PROMPT}"; then | |
| echo "Primary model failed. Trying fallback: ${FALLBACK_MODEL}" | |
| if ! call_model "${FALLBACK_MODEL}" "${PROMPT}"; then | |
| echo "Both models failed. Posting fallback comment." | |
| REVIEW_BODY="Automated review unavailable — please review manually." | |
| USED_MODEL="(unavailable)" | |
| fi | |
| fi | |
| # ── compose and post comment ────────────────────── | |
| COMMENT="## 🤖 Automated PR Review | |
| ${REVIEW_BODY} | |
| --- | |
| <sub>Reviewed by \`${USED_MODEL}\` via GitHub Models · [Workflow run](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }})</sub>" | |
| COMMENT=$(printf '%s' "${COMMENT}" | sed 's/^ //') | |
| post_review "${PR_NUMBER}" "${COMMENT}" | |
| echo "Review posted on PR #${PR_NUMBER} (model: ${USED_MODEL})." | |
| # ────────────────────────────────────────────────────────── | |
| # MANUAL: backlog sweep or single-PR review | |
| # ────────────────────────────────────────────────────────── | |
| review-pr-manual: | |
| name: Review PR (manual — ${{ inputs.mode }}) | |
| if: github.event_name == 'workflow_dispatch' && github.actor == github.repository_owner | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Run review (${{ inputs.mode }}) | |
| env: | |
| GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| REPO: ${{ github.repository }} | |
| MODE: ${{ inputs.mode }} | |
| INPUT_PR_NUMBER: ${{ inputs.pr_number }} | |
| run: | | |
| set -euo pipefail | |
| # ── helpers ────────────────────────────────────── | |
| MARKER="<!-- pr-review -->" | |
| PRIMARY_MODEL="openai/gpt-4o" | |
| FALLBACK_MODEL="openai/gpt-4o-mini" | |
| MODELS_ENDPOINT="https://models.github.ai/inference/chat/completions" | |
| already_reviewed() { | |
| local pr="$1" | |
| local comments | |
| if ! comments=$(gh api "repos/${REPO}/issues/${pr}/comments" --jq '.[].body' 2>/dev/null); then | |
| # API failed — conservative: assume already reviewed to avoid spam | |
| echo "yes" | |
| return | |
| fi | |
| if printf '%s' "$comments" | grep -qF "${MARKER}"; then | |
| echo "yes" | |
| else | |
| echo "no" | |
| fi | |
| } | |
| post_review() { | |
| local pr="$1" | |
| local body="$2" | |
| gh api "repos/${REPO}/issues/${pr}/comments" \ | |
| -X POST \ | |
| -f body="${MARKER}"$'\n'"${body}" | |
| } | |
| # call_model <model> <prompt> | |
| # Returns 0 and sets REVIEW_BODY on success. | |
| # Returns 1 on all retries exhausted. | |
| call_model() { | |
| local model="$1" | |
| local prompt="$2" | |
| local attempt delay http_code response body | |
| for attempt in 1 2 3; do | |
| delay=$(( 2 ** attempt )) # 2s, 4s, 8s | |
| # Capture body + HTTP status in one call | |
| response=$(curl -sS -w "\n%{http_code}" \ | |
| -X POST "${MODELS_ENDPOINT}" \ | |
| -H "Authorization: Bearer ${GH_TOKEN}" \ | |
| -H "Content-Type: application/json" \ | |
| -d "$(jq -n \ | |
| --arg model "${model}" \ | |
| --arg content "${prompt}" \ | |
| '{ | |
| model: $model, | |
| max_tokens: 1024, | |
| temperature: 0.3, | |
| messages: [{ role: "user", content: $content }] | |
| }')" 2>/dev/null) || true | |
| http_code=$(printf '%s' "${response}" | tail -n1) | |
| body=$(printf '%s' "${response}" | head -n -1) | |
| echo " [attempt ${attempt}] model=${model} http=${http_code}" | |
| # Check for API-level error in body | |
| if printf '%s' "${body}" | jq -e '.error' >/dev/null 2>&1; then | |
| echo " API error: $(printf '%s' "${body}" | jq -r '.error.message // .error')" | |
| if [ "${attempt}" -lt 3 ]; then | |
| echo " Retrying in ${delay}s..." | |
| sleep "${delay}" | |
| fi | |
| continue | |
| fi | |
| if [ "${http_code}" = "200" ]; then | |
| local text | |
| text=$(printf '%s' "${body}" | jq -r '.choices[0].message.content // empty' 2>/dev/null || true) | |
| if [ -n "${text}" ]; then | |
| REVIEW_BODY="${text}" | |
| USED_MODEL="${model}" | |
| return 0 | |
| fi | |
| echo " Empty content in response; retrying..." | |
| elif [ "${http_code}" = "429" ] || [ "${http_code}" -ge 500 ]; then | |
| local retry_after | |
| # respect retry-after if the header was surfaced in the body | |
| retry_after=$(printf '%s' "${body}" | jq -r '.error.retry_after // empty' 2>/dev/null || true) | |
| if [ -n "${retry_after}" ]; then | |
| echo " Rate limited; waiting ${retry_after}s (retry-after)..." | |
| sleep "${retry_after}" | |
| else | |
| echo " Retryable error ${http_code}; waiting ${delay}s..." | |
| sleep "${delay}" | |
| fi | |
| else | |
| echo " Non-retryable error ${http_code} from ${model}" | |
| break | |
| fi | |
| done | |
| return 1 | |
| } | |
| review_one_pr() { | |
| local PR_NUMBER="$1" | |
| echo "--- Reviewing PR #${PR_NUMBER} ---" | |
| # fetch metadata (no checkout) | |
| PR_JSON=$(gh api "repos/${REPO}/pulls/${PR_NUMBER}") | |
| PR_TITLE=$(echo "${PR_JSON}" | jq -r '.title') | |
| PR_BODY=$(echo "${PR_JSON}" | jq -r '.body // "(no description)"') | |
| PR_AUTHOR=$(echo "${PR_JSON}" | jq -r '.user.login') | |
| FILES=$(gh api "repos/${REPO}/pulls/${PR_NUMBER}/files" \ | |
| --jq '[.[] | .filename] | join("\n")' 2>/dev/null || echo "(unavailable)") | |
| DIFF=$(gh pr diff "${PR_NUMBER}" --repo "${REPO}" 2>/dev/null | head -c 51200 | iconv -c -f UTF-8 -t UTF-8 || echo "(diff unavailable)") | |
| # Note: indented here to satisfy YAML block scalar; sed strips the indent. | |
| PROMPT="You are a security-aware code reviewer for an open-source repository of n8n workflow JSON templates. Your job is to review pull requests submitted by contributors and flag issues specific to n8n template repositories. | |
| ## PR Details | |
| - **PR number**: #${PR_NUMBER} | |
| - **Author**: @${PR_AUTHOR} | |
| - **Title**: ${PR_TITLE} | |
| - **Description**: ${PR_BODY} | |
| ## Changed Files | |
| \`\`\` | |
| ${FILES} | |
| \`\`\` | |
| ## Diff (may be truncated at 50 KB) | |
| \`\`\`diff | |
| ${DIFF} | |
| \`\`\` | |
| ## Your task | |
| Respond with a structured markdown review containing EXACTLY these three sections: | |
| ### Summary | |
| 2-4 bullet points describing what this PR does. | |
| ### Safety Assessment | |
| One of: **SAFE TO MERGE** | **NEEDS CHANGES** | **DO NOT MERGE** | |
| Followed by a brief rationale (1-3 sentences). | |
| ### Action Items | |
| If the assessment is NEEDS CHANGES or DO NOT MERGE, list specific actionable steps the maintainer should request from the contributor. If SAFE TO MERGE, write \"None — looks good.\". | |
| ## Review checklist (n8n template repo specific — flag any of these) | |
| - Hardcoded credentials, API keys, tokens, passwords, or secrets in JSON | |
| - Suspicious or malicious URLs (phishing, C2, data exfiltration patterns) | |
| - Malicious or obfuscated code/expressions inside n8n nodes | |
| - Duplicate templates (the same workflow already exists in the repo) | |
| - Broken or invalid JSON that would fail to import into n8n | |
| - Missing README entry for the new template (contributors should add one) | |
| - Overly broad permissions requested by the workflow (e.g. admin-level API scopes) | |
| Be concise. This review will be posted as a GitHub comment." | |
| PROMPT=$(printf '%s' "${PROMPT}" | sed 's/^ //') | |
| REVIEW_BODY="" | |
| USED_MODEL="" | |
| echo "Trying primary model: ${PRIMARY_MODEL}" | |
| if ! call_model "${PRIMARY_MODEL}" "${PROMPT}"; then | |
| echo "Primary model failed. Trying fallback: ${FALLBACK_MODEL}" | |
| if ! call_model "${FALLBACK_MODEL}" "${PROMPT}"; then | |
| echo "Both models failed. Posting fallback comment." | |
| REVIEW_BODY="Automated review unavailable — please review manually." | |
| USED_MODEL="(unavailable)" | |
| fi | |
| fi | |
| COMMENT=" ## 🤖 Automated PR Review | |
| ${REVIEW_BODY} | |
| --- | |
| <sub>Reviewed by \`${USED_MODEL}\` via GitHub Models · [Workflow run](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }})</sub>" | |
| COMMENT=$(printf '%s' "${COMMENT}" | sed 's/^ //') | |
| post_review "${PR_NUMBER}" "${COMMENT}" | |
| echo "Review posted on PR #${PR_NUMBER} (model: ${USED_MODEL})." | |
| } | |
| # ── mode dispatch ───────────────────────────────── | |
| if [ "${MODE}" = "latest" ]; then | |
| echo "Mode=latest: finding most recently opened open PR..." | |
| LATEST_PR=$(gh pr list --repo "${REPO}" --state open --json number,createdAt \ | |
| --jq 'sort_by(.createdAt) | reverse | .[0].number') | |
| if [ -z "${LATEST_PR}" ] || [ "${LATEST_PR}" = "null" ]; then | |
| echo "No open PRs found." | |
| exit 0 | |
| fi | |
| echo "Latest open PR: #${LATEST_PR}" | |
| if [ "$(already_reviewed "${LATEST_PR}")" = "yes" ]; then | |
| echo "PR #${LATEST_PR} already reviewed — use 'single' mode with the PR number to force re-review." | |
| exit 0 | |
| fi | |
| review_one_pr "${LATEST_PR}" | |
| elif [ "${MODE}" = "single" ]; then | |
| if [ -z "${INPUT_PR_NUMBER}" ]; then | |
| echo "ERROR: pr_number input is required when mode=single" | |
| exit 1 | |
| fi | |
| review_one_pr "${INPUT_PR_NUMBER}" | |
| elif [ "${MODE}" = "backlog" ]; then | |
| echo "Fetching all open PRs..." | |
| OPEN_PRS=$(gh pr list --repo "${REPO}" --state open --json number \ | |
| --jq '.[].number') | |
| if [ -z "${OPEN_PRS}" ]; then | |
| echo "No open PRs found." | |
| exit 0 | |
| fi | |
| for PR_NUM in ${OPEN_PRS}; do | |
| if [ "$(already_reviewed "${PR_NUM}")" = "yes" ]; then | |
| echo "PR #${PR_NUM} already reviewed — skipping." | |
| else | |
| review_one_pr "${PR_NUM}" | |
| # Small delay between PRs to avoid rate limits | |
| sleep 10 | |
| fi | |
| done | |
| echo "Backlog sweep complete." | |
| else | |
| echo "ERROR: unknown mode '${MODE}'" | |
| exit 1 | |
| fi |