Skip to content

Commit 1fe4b58

Browse files
andrewgazelkacodex
andrewgazelka
and
codex
authored
ci: add Codex review gate (#121)
## Summary - add a GitHub Actions check that requires chatgpt-codex-connector to review the current PR head SHA - document the PR follow-through loop in AGENTS.md ## Checks - nix run .#lint - nix run nixpkgs#actionlint -- .github/workflows/codex-review-gate.yml --------- Co-authored-by: Codex <codex@openai.com>
1 parent fe71839 commit 1fe4b58

2 files changed

Lines changed: 338 additions & 3 deletions

File tree

‎.github/workflows/codex-review-gate.yml‎

Lines changed: 308 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,308 @@
1+
name: Codex review gate
2+
3+
on:
4+
pull_request:
5+
branches:
6+
- main
7+
types:
8+
- opened
9+
- synchronize
10+
- reopened
11+
- ready_for_review
12+
pull_request_review:
13+
types:
14+
- submitted
15+
- edited
16+
- dismissed
17+
pull_request_review_comment:
18+
types:
19+
- created
20+
- edited
21+
- deleted
22+
issue_comment:
23+
types:
24+
- created
25+
- edited
26+
- deleted
27+
28+
permissions:
29+
contents: read
30+
issues: read
31+
pull-requests: read
32+
33+
concurrency:
34+
group: codex-review-gate-${{ github.event.pull_request.number || github.event.issue.number }}
35+
cancel-in-progress: true
36+
37+
jobs:
38+
codex-review:
39+
name: chatgpt-codex-connector reviewed head
40+
if: github.event_name != 'issue_comment' || github.event.issue.pull_request != null
41+
runs-on: ubuntu-latest
42+
steps:
43+
- name: Verify Codex reviewed the current head
44+
env:
45+
GH_TOKEN: ${{ github.token }}
46+
PR_NUMBER: ${{ github.event.pull_request.number || github.event.issue.number }}
47+
REPO: ${{ github.repository }}
48+
run: |
49+
set -euo pipefail
50+
51+
pr_json="$(gh api "repos/${REPO}/pulls/${PR_NUMBER}")"
52+
head_sha="$(jq -r '.head.sha' <<<"$pr_json")"
53+
owner="${REPO%%/*}"
54+
repo_name="${REPO#*/}"
55+
codex_logins='["chatgpt-codex-connector[bot]","chatgpt-codex-connector"]'
56+
threads_query="$(
57+
cat <<'GRAPHQL'
58+
query($owner: String!, $repo: String!, $number: Int!, $after: String) {
59+
repository(owner: $owner, name: $repo) {
60+
pullRequest(number: $number) {
61+
reviewThreads(first: 100, after: $after) {
62+
pageInfo {
63+
hasNextPage
64+
endCursor
65+
}
66+
nodes {
67+
id
68+
isResolved
69+
comments(first: 100) {
70+
pageInfo {
71+
hasNextPage
72+
}
73+
nodes {
74+
author { login }
75+
url
76+
}
77+
}
78+
}
79+
}
80+
}
81+
}
82+
}
83+
GRAPHQL
84+
)"
85+
comments_query="$(
86+
cat <<'GRAPHQL'
87+
query($threadId: ID!, $after: String) {
88+
node(id: $threadId) {
89+
... on PullRequestReviewThread {
90+
comments(first: 100, after: $after) {
91+
pageInfo {
92+
hasNextPage
93+
endCursor
94+
}
95+
nodes {
96+
author { login }
97+
url
98+
}
99+
}
100+
}
101+
}
102+
}
103+
GRAPHQL
104+
)"
105+
106+
find_codex_comment_url() {
107+
local thread_id="$1"
108+
local after=""
109+
local comments_json
110+
local url
111+
112+
while true; do
113+
local args=(-f threadId="$thread_id" -f query="$comments_query")
114+
if [[ -n "$after" ]]; then
115+
args+=(-f after="$after")
116+
fi
117+
118+
comments_json="$(gh api graphql "${args[@]}")"
119+
url="$(
120+
jq -r --argjson codexLogins "$codex_logins" '
121+
[
122+
.data.node.comments.nodes[]?
123+
| select(.author.login as $login | $codexLogins | index($login))
124+
| .url
125+
][0] // empty
126+
' <<<"$comments_json"
127+
)"
128+
129+
if [[ -n "$url" ]]; then
130+
printf '%s\n' "$url"
131+
return 0
132+
fi
133+
134+
if [[ "$(jq -r '.data.node.comments.pageInfo.hasNextPage' <<<"$comments_json")" != "true" ]]; then
135+
return 1
136+
fi
137+
138+
after="$(jq -r '.data.node.comments.pageInfo.endCursor' <<<"$comments_json")"
139+
done
140+
}
141+
142+
unresolved_codex_threads=()
143+
after=""
144+
while true; do
145+
args=(
146+
-f owner="$owner"
147+
-f repo="$repo_name"
148+
-F number="$PR_NUMBER"
149+
-f query="$threads_query"
150+
)
151+
if [[ -n "$after" ]]; then
152+
args+=(-f after="$after")
153+
fi
154+
155+
threads_json="$(gh api graphql "${args[@]}")"
156+
157+
while IFS= read -r thread_json; do
158+
thread_id="$(jq -r '.id' <<<"$thread_json")"
159+
first_codex_url="$(jq -r '.firstCodexUrl // empty' <<<"$thread_json")"
160+
has_more_comments="$(jq -r '.hasMoreComments' <<<"$thread_json")"
161+
162+
if [[ -n "$first_codex_url" ]]; then
163+
unresolved_codex_threads+=("$first_codex_url")
164+
continue
165+
fi
166+
167+
if [[ "$has_more_comments" == "true" ]]; then
168+
if url="$(find_codex_comment_url "$thread_id")"; then
169+
unresolved_codex_threads+=("$url")
170+
fi
171+
fi
172+
done < <(
173+
jq -c --argjson codexLogins "$codex_logins" '
174+
.data.repository.pullRequest.reviewThreads.nodes[]
175+
| select(.isResolved == false)
176+
| {
177+
id,
178+
firstCodexUrl: (
179+
[
180+
.comments.nodes[]?
181+
| select(.author.login as $login | $codexLogins | index($login))
182+
| .url
183+
][0] // ""
184+
),
185+
hasMoreComments: .comments.pageInfo.hasNextPage
186+
}
187+
' <<<"$threads_json"
188+
)
189+
190+
if [[ "$(jq -r '.data.repository.pullRequest.reviewThreads.pageInfo.hasNextPage' <<<"$threads_json")" != "true" ]]; then
191+
break
192+
fi
193+
194+
after="$(jq -r '.data.repository.pullRequest.reviewThreads.pageInfo.endCursor' <<<"$threads_json")"
195+
done
196+
197+
if ((${#unresolved_codex_threads[@]} > 0)); then
198+
{
199+
echo "## Resolve Codex review threads"
200+
echo
201+
echo "Codex has unresolved review feedback on this PR. Fix or intentionally resolve every thread before this gate can pass."
202+
echo
203+
printf -- '- %s\n' "${unresolved_codex_threads[@]}"
204+
} >>"$GITHUB_STEP_SUMMARY"
205+
206+
exit 1
207+
fi
208+
209+
reviews_json="$(
210+
gh api --paginate --slurp \
211+
"repos/${REPO}/pulls/${PR_NUMBER}/reviews"
212+
)"
213+
214+
matching_review="$(
215+
jq --arg head "$head_sha" -c '
216+
[
217+
.[][]
218+
| select(
219+
(.user.login == "chatgpt-codex-connector[bot]" or .user.login == "chatgpt-codex-connector")
220+
and .commit_id == $head
221+
and (.state == "COMMENTED" or .state == "APPROVED" or .state == "CHANGES_REQUESTED")
222+
)
223+
][0] // empty
224+
' <<<"$reviews_json"
225+
)"
226+
227+
if [[ -n "$matching_review" ]]; then
228+
reviewer="$(jq -r '.user.login' <<<"$matching_review")"
229+
state="$(jq -r '.state' <<<"$matching_review")"
230+
submitted_at="$(jq -r '.submitted_at' <<<"$matching_review")"
231+
echo "Found ${state} review from ${reviewer} for ${head_sha} at ${submitted_at}."
232+
exit 0
233+
fi
234+
235+
comments_json="$(
236+
gh api --paginate --slurp \
237+
"repos/${REPO}/issues/${PR_NUMBER}/comments"
238+
)"
239+
240+
matching_comment="$(
241+
jq --arg head "$head_sha" -c '
242+
[
243+
.[][]
244+
| select(
245+
(.user.login == "chatgpt-codex-connector[bot]" or .user.login == "chatgpt-codex-connector")
246+
and (.body | test("Codex Review|Reviewed commit"; "i"))
247+
and (.body | contains($head))
248+
)
249+
][0] // empty
250+
' <<<"$comments_json"
251+
)"
252+
253+
if [[ -n "$matching_comment" ]]; then
254+
commenter="$(jq -r '.user.login' <<<"$matching_comment")"
255+
created_at="$(jq -r '.created_at' <<<"$matching_comment")"
256+
echo "Found Codex review comment from ${commenter} for ${head_sha} at ${created_at}."
257+
exit 0
258+
fi
259+
260+
matching_requested_comment="$(
261+
jq --arg head "$head_sha" -c '
262+
def is_codex:
263+
.user.login == "chatgpt-codex-connector[bot]"
264+
or .user.login == "chatgpt-codex-connector";
265+
def is_review_request:
266+
.body | test("@codex[[:space:]]+review"; "i");
267+
def is_no_findings:
268+
is_codex
269+
and (.body | test("Codex Review"; "i"))
270+
and (.body | test("didn.?t find any major issues"; "i"));
271+
272+
[ .[][] ] as $comments
273+
| [
274+
range(0; $comments | length) as $i
275+
| $comments[$i]
276+
| select(is_no_findings)
277+
| . as $response
278+
| (
279+
[
280+
range(0; $i) as $j
281+
| $comments[$j]
282+
| select((is_codex | not) and is_review_request)
283+
]
284+
| last
285+
) as $request
286+
| select($request != null and ($request.body | contains($head)))
287+
| {request: $request, response: $response}
288+
][-1] // empty
289+
' <<<"$comments_json"
290+
)"
291+
292+
if [[ -n "$matching_requested_comment" ]]; then
293+
commenter="$(jq -r '.response.user.login' <<<"$matching_requested_comment")"
294+
created_at="$(jq -r '.response.created_at' <<<"$matching_requested_comment")"
295+
request_created_at="$(jq -r '.request.created_at' <<<"$matching_requested_comment")"
296+
echo "Found no-findings Codex response from ${commenter} for ${head_sha} at ${created_at}, after a SHA-specific request at ${request_created_at}."
297+
exit 0
298+
fi
299+
300+
{
301+
echo "## Codex review required"
302+
echo
303+
echo "No review from \`chatgpt-codex-connector[bot]\` was found for PR #${PR_NUMBER} at \`${head_sha}\`."
304+
echo
305+
echo "Wait for automatic Codex review or comment \`@codex review head ${head_sha}\`, then rerun this check if GitHub does not rerun it automatically. A no-findings Codex comment only satisfies this gate when the latest review request before it names the exact head SHA."
306+
} >>"$GITHUB_STEP_SUMMARY"
307+
308+
exit 1

‎AGENTS.md‎

Lines changed: 30 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -41,9 +41,36 @@ If the shared checkout already has unrelated edits, name the paths and the one
4141
line summary of what they appear to be doing before creating the new worktree.
4242
Avoid stashing operator work out of the way.
4343

44-
After local checks pass, push the branch and open a PR targeting `main`. Enable
45-
auto-merge when branch protection and review state allow it. Remove the worktree
46-
and delete the local branch after the PR has merged.
44+
After local checks pass, push the branch and open a PR targeting `main`. Watch
45+
required checks with `gh pr checks --watch --fail-fast`; if a check fails,
46+
inspect the run logs, fix the branch, push again, and keep watching until the PR
47+
is green.
48+
49+
`gh pr checks` may show stale failed runs next to newer passing reruns for the
50+
same check name. When the output is mixed, inspect
51+
`gh pr view --json mergeStateStatus,statusCheckRollup,latestReviews` and trust
52+
the latest run for the current head SHA rather than the oldest failure in the
53+
list.
54+
55+
Treat PR comments and reviews as part of the work. Read them with
56+
`gh pr view --comments` and the review fields from `gh pr view --json reviews`.
57+
Address Codex comments in code when they identify a real issue, reply when a
58+
comment is intentionally declined, and resolve review threads before enabling
59+
auto-merge. Enable auto-merge only after required checks pass and required
60+
review state is clear.
61+
62+
Unresolved Codex review threads are immediate blockers. Do not wait on more
63+
checks when Codex has left an open thread: fix the code or resolve the thread
64+
with the GitHub review-thread API, then request a fresh Codex review if the head
65+
changed. If the head did not change and GitHub does not rerun the failed gate,
66+
rerun it with `gh run rerun <run-id> --failed`.
67+
68+
When manually triggering Codex, include the full current head SHA in the request,
69+
for example `@codex review head <sha>`. This gives no-findings responses a
70+
specific revision to answer. Avoid sending a later generic `@codex review` for
71+
the same head because it weakens that audit trail.
72+
73+
Remove the worktree and delete the local branch after the PR has merged.
4774

4875
Commit one logical change at a time. Use the pathspec form so unrelated staged
4976
or unstaged files cannot ride along:

0 commit comments

Comments
 (0)