|
| 1 | +# Copybara Sync Gate (OSS) |
| 2 | +# |
| 3 | +# Prevents merging OSS PRs while unsynced internal→OSS changes exist. |
| 4 | +# Without this gate, the Copybara OSS→internal sync of this commit would |
| 5 | +# overwrite those internal changes, silently reverting them. |
| 6 | +# |
| 7 | +# Required status check: the job name "copybara-sync-gate". |
| 8 | +# |
| 9 | +# Works with GitHub merge queue: the pull_request trigger provides a |
| 10 | +# passthrough (instant success) so the PR can enter the merge queue. The |
| 11 | +# real check runs on the merge_group trigger at merge time, polling for |
| 12 | +# up to 10 minutes if the sync is behind. |
| 13 | +# |
| 14 | +# Prerequisites: |
| 15 | +# - Repository secret ELEMENTL_DEVTOOLS_PAT: a GitHub PAT with read |
| 16 | +# access to dagster-io/internal (needed to check internal commit history). |
| 17 | + |
| 18 | +name: Copybara Sync Gate |
| 19 | + |
| 20 | +on: |
| 21 | + pull_request: |
| 22 | + types: [opened, synchronize, reopened] |
| 23 | + merge_group: |
| 24 | + |
| 25 | +permissions: |
| 26 | + contents: read |
| 27 | + |
| 28 | +jobs: |
| 29 | + copybara-sync-gate: |
| 30 | + runs-on: ubuntu-latest |
| 31 | + steps: |
| 32 | + - name: Wait for internal-to-OSS sync to catch up |
| 33 | + if: github.event_name == 'merge_group' |
| 34 | + env: |
| 35 | + GH_TOKEN: ${{ secrets.ELEMENTL_DEVTOOLS_PAT }} |
| 36 | + run: | |
| 37 | + set -euo pipefail |
| 38 | +
|
| 39 | + check_sync() { |
| 40 | + # Find the last internal commit that was synced to OSS. |
| 41 | + # Copybara labels synced commits with "Internal-RevId: <internal-hash>". |
| 42 | + LAST_SYNCED=$(gh api "repos/${{ github.repository }}/commits" \ |
| 43 | + --method GET -f sha=master -f per_page=100 \ |
| 44 | + --jq 'map(select(.commit.message | test("Internal-RevId:"))) | .[0].commit.message' \ |
| 45 | + | grep -oP 'Internal-RevId: \K[a-f0-9]+') |
| 46 | +
|
| 47 | + if [ -z "$LAST_SYNCED" ]; then |
| 48 | + echo "Could not find any Internal-RevId in the last 100 OSS commits" |
| 49 | + return 1 |
| 50 | + fi |
| 51 | +
|
| 52 | + # Check how many internal commits are ahead of the last synced one. |
| 53 | + COMPARE=$(gh api "repos/dagster-io/internal/compare/${LAST_SYNCED}...master") |
| 54 | + AHEAD=$(echo "$COMPARE" | jq '.ahead_by') |
| 55 | +
|
| 56 | + if [ "$AHEAD" -eq 0 ]; then |
| 57 | + echo "Internal-to-OSS sync is caught up" |
| 58 | + return 0 |
| 59 | + fi |
| 60 | +
|
| 61 | + # Filter out commits that originated from OSS (noops for this direction). |
| 62 | + # Then check if any remaining commits touch dagster-oss/. |
| 63 | + UNSYNCED_SHAS=$(echo "$COMPARE" | jq -r \ |
| 64 | + '[.commits[] | select(.commit.message | test("Dagster-RevId:") | not)] | .[].sha') |
| 65 | +
|
| 66 | + if [ -z "$UNSYNCED_SHAS" ]; then |
| 67 | + echo "All $AHEAD ahead commits originated from OSS — sync is effectively caught up" |
| 68 | + return 0 |
| 69 | + fi |
| 70 | +
|
| 71 | + # Check if any unsynced commits touch dagster-oss/. |
| 72 | + BLOCKING=0 |
| 73 | + for SHA in $UNSYNCED_SHAS; do |
| 74 | + TOUCHES_OSS=$(gh api "repos/dagster-io/internal/commits/$SHA" \ |
| 75 | + --jq '[.files[].filename | select(startswith("dagster-oss/"))] | length') |
| 76 | + if [ "$TOUCHES_OSS" -gt 0 ]; then |
| 77 | + BLOCKING=$((BLOCKING + 1)) |
| 78 | + fi |
| 79 | + done |
| 80 | +
|
| 81 | + if [ "$BLOCKING" -eq 0 ]; then |
| 82 | + echo "No unsynced internal commits touch dagster-oss/ — safe to merge" |
| 83 | + return 0 |
| 84 | + fi |
| 85 | +
|
| 86 | + echo "$BLOCKING unsynced internal commit(s) touch dagster-oss/" |
| 87 | + return 1 |
| 88 | + } |
| 89 | +
|
| 90 | + MAX_ATTEMPTS=30 # 10 minutes at 20s intervals |
| 91 | + for i in $(seq 1 $MAX_ATTEMPTS); do |
| 92 | + if check_sync; then |
| 93 | + echo "Sync gate passed" |
| 94 | + exit 0 |
| 95 | + fi |
| 96 | +
|
| 97 | + if [ "$i" -eq 1 ]; then |
| 98 | + echo "Waiting for copybara internal-to-public sync to complete..." |
| 99 | + fi |
| 100 | +
|
| 101 | + if [ "$i" -eq "$MAX_ATTEMPTS" ]; then |
| 102 | + echo "::error::Internal-to-OSS sync did not catch up within 10 minutes." |
| 103 | + echo "::error::Merging this PR risks having internal changes silently reverted." |
| 104 | + echo "::error::Check the 'copybara: internal to public' Buildkite pipeline." |
| 105 | + exit 1 |
| 106 | + fi |
| 107 | +
|
| 108 | + sleep 20 |
| 109 | + done |
0 commit comments