dependency-security-sync #13
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: dependency-security-sync | |
| on: | |
| schedule: | |
| - cron: '0 2 * * 0' | |
| workflow_dispatch: | |
| concurrency: | |
| group: dependency-security-sync | |
| cancel-in-progress: false | |
| permissions: | |
| contents: write | |
| pull-requests: write | |
| issues: write | |
| jobs: | |
| sync: | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 60 | |
| steps: | |
| - name: Generate GitHub App token | |
| id: app-token | |
| uses: actions/create-github-app-token@v3 | |
| with: | |
| app-id: ${{ vars.GH_APP_ID }} | |
| private-key: ${{ secrets.GH_APP_KEY }} | |
| owner: citusdata | |
| - name: Export GitHub App token to environment | |
| run: echo "GH_TOKEN=${{ steps.app-token.outputs.token }}" >> "$GITHUB_ENV" | |
| - name: Checkout the-process | |
| uses: actions/checkout@v4 | |
| with: | |
| token: ${{ steps.app-token.outputs.token }} | |
| fetch-depth: 0 | |
| - name: Checkout citus | |
| uses: actions/checkout@v4 | |
| with: | |
| repository: citusdata/citus | |
| path: citus | |
| token: ${{ steps.app-token.outputs.token }} | |
| fetch-depth: 0 | |
| - name: Setup Python | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: '3.12' | |
| - name: Install pipenv and gh auth | |
| run: | | |
| python -m pip install --upgrade pip pipenv | |
| gh auth setup-git | |
| - name: Fetch open Dependabot alerts from citus | |
| id: alerts | |
| run: | | |
| gh api --paginate -H "Accept: application/vnd.github+json" \ | |
| "/repos/citusdata/citus/dependabot/alerts?state=open&per_page=100" \ | |
| | jq -s 'add // []' > alerts.json | |
| count=$(jq 'length' alerts.json) | |
| echo "count=$count" >> "$GITHUB_OUTPUT" | |
| if [ "$count" -eq 0 ]; then | |
| echo "No open alerts found. Exiting." >> "$GITHUB_STEP_SUMMARY" | |
| fi | |
| - name: Run cross-repo sync | |
| if: steps.alerts.outputs.count != '0' | |
| id: sync | |
| run: | | |
| python -m pip install --quiet packaging | |
| python .github/scripts/security_sync.py \ | |
| --alerts alerts.json \ | |
| --citus-root "$PWD/citus" \ | |
| --summary-out sync-summary.json | |
| addressed=$(jq -r '.addressed | join(" ")' sync-summary.json) | |
| echo "addressed=$addressed" >> "$GITHUB_OUTPUT" | |
| echo "## Sync summary" >> "$GITHUB_STEP_SUMMARY" | |
| jq -r '.details[] | "- \(.overall): \(.package) -> \(.patched) (\(.scope))"' \ | |
| sync-summary.json >> "$GITHUB_STEP_SUMMARY" | |
| - name: Create or update citus PR | |
| if: steps.sync.outputs.addressed != '' | |
| env: | |
| ADDRESSED: ${{ steps.sync.outputs.addressed }} | |
| run: | | |
| cd citus | |
| git config user.name "packagingApp[bot]" | |
| git config user.email "packagingApp[bot]@users.noreply.github.com" | |
| source "$GITHUB_WORKSPACE/.github/scripts/safe_push.sh" | |
| git checkout -B automation/dependency-security-sync | |
| git add src/test/regress/Pipfile src/test/regress/Pipfile.lock .devcontainer/src/test/regress/Pipfile .devcontainer/src/test/regress/Pipfile.lock || true | |
| if git diff --cached --quiet; then | |
| echo "No citus dependency changes to commit." | |
| else | |
| git commit -m "Automate Dependabot alert security dependency sync" | |
| safe_push automation/dependency-security-sync origin/main | |
| fi | |
| pr_number=$(gh pr list --repo citusdata/citus --head automation/dependency-security-sync --state open --json number --jq '.[0].number // empty') | |
| if [ -z "$pr_number" ]; then | |
| gh pr create \ | |
| --repo citusdata/citus \ | |
| --base main \ | |
| --head automation/dependency-security-sync \ | |
| --title "Automated security dependency sync from Dependabot alerts" \ | |
| --body $'Automated weekly security sync based on open Dependabot alerts.\n\nThis PR is managed by dependency-security-sync workflow.\n\nSupersedes the individual Dependabot PRs for: '"${ADDRESSED}"$'. Those PRs are intentionally left open and should be closed when this PR merges.' \ | |
| --label dependencies \ | |
| --draft | |
| pr_number=$(gh pr list --repo citusdata/citus --head automation/dependency-security-sync --state open --json number --jq '.[0].number') | |
| fi | |
| echo "CITUS_PR=$pr_number" >> "$GITHUB_ENV" | |
| - name: Regenerate the-process requirements | |
| if: steps.alerts.outputs.count != '0' && steps.sync.outputs.addressed != '' | |
| run: | | |
| cd citus/src/test/regress | |
| base=$(pipenv requirements) | |
| dev=$(pipenv requirements --dev-only) | |
| cd "$GITHUB_WORKSPACE" | |
| ref="citusdata/citus#${CITUS_PR}" | |
| base_header=$'# generated from Citus\'s Pipfile.lock (in src/test/regress) as of '"${ref}"$'\n# using `pipenv requirements > requirements.txt`, so as to avoid the\n# need for pipenv/pyenv in this image\n\n' | |
| dev_header=$'# generated from Citus\'s Pipfile.lock (in src/test/regress) as of '"${ref}"$'\n# using `pipenv requirements --dev-only > requirements.txt`, so as to avoid the\n# need for pipenv/pyenv in this image\n\n' | |
| for f in \ | |
| circleci/images/citusupgradetester/files/etc/requirements.txt \ | |
| circleci/images/failtester/files/etc/requirements.txt \ | |
| circleci/images/pgupgradetester/files/etc/requirements.txt; do | |
| printf '%s%s\n' "$base_header" "$base" > "$f" | |
| done | |
| printf '%s%s\n' "$dev_header" "$dev" > circleci/images/stylechecker/files/etc/requirements.txt | |
| # Newer pipenv emits the canonical https://pypi.org/simple index URL; | |
| # keep the historical https://pypi.python.org/simple to avoid a noisy | |
| # one-line diff on every run. | |
| for f in \ | |
| circleci/images/citusupgradetester/files/etc/requirements.txt \ | |
| circleci/images/failtester/files/etc/requirements.txt \ | |
| circleci/images/pgupgradetester/files/etc/requirements.txt \ | |
| circleci/images/stylechecker/files/etc/requirements.txt; do | |
| sed -i 's#https://pypi.org/simple#https://pypi.python.org/simple#' "$f" | |
| done | |
| - name: Create or update the-process PR | |
| if: steps.sync.outputs.addressed != '' | |
| env: | |
| ADDRESSED: ${{ steps.sync.outputs.addressed }} | |
| run: | | |
| git config user.name "packagingApp[bot]" | |
| git config user.email "packagingApp[bot]@users.noreply.github.com" | |
| source "$GITHUB_WORKSPACE/.github/scripts/safe_push.sh" | |
| git checkout -B automation/dependency-security-sync | |
| git add circleci/images/citusupgradetester/files/etc/requirements.txt circleci/images/failtester/files/etc/requirements.txt circleci/images/pgupgradetester/files/etc/requirements.txt circleci/images/stylechecker/files/etc/requirements.txt || true | |
| if git diff --cached --quiet; then | |
| echo "No the-process dependency changes to commit." | |
| else | |
| git commit -m "Automate security requirements sync from citus alerts" | |
| safe_push automation/dependency-security-sync origin/master | |
| fi | |
| # The machine-readable marker is consumed by the post-merge workflow to | |
| # close exactly the Dependabot PRs this sync addressed -- and only once | |
| # this PR actually merges. | |
| body=$'Automated weekly requirements refresh based on open Dependabot alerts from citus.\n\nThis PR is managed by dependency-security-sync workflow.\n\n<!-- addressed: '"${ADDRESSED}"$' -->' | |
| pr_number=$(gh pr list --repo citusdata/the-process --head automation/dependency-security-sync --state open --json number --jq '.[0].number // empty') | |
| if [ -z "$pr_number" ]; then | |
| gh pr create \ | |
| --repo citusdata/the-process \ | |
| --base master \ | |
| --head automation/dependency-security-sync \ | |
| --title "Automated requirements sync for Dependabot security alerts" \ | |
| --body "$body" \ | |
| --label dependencies \ | |
| --draft | |
| pr_number=$(gh pr list --repo citusdata/the-process --head automation/dependency-security-sync --state open --json number --jq '.[0].number') | |
| else | |
| gh pr edit "$pr_number" --repo citusdata/the-process --body "$body" | |
| fi | |
| echo "THE_PROCESS_PR=$pr_number" >> "$GITHUB_ENV" | |
| - name: Point citus CI at the-process dev image | |
| if: steps.sync.outputs.addressed != '' | |
| run: | | |
| # Synced requirements ship in the-process dev images tagged -dev-<sha> | |
| # built when the automation branch is pushed. Point the citus PR at that | |
| # image so its CI validates the new deps (skip if nothing was rebuilt). | |
| if [ "$(git rev-list --count origin/master..HEAD)" -eq 0 ]; then | |
| echo "No the-process commit; leaving citus image_suffix unchanged." | |
| exit 0 | |
| fi | |
| tp_sha=$(git rev-parse --short HEAD) | |
| cd citus | |
| source "$GITHUB_WORKSPACE/.github/scripts/safe_push.sh" | |
| sed -i -E "s|(image_suffix:\s*)\"[^\"]*\"|\1\"-dev-${tp_sha}\"|" .github/workflows/build_and_test.yml | |
| if git diff --quiet -- .github/workflows/build_and_test.yml; then | |
| echo "citus image_suffix already -dev-${tp_sha}." | |
| else | |
| git add .github/workflows/build_and_test.yml | |
| git commit -m "Point CI at the-process dev image for synced requirements" | |
| safe_push automation/dependency-security-sync origin/main | |
| fi | |
| - name: Summary | |
| if: steps.alerts.outputs.count != '0' | |
| run: | | |
| { | |
| echo "## Dependency security sync" | |
| echo "- Alerts processed: ${{ steps.alerts.outputs.count }}" | |
| echo "- Addressed packages: ${{ steps.sync.outputs.addressed || 'none' }}" | |
| echo "- the-process PR: #${THE_PROCESS_PR:-n/a}" | |
| echo "- citus PR: #${CITUS_PR:-n/a}" | |
| } >> "$GITHUB_STEP_SUMMARY" |