Publish to PyPI #1
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| --- | |
| name: Publish to PyPI | |
| # Publishes an ALREADY-TAGGED release to Soda PyPI and public PyPI. | |
| # | |
| # Why this exists, separate from release.yaml: | |
| # release.yaml ("one-button release") resolves a version, tags it, builds, | |
| # publishes, and bumps the next prerelease — all in one shot, and it refuses | |
| # to run against a tag that already exists. Releases are now driven by | |
| # release-buddy (bump -> PR -> merge -> tag -> GitHub Release), so the tag | |
| # already exists by the time we want to publish, and release.yaml can't be | |
| # used. As a result nothing published to PUBLIC PyPI between v4.7.0 | |
| # (2026-04-17) and v4.14.0 — only dev PyPI kept flowing. See incident notes. | |
| # | |
| # This workflow only BUILDS A GIVEN REF AND UPLOADS. It never tags, bumps, or | |
| # creates releases. | |
| # | |
| # Triggers: | |
| # push: tags (v*) -> standard path. The release tag is the canonical | |
| # "this version exists" signal and is always pushed | |
| # by release-buddy, so we key off the tag directly. | |
| # (A GitHub Release is best-effort — release-buddy | |
| # tolerates `gh release create` failing — so keying | |
| # off `release: published` could silently skip a | |
| # publish, the very gap this closes. A tag can't be | |
| # missed.) | |
| # workflow_dispatch -> backfill / recovery for an already-existing tag | |
| # (e.g. v4.14.0, or the 4.8.0-4.13.0 backlog); a tag | |
| # pushed before this workflow existed won't re-fire. | |
| # | |
| # twine runs with --skip-existing, so re-runs are safe and the automatic path | |
| # does not fail when a registry already has the version (e.g. Soda PyPI). | |
| on: | |
| push: | |
| tags: | |
| - "v*" | |
| workflow_dispatch: | |
| inputs: | |
| tag: | |
| description: "Existing tag to publish (e.g. v4.14.0)." | |
| required: true | |
| type: string | |
| dry_run: | |
| description: "Build and verify only — do not upload." | |
| required: false | |
| type: boolean | |
| default: false | |
| concurrency: | |
| group: publish-${{ inputs.tag || github.ref_name }} | |
| cancel-in-progress: false | |
| jobs: | |
| resolve: | |
| name: Resolve tag & modules | |
| runs-on: ubuntu-24.04 | |
| outputs: | |
| tag: ${{ steps.ref.outputs.tag }} | |
| version: ${{ steps.ref.outputs.version }} | |
| modules: ${{ steps.modules.outputs.modules }} | |
| steps: | |
| - name: Resolve tag | |
| id: ref | |
| run: | | |
| TAG="${{ inputs.tag || github.ref_name }}" | |
| if [ -z "$TAG" ]; then | |
| echo "::error::No tag provided."; exit 1 | |
| fi | |
| echo "tag=$TAG" >> "$GITHUB_OUTPUT" | |
| echo "version=${TAG#v}" >> "$GITHUB_OUTPUT" | |
| echo "Resolved tag '$TAG' -> version '${TAG#v}'" | |
| - uses: actions/checkout@v4 | |
| with: | |
| ref: ${{ steps.ref.outputs.tag }} | |
| fetch-depth: 0 | |
| - name: Verify tag exists | |
| run: | | |
| git rev-parse "refs/tags/${{ steps.ref.outputs.tag }}" >/dev/null 2>&1 \ | |
| || { echo "::error::Tag ${{ steps.ref.outputs.tag }} not found."; exit 1; } | |
| - name: Define release matrix | |
| id: modules | |
| run: echo "modules=$(bash scripts/release_matrix.sh)" >> "$GITHUB_OUTPUT" | |
| publish: | |
| name: Publish ${{ matrix.module }} | |
| needs: [resolve] | |
| runs-on: ubuntu-24.04 | |
| environment: production-release | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| module: ${{ fromJSON(needs.resolve.outputs.modules) }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| ref: ${{ needs.resolve.outputs.tag }} | |
| - name: Set up Python 3.10 | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: "3.10" | |
| - name: Set up UV | |
| uses: astral-sh/setup-uv@e58605a9b6da7c637471fab8847a5e5a6b8df081 # v5 | |
| - name: Get external secrets | |
| uses: aws-actions/aws-secretsmanager-get-secrets@a9a7eb4e2f2871d30dc5b892576fde60a2ecc802 # v2.0.10 | |
| env: | |
| AWS_ACCESS_KEY_ID: ${{ secrets.AWS_BUILD_ACCESS_KEY_ID }} | |
| AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_BUILD_SECRET_ACCESS_KEY }} | |
| AWS_REGION: ${{ secrets.AWS_BUILD_DEFAULT_REGION }} | |
| with: | |
| secret-ids: | | |
| PYPI_USERNAME,/soda/github/common/SODA_PYPI_CLOUD_WRITE_USERNAME | |
| PYPI_PASSWORD,/soda/github/common/SODA_PYPI_CLOUD_WRITE_PASSWORD | |
| - name: Build ${{ matrix.module }} | |
| run: | | |
| uv venv .venv | |
| source .venv/bin/activate | |
| uv pip install build twine | |
| cd ${{ matrix.module }} | |
| python3 -m build | |
| - name: Verify built artifacts match the tag version | |
| env: | |
| VERSION: ${{ needs.resolve.outputs.version }} | |
| run: | | |
| # Built artifacts MUST be the clean release version (not a .devN). | |
| # PyPI uploads are irreversible, so fail loudly on a mismatch. | |
| shopt -s nullglob | |
| MATCHED=("${{ matrix.module }}"/dist/*-"$VERSION"-*.whl "${{ matrix.module }}"/dist/*-"$VERSION".tar.gz) | |
| if [ ${#MATCHED[@]} -eq 0 ]; then | |
| echo "::error::No artifacts matching version $VERSION in ${{ matrix.module }}/dist:" | |
| ls -1 "${{ matrix.module }}"/dist || true | |
| exit 1 | |
| fi | |
| echo "Artifacts for $VERSION:"; printf ' %s\n' "${MATCHED[@]}" | |
| - name: Stagger uploads | |
| run: | | |
| # Spread concurrent matrix uploads to reduce PyPI 503s. | |
| DELAY=$(echo -n "${{ matrix.module }}" | cksum | awk '{print $1 % 91}') | |
| echo "Staggering upload by ${DELAY}s..." | |
| sleep "$DELAY" | |
| - name: Publish to Soda PyPI | |
| id: soda_pypi | |
| if: ${{ !inputs.dry_run }} | |
| env: | |
| TWINE_REPOSITORY_URL: ${{ vars.CLOUD_PYPI_REPOSITORY }} | |
| TWINE_USERNAME: ${{ env.PYPI_USERNAME }} | |
| TWINE_PASSWORD: ${{ env.PYPI_PASSWORD }} | |
| run: | | |
| source .venv/bin/activate | |
| RESULT="failure" | |
| for attempt in 1 2 3; do | |
| if twine upload --skip-existing "${{ matrix.module }}"/dist/*; then | |
| RESULT="success"; echo "Soda PyPI upload succeeded on attempt $attempt"; break | |
| fi | |
| if [ "$attempt" -lt 3 ]; then | |
| DELAY=$((attempt * 15)) | |
| echo "::warning::Soda PyPI upload failed (attempt $attempt/3), retrying in ${DELAY}s..." | |
| sleep "$DELAY" | |
| else | |
| echo "::error::Soda PyPI upload failed after 3 attempts" | |
| fi | |
| done | |
| echo "result=$RESULT" >> "$GITHUB_OUTPUT" | |
| - name: Publish to public PyPI | |
| id: public_pypi | |
| if: ${{ !inputs.dry_run }} | |
| env: | |
| TWINE_USERNAME: __token__ | |
| TWINE_PASSWORD: ${{ secrets.PYPI_API_TOKEN }} | |
| run: | | |
| source .venv/bin/activate | |
| RESULT="failure" | |
| for attempt in 1 2 3; do | |
| if twine upload --skip-existing "${{ matrix.module }}"/dist/*; then | |
| RESULT="success"; echo "Public PyPI upload succeeded on attempt $attempt"; break | |
| fi | |
| if [ "$attempt" -lt 3 ]; then | |
| DELAY=$((attempt * 15)) | |
| echo "::warning::Public PyPI upload failed (attempt $attempt/3), retrying in ${DELAY}s..." | |
| sleep "$DELAY" | |
| else | |
| echo "::error::Public PyPI upload failed after 3 attempts" | |
| fi | |
| done | |
| echo "result=$RESULT" >> "$GITHUB_OUTPUT" | |
| - name: Report status | |
| if: ${{ always() && !inputs.dry_run }} | |
| run: | | |
| echo "### ${{ matrix.module }} @ ${{ needs.resolve.outputs.tag }}" >> "$GITHUB_STEP_SUMMARY" | |
| echo "- Soda PyPI: ${{ steps.soda_pypi.outputs.result }}" >> "$GITHUB_STEP_SUMMARY" | |
| echo "- Public PyPI: ${{ steps.public_pypi.outputs.result }}" >> "$GITHUB_STEP_SUMMARY" | |
| if [ "${{ steps.soda_pypi.outputs.result }}" != "success" ] || [ "${{ steps.public_pypi.outputs.result }}" != "success" ]; then | |
| echo "::error::${{ matrix.module }} did not publish cleanly to both registries — see logs." | |
| exit 1 | |
| fi |