publish.yaml

---
name: Publish to PyPI

# Publishes an ALREADY-TAGGED release to Soda PyPI and public PyPI.
#
# Why this exists, separate from release.yaml:
#   release.yaml ("one-button release") resolves a version, tags it, builds,
#   publishes, and bumps the next prerelease — all in one shot, and it refuses
#   to run against a tag that already exists. Releases are now driven by
#   release-buddy (bump -> PR -> merge -> tag -> GitHub Release), so the tag
#   already exists by the time we want to publish, and release.yaml can't be
#   used. As a result nothing published to PUBLIC PyPI between v4.7.0
#   (2026-04-17) and v4.14.0 — only dev PyPI kept flowing. See incident notes.
#
# This workflow only BUILDS A GIVEN REF AND UPLOADS. It never tags, bumps, or
# creates releases.
#
# Triggers:
#   push: tags (v*)      -> standard path. The release tag is the canonical
#                           "this version exists" signal and is always pushed
#                           by release-buddy, so we key off the tag directly.
#                           (A GitHub Release is best-effort — release-buddy
#                           tolerates `gh release create` failing — so keying
#                           off `release: published` could silently skip a
#                           publish, the very gap this closes. A tag can't be
#                           missed.)
#   workflow_dispatch    -> backfill / recovery for an already-existing tag
#                           (e.g. v4.14.0, or the 4.8.0-4.13.0 backlog); a tag
#                           pushed before this workflow existed won't re-fire.
#
# Idempotency (re-runs / backfills must not fail on an already-published
# version) is handled per-registry, because the two indexes differ:
#   public PyPI        -> supports `twine upload --skip-existing`.
#   pypi.cloud.soda.io -> devpi index; does NOT support --skip-existing
#                         (`UnsupportedConfiguration: ... --skip-existing`).
#                         So we upload plain and treat an "already exists"
#                         rejection as success.
#
# Approval: only the `approve` job references the production-release
# environment, so reviewers get ONE approval request per run rather than one
# per matrix leg.

on:
  push:
    tags:
      - "v*"
  workflow_dispatch:
    inputs:
      tag:
        description: "Existing tag to publish (e.g. v4.14.0)."
        required: true
        type: string
      dry_run:
        description: "Build and verify only — do not upload."
        required: false
        type: boolean
        default: false

concurrency:
  group: publish-${{ inputs.tag || github.ref_name }}
  cancel-in-progress: false

jobs:
  resolve:
    name: Resolve tag & modules
    runs-on: ubuntu-24.04
    outputs:
      tag: ${{ steps.ref.outputs.tag }}
      version: ${{ steps.ref.outputs.version }}
      modules: ${{ steps.modules.outputs.modules }}
    steps:
      - name: Resolve tag
        id: ref
        run: |
          TAG="${{ inputs.tag || github.ref_name }}"
          if [ -z "$TAG" ]; then
            echo "::error::No tag provided."; exit 1
          fi
          echo "tag=$TAG" >> "$GITHUB_OUTPUT"
          echo "version=${TAG#v}" >> "$GITHUB_OUTPUT"
          echo "Resolved tag '$TAG' -> version '${TAG#v}'"

      - uses: actions/checkout@v4
        with:
          ref: ${{ steps.ref.outputs.tag }}
          fetch-depth: 0

      - name: Verify tag exists
        run: |
          git rev-parse "refs/tags/${{ steps.ref.outputs.tag }}" >/dev/null 2>&1 \
            || { echo "::error::Tag ${{ steps.ref.outputs.tag }} not found."; exit 1; }

      - name: Define release matrix
        id: modules
        run: echo "modules=$(bash scripts/release_matrix.sh)" >> "$GITHUB_OUTPUT"

  # Single approval gate for the whole run. Only this job carries the
  # production-release environment, so reviewers approve ONCE per run instead
  # of once per matrix leg (one email, not 14). Secrets are repo-level, so the
  # publish matrix below still has access without referencing the environment.
  approve:
    name: Approve publish
    needs: [resolve]
    runs-on: ubuntu-24.04
    environment: production-release
    steps:
      - run: echo "Publishing ${{ needs.resolve.outputs.tag }} approved."

  publish:
    name: Publish ${{ matrix.module }}
    needs: [resolve, approve]
    runs-on: ubuntu-24.04
    strategy:
      fail-fast: false
      matrix:
        module: ${{ fromJSON(needs.resolve.outputs.modules) }}
    steps:
      - uses: actions/checkout@v4
        with:
          ref: ${{ needs.resolve.outputs.tag }}

      - name: Set up Python 3.10
        uses: actions/setup-python@v5
        with:
          python-version: "3.10"

      - name: Set up UV
        uses: astral-sh/setup-uv@e58605a9b6da7c637471fab8847a5e5a6b8df081  # v5

      - name: Get external secrets
        uses: aws-actions/aws-secretsmanager-get-secrets@a9a7eb4e2f2871d30dc5b892576fde60a2ecc802  # v2.0.10
        env:
          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_BUILD_ACCESS_KEY_ID }}
          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_BUILD_SECRET_ACCESS_KEY }}
          AWS_REGION: ${{ secrets.AWS_BUILD_DEFAULT_REGION }}
        with:
          secret-ids: |
            PYPI_USERNAME,/soda/github/common/SODA_PYPI_CLOUD_WRITE_USERNAME
            PYPI_PASSWORD,/soda/github/common/SODA_PYPI_CLOUD_WRITE_PASSWORD

      - name: Build ${{ matrix.module }}
        run: |
          uv venv .venv
          source .venv/bin/activate
          uv pip install build twine
          cd ${{ matrix.module }}
          python3 -m build

      - name: Verify built artifacts match the tag version
        env:
          VERSION: ${{ needs.resolve.outputs.version }}
        run: |
          # Built artifacts MUST be the clean release version (not a .devN).
          # PyPI uploads are irreversible, so fail loudly on a mismatch.
          shopt -s nullglob
          MATCHED=("${{ matrix.module }}"/dist/*-"$VERSION"-*.whl "${{ matrix.module }}"/dist/*-"$VERSION".tar.gz)
          if [ ${#MATCHED[@]} -eq 0 ]; then
            echo "::error::No artifacts matching version $VERSION in ${{ matrix.module }}/dist:"
            ls -1 "${{ matrix.module }}"/dist || true
            exit 1
          fi
          echo "Artifacts for $VERSION:"; printf '  %s\n' "${MATCHED[@]}"

      - name: Stagger uploads
        run: |
          # Spread concurrent matrix uploads to reduce PyPI 503s.
          DELAY=$(echo -n "${{ matrix.module }}" | cksum | awk '{print $1 % 91}')
          echo "Staggering upload by ${DELAY}s..."
          sleep "$DELAY"

      - name: Publish to Soda PyPI
        id: soda_pypi
        if: ${{ !inputs.dry_run }}
        env:
          TWINE_REPOSITORY_URL: ${{ vars.CLOUD_PYPI_REPOSITORY }}
          TWINE_USERNAME: ${{ env.PYPI_USERNAME }}
          TWINE_PASSWORD: ${{ env.PYPI_PASSWORD }}
        run: |
          source .venv/bin/activate
          # pypi.cloud.soda.io (devpi) does NOT support --skip-existing, so we
          # upload plain and treat an "already exists" rejection as success to
          # keep re-runs / backfills idempotent.
          RESULT="failure"
          for attempt in 1 2 3; do
            OUT="$(twine upload "${{ matrix.module }}"/dist/* 2>&1)" && {
              echo "$OUT"; RESULT="success"
              echo "Soda PyPI upload succeeded on attempt $attempt"; break
            }
            echo "$OUT"
            if echo "$OUT" | grep -qiE "already (exists|been uploaded)|this filename has already been used|409 (conflict|client error)"; then
              RESULT="success"
              echo "Soda PyPI already has this version — treating as success."; break
            fi
            if [ "$attempt" -lt 3 ]; then
              DELAY=$((attempt * 15))
              echo "::warning::Soda PyPI upload failed (attempt $attempt/3), retrying in ${DELAY}s..."
              sleep "$DELAY"
            else
              echo "::error::Soda PyPI upload failed after 3 attempts"
            fi
          done
          echo "result=$RESULT" >> "$GITHUB_OUTPUT"

      - name: Publish to public PyPI
        id: public_pypi
        if: ${{ !inputs.dry_run }}
        env:
          TWINE_USERNAME: __token__
          TWINE_PASSWORD: ${{ secrets.PYPI_API_TOKEN }}
        run: |
          source .venv/bin/activate
          RESULT="failure"
          for attempt in 1 2 3; do
            if twine upload --skip-existing "${{ matrix.module }}"/dist/*; then
              RESULT="success"; echo "Public PyPI upload succeeded on attempt $attempt"; break
            fi
            if [ "$attempt" -lt 3 ]; then
              DELAY=$((attempt * 15))
              echo "::warning::Public PyPI upload failed (attempt $attempt/3), retrying in ${DELAY}s..."
              sleep "$DELAY"
            else
              echo "::error::Public PyPI upload failed after 3 attempts"
            fi
          done
          echo "result=$RESULT" >> "$GITHUB_OUTPUT"

      - name: Report status
        if: ${{ always() && !inputs.dry_run }}
        run: |
          echo "### ${{ matrix.module }} @ ${{ needs.resolve.outputs.tag }}" >> "$GITHUB_STEP_SUMMARY"
          echo "- Soda PyPI: ${{ steps.soda_pypi.outputs.result }}" >> "$GITHUB_STEP_SUMMARY"
          echo "- Public PyPI: ${{ steps.public_pypi.outputs.result }}" >> "$GITHUB_STEP_SUMMARY"
          if [ "${{ steps.soda_pypi.outputs.result }}" != "success" ] || [ "${{ steps.public_pypi.outputs.result }}" != "success" ]; then
            echo "::error::${{ matrix.module }} did not publish cleanly to both registries — see logs."
            exit 1
          fi