publish.yml

name: Publish

on:
  release:
    types: [published]
  push:
    branches: [revisit-protos-ci]
  pull_request:
    paths:
      - ".github/workflows/publish.yml"
      - "maint_tools/build_default_image.py"

jobs:
  rs-controller-wheels:
    name: Build RS controller wheel (${{ matrix.target }})
    runs-on: ${{ matrix.os }}
    strategy:
      matrix:
        include:
          - target: x86_64
            os: ubuntu-latest
          - target: aarch64
            os: ubuntu-24.04-arm
          - target: aarch64-apple-darwin
            os: macos-latest
    steps:
      - uses: actions/checkout@v4
        with:
          fetch-depth: "0"
      - name: Set version from tag
        run: |
          if [[ "$GITHUB_REF" == refs/tags/v* ]]; then
            VERSION=$(echo "$GITHUB_REF" | sed 's|refs/tags/v||')
          else
            VERSION="0.0.0.dev0"
          fi
          echo "VERSION=$VERSION" >> $GITHUB_ENV
          sed "s/^version = .*/version = \"$VERSION\"/" rs_controller/pyproject.toml > tmp && mv tmp rs_controller/pyproject.toml
          echo "Set version to $VERSION"
          cat rs_controller/pyproject.toml
      - name: Build wheels
        uses: PyO3/maturin-action@v1
        with:
          target: ${{ matrix.target }}
          manylinux: auto
          args: --release --out dist -m rs_controller/Cargo.toml
          sccache: true
      - name: Upload wheels
        uses: actions/upload-artifact@v4
        with:
          name: rs-controller-wheel-${{ matrix.target }}
          path: dist/*.whl

  rs-controller-publish:
    name: Publish RS controller to PyPI
    needs: rs-controller-wheels
    runs-on: ubuntu-latest
    if: github.event_name == 'release'
    steps:
      - name: Download all wheel artifacts
        uses: actions/download-artifact@v4
        with:
          pattern: rs-controller-wheel-*
          merge-multiple: true
          path: dist/
      - name: Install twine
        run: pip install twine
      - name: Publish to PyPI
        env:
          TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }}
          TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }}
        run: |
          twine upload --verbose dist/*
      - name: Wait for PyPI availability
        run: |
          VERSION=$(echo "$GITHUB_REF" | sed 's|refs/tags/v||')
          LINK="https://pypi.org/project/flyte_controller_base/${VERSION}/"
          echo "Waiting for $LINK"
          for i in $(seq 1 60); do
            if curl -L -I -s -f "$LINK"; then
              echo "Found on PyPI: $LINK"
              exit 0
            else
              echo "Attempt $i: not yet available, retrying in 10s..."
              sleep 10
            fi
          done
          echo "ERROR: timed out waiting for PyPI"
          exit 1

  flyte-pypi:
    name: PyPI package
    needs: rs-controller-publish
    if: always() && (needs.rs-controller-publish.result == 'success' || needs.rs-controller-publish.result == 'skipped')
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
        with:
          fetch-depth: "0"
      - name: Set up Python
        uses: actions/setup-python@v1
        with:
          python-version: "3.13"
      - name: Install uv
        uses: astral-sh/setup-uv@v6
        id: setup-uv
      - name: Install dependencies
        run: |
          uv venv
          uv pip install build twine setuptools wheel
      - name: Pin flyte_controller_base version (release only)
        if: github.event_name == 'release'
        run: |
          VERSION=$(echo "$GITHUB_REF" | sed 's|refs/tags/v||')
          sed -i "s/flyte_controller_base>=2.0.0b0/flyte_controller_base==${VERSION}/" pyproject.toml
          echo "Pinned flyte_controller_base==${VERSION}"
          grep flyte_controller_base pyproject.toml
      - name: Build and publish
        run: |
          uv run python -m build --wheel --installer uv
      - name: Publish
        if: ${{ github.event_name == 'release' }}
        env:
          TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }}
          TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }}
        run: |
          uv run python -m twine upload --verbose dist/*
      - name: Sleep until pypi is available
        if: ${{ github.event_name == 'release' }}
        id: pypiwait
        run: |
          # from 'v1.2.3' get '1.2.3' and make sure it's not an empty string
          VERSION="${{ github.ref_name }}"
          VERSION="${VERSION#v}"
          if [ -z "$VERSION" ]
          then
            echo "No tagged version found, exiting"
            exit 1
          fi
          sleep 300
          LINK="https://pypi.org/project/flyte/${VERSION}/"
          for i in {1..60}; do
            result=$(curl -L -I -s -f ${LINK})
            if [ $? -eq 0 ]; then
              echo "Found pypi for $LINK"
              exit 0
            else
              echo "Did not find - Retrying in 10 seconds..."
              sleep 10
            fi
          done
          exit 1
        shell: bash
  plugin-pypi:
    name: PyPI package
    runs-on: ubuntu-latest
    strategy:
      fail-fast: false
      matrix:
        workdir:
          - "plugins/ray"
          - "plugins/spark"
          - "plugins/openai"
          - "plugins/dask"
          - "plugins/pytorch"
          - "plugins/bigquery"
          - "plugins/databricks"
          - "plugins/snowflake"
          - "plugins/sglang"
          - "plugins/vllm"
          - "plugins/wandb"
          - "plugins/polars"
          - "plugins/codegen"
          - "plugins/hitl"
          - "plugins/jsonl"
          - "plugins/anthropic"
          - "plugins/gemini"
          - "plugins/mlflow"
          - "plugins/papermill"
          - "plugins/pandera"
          - "plugins/hydra"
          - "plugins/omegaconf"
          - "plugins/huggingface"
        include:
          - workdir: "plugins/sglang"
            image-type: sglang
          - workdir: "plugins/vllm"
            image-type: vllm
    steps:
      - uses: actions/checkout@v4
        with:
          fetch-depth: "0"
      - name: Set up Python
        uses: actions/setup-python@v1
        with:
          python-version: "3.13"
      - name: Install uv
        uses: astral-sh/setup-uv@v6
        id: setup-uv
      - name: Install dependencies
        working-directory: ${{ matrix.workdir }}
        run: |
          uv venv
          uv pip install build twine setuptools wheel
      - name: Build and publish
        working-directory: ${{ matrix.workdir }}
        run: |
          uv run python -m build --wheel --installer uv
      - name: Publish
        if: ${{ github.event_name == 'release' }}
        working-directory: ${{ matrix.workdir }}
        env:
          TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }}
          TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }}
        run: |
          uv run python -m twine upload --verbose dist/*

  build-and-push-flyte-docker-images:
    needs: flyte-pypi
    name: Flyte image for Python ${{ matrix.python-version }}
    runs-on: ubuntu-latest
    strategy:
      matrix:
        python-version:
          - "3.10"
          - "3.11"
          - "3.12"
          - "3.13"
          - "3.14"
    steps:
      - uses: actions/checkout@v4
        with:
          fetch-depth: "0"
      - name: Set up QEMU
        uses: docker/setup-qemu-action@v3
      - name: Set up Docker Buildx
        id: buildx
        uses: docker/setup-buildx-action@v3
      - name: Login to GitHub Container Registry
        uses: docker/login-action@v3
        with:
          registry: ghcr.io/flyteorg
          username: "${{ secrets.FLYTE_BOT_USERNAME }}"
          password: "${{ secrets.FLYTE_BOT_PAT }}"
      - name: Fetch the code
        uses: actions/checkout@v4
      - name: Set up Python ${{ matrix.python-version }}
        uses: ./.github/actions/setup-python-env
        with:
          python-version: "${{ matrix.python-version }}"
      - name: Install dependencies
        run: |
          uv venv
          uv pip install build twine setuptools wheel
          uv pip freeze
      - name: Build wheel
        run: |
          make dist
      - name: Build and push the flyte image
        env:
          FLYTE_DOCKER_BUILDER_CACHE_FROM: "type=gha"
          FLYTE_DOCKER_BUILDER_CACHE_TO: "type=gha,mode=max"
        run: |
          uv run python maint_tools/build_default_image.py --type flyte
      - name: Build and push the connector image
        env:
          FLYTE_DOCKER_BUILDER_CACHE_FROM: "type=gha"
          FLYTE_DOCKER_BUILDER_CACHE_TO: "type=gha,mode=max"
        run: |
          uv run python maint_tools/build_default_image.py --type connector