PR into #488 #268
Workflow file for this run
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Publish | |
| on: | |
| release: | |
| types: [published] | |
| push: | |
| branches: [revisit-protos-ci] | |
| pull_request: | |
| paths: | |
| - ".github/workflows/publish.yml" | |
| - "maint_tools/build_default_image.py" | |
| jobs: | |
| rs-controller-wheels: | |
| name: Build RS controller wheel (${{ matrix.target }}) | |
| runs-on: ${{ matrix.os }} | |
| strategy: | |
| matrix: | |
| include: | |
| - target: x86_64 | |
| os: ubuntu-latest | |
| - target: aarch64 | |
| os: ubuntu-24.04-arm | |
| - target: aarch64-apple-darwin | |
| os: macos-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: "0" | |
| - name: Set version from tag | |
| run: | | |
| if [[ "$GITHUB_REF" == refs/tags/v* ]]; then | |
| VERSION=$(echo "$GITHUB_REF" | sed 's|refs/tags/v||') | |
| else | |
| VERSION="0.0.0.dev0" | |
| fi | |
| echo "VERSION=$VERSION" >> $GITHUB_ENV | |
| sed "s/^version = .*/version = \"$VERSION\"/" rs_controller/pyproject.toml > tmp && mv tmp rs_controller/pyproject.toml | |
| echo "Set version to $VERSION" | |
| cat rs_controller/pyproject.toml | |
| - name: Build wheels | |
| uses: PyO3/maturin-action@v1 | |
| with: | |
| target: ${{ matrix.target }} | |
| manylinux: auto | |
| args: --release --out dist -m rs_controller/Cargo.toml | |
| sccache: true | |
| - name: Upload wheels | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: rs-controller-wheel-${{ matrix.target }} | |
| path: dist/*.whl | |
| rs-controller-publish: | |
| name: Publish RS controller to PyPI | |
| needs: rs-controller-wheels | |
| runs-on: ubuntu-latest | |
| if: github.event_name == 'release' | |
| steps: | |
| - name: Download all wheel artifacts | |
| uses: actions/download-artifact@v4 | |
| with: | |
| pattern: rs-controller-wheel-* | |
| merge-multiple: true | |
| path: dist/ | |
| - name: Install twine | |
| run: pip install twine | |
| - name: Publish to PyPI | |
| env: | |
| TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }} | |
| TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }} | |
| run: | | |
| twine upload --verbose dist/* | |
| - name: Wait for PyPI availability | |
| run: | | |
| VERSION=$(echo "$GITHUB_REF" | sed 's|refs/tags/v||') | |
| LINK="https://pypi.org/project/flyte_controller_base/${VERSION}/" | |
| echo "Waiting for $LINK" | |
| for i in $(seq 1 60); do | |
| if curl -L -I -s -f "$LINK"; then | |
| echo "Found on PyPI: $LINK" | |
| exit 0 | |
| else | |
| echo "Attempt $i: not yet available, retrying in 10s..." | |
| sleep 10 | |
| fi | |
| done | |
| echo "ERROR: timed out waiting for PyPI" | |
| exit 1 | |
| flyte-pypi: | |
| name: PyPI package | |
| needs: rs-controller-publish | |
| if: always() && (needs.rs-controller-publish.result == 'success' || needs.rs-controller-publish.result == 'skipped') | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: "0" | |
| - name: Set up Python | |
| uses: actions/setup-python@v1 | |
| with: | |
| python-version: "3.13" | |
| - name: Install uv | |
| uses: astral-sh/setup-uv@v6 | |
| id: setup-uv | |
| - name: Install dependencies | |
| run: | | |
| uv venv | |
| uv pip install build twine setuptools wheel | |
| - name: Pin flyte_controller_base version (release only) | |
| if: github.event_name == 'release' | |
| run: | | |
| VERSION=$(echo "$GITHUB_REF" | sed 's|refs/tags/v||') | |
| sed -i "s/flyte_controller_base>=2.0.0b0/flyte_controller_base==${VERSION}/" pyproject.toml | |
| echo "Pinned flyte_controller_base==${VERSION}" | |
| grep flyte_controller_base pyproject.toml | |
| - name: Build and publish | |
| run: | | |
| uv run python -m build --wheel --installer uv | |
| - name: Publish | |
| if: ${{ github.event_name == 'release' }} | |
| env: | |
| TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }} | |
| TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }} | |
| run: | | |
| uv run python -m twine upload --verbose dist/* | |
| - name: Sleep until pypi is available | |
| if: ${{ github.event_name == 'release' }} | |
| id: pypiwait | |
| run: | | |
| # from 'v1.2.3' get '1.2.3' and make sure it's not an empty string | |
| VERSION="${{ github.ref_name }}" | |
| VERSION="${VERSION#v}" | |
| if [ -z "$VERSION" ] | |
| then | |
| echo "No tagged version found, exiting" | |
| exit 1 | |
| fi | |
| sleep 300 | |
| LINK="https://pypi.org/project/flyte/${VERSION}/" | |
| for i in {1..60}; do | |
| result=$(curl -L -I -s -f ${LINK}) | |
| if [ $? -eq 0 ]; then | |
| echo "Found pypi for $LINK" | |
| exit 0 | |
| else | |
| echo "Did not find - Retrying in 10 seconds..." | |
| sleep 10 | |
| fi | |
| done | |
| exit 1 | |
| shell: bash | |
| plugin-pypi: | |
| name: PyPI package | |
| runs-on: ubuntu-latest | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| workdir: | |
| - "plugins/ray" | |
| - "plugins/spark" | |
| - "plugins/openai" | |
| - "plugins/dask" | |
| - "plugins/pytorch" | |
| - "plugins/bigquery" | |
| - "plugins/databricks" | |
| - "plugins/snowflake" | |
| - "plugins/sglang" | |
| - "plugins/vllm" | |
| - "plugins/wandb" | |
| - "plugins/polars" | |
| - "plugins/codegen" | |
| - "plugins/hitl" | |
| - "plugins/jsonl" | |
| - "plugins/anthropic" | |
| - "plugins/gemini" | |
| - "plugins/mlflow" | |
| - "plugins/papermill" | |
| - "plugins/pandera" | |
| - "plugins/hydra" | |
| - "plugins/omegaconf" | |
| - "plugins/huggingface" | |
| include: | |
| - workdir: "plugins/sglang" | |
| image-type: sglang | |
| - workdir: "plugins/vllm" | |
| image-type: vllm | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: "0" | |
| - name: Set up Python | |
| uses: actions/setup-python@v1 | |
| with: | |
| python-version: "3.13" | |
| - name: Install uv | |
| uses: astral-sh/setup-uv@v6 | |
| id: setup-uv | |
| - name: Install dependencies | |
| working-directory: ${{ matrix.workdir }} | |
| run: | | |
| uv venv | |
| uv pip install build twine setuptools wheel | |
| - name: Build and publish | |
| working-directory: ${{ matrix.workdir }} | |
| run: | | |
| uv run python -m build --wheel --installer uv | |
| - name: Publish | |
| if: ${{ github.event_name == 'release' }} | |
| working-directory: ${{ matrix.workdir }} | |
| env: | |
| TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }} | |
| TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }} | |
| run: | | |
| uv run python -m twine upload --verbose dist/* | |
| build-and-push-flyte-docker-images: | |
| needs: flyte-pypi | |
| name: Flyte image for Python ${{ matrix.python-version }} | |
| runs-on: ubuntu-latest | |
| strategy: | |
| matrix: | |
| python-version: | |
| - "3.10" | |
| - "3.11" | |
| - "3.12" | |
| - "3.13" | |
| - "3.14" | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: "0" | |
| - name: Set up QEMU | |
| uses: docker/setup-qemu-action@v3 | |
| - name: Set up Docker Buildx | |
| id: buildx | |
| uses: docker/setup-buildx-action@v3 | |
| - name: Login to GitHub Container Registry | |
| uses: docker/login-action@v3 | |
| with: | |
| registry: ghcr.io/flyteorg | |
| username: "${{ secrets.FLYTE_BOT_USERNAME }}" | |
| password: "${{ secrets.FLYTE_BOT_PAT }}" | |
| - name: Fetch the code | |
| uses: actions/checkout@v4 | |
| - name: Set up Python ${{ matrix.python-version }} | |
| uses: ./.github/actions/setup-python-env | |
| with: | |
| python-version: "${{ matrix.python-version }}" | |
| - name: Install dependencies | |
| run: | | |
| uv venv | |
| uv pip install build twine setuptools wheel | |
| uv pip freeze | |
| - name: Build wheel | |
| run: | | |
| make dist | |
| - name: Build and push the flyte image | |
| env: | |
| FLYTE_DOCKER_BUILDER_CACHE_FROM: "type=gha" | |
| FLYTE_DOCKER_BUILDER_CACHE_TO: "type=gha,mode=max" | |
| run: | | |
| uv run python maint_tools/build_default_image.py --type flyte | |
| - name: Build and push the connector image | |
| env: | |
| FLYTE_DOCKER_BUILDER_CACHE_FROM: "type=gha" | |
| FLYTE_DOCKER_BUILDER_CACHE_TO: "type=gha,mode=max" | |
| run: | | |
| uv run python maint_tools/build_default_image.py --type connector |