ci: Add build step to run-cluster (Eventual-Inc#3606)

# Overview

The `run-cluster` workflow now also builds the commit by calling the
`build-commit` workflow first.

Author: raunakab

.github/workflows/build-commit.yaml

@@ -1,6 +1,24 @@
 name: build-commit
 
 on:
+  workflow_call:
+    inputs:
+      arch:
+        description: The machine architecture to build for
+        required: true
+        type: string
+      python_version:
+        description: The version of Python to use
+        required: true
+        type: string
+    secrets:
+      ACTIONS_AWS_ROLE_ARN:
+        description: The AWS Role ARN for secrets access
+        required: true
+    outputs:
+      wheel_url:
+        description: The name of the built wheel
+        value: ${{ jobs.build-commit.outputs.wheel_url }}
   workflow_dispatch:
     inputs:
       arch:
@@ -24,6 +42,8 @@ jobs:
     permissions:
       id-token: write
       contents: read
+    outputs:
+      wheel_url: ${{ steps.set-wheel-name.outputs.wheel_url }}
     steps:
     - name: Set platform substring
       run: |
@@ -96,6 +116,7 @@ jobs:
         )
         echo "wheel_name=$wheel_name" >> $GITHUB_ENV
     - name: Print url of the built wheel to GitHub Actions Summary Page
+      id: set-wheel-name
       run: |
         console_url="https://us-west-2.console.aws.amazon.com/s3/object/github-actions-artifacts-bucket?prefix=builds/${{ github.sha }}/$wheel_name"
         download_url="https://github-actions-artifacts-bucket.s3.us-west-2.amazonaws.com/builds/${{ github.sha }}/$wheel_name"
@@ -105,3 +126,6 @@ jobs:
 
         echo "Directly download the wheel here:" >> $GITHUB_STEP_SUMMARY
         echo "$download_url" >> $GITHUB_STEP_SUMMARY
+        echo "wheel_url=$download_url" >> $GITHUB_OUTPUT
+      env:
+        wheel_url: ${{ env.wheel_url }}

.github/workflows/run-cluster.yaml

@@ -45,12 +45,25 @@ on:
         default: ""
 
 jobs:
+  build-commit:
+    uses: ./.github/workflows/build-commit.yaml
+    if: ${{ inputs.daft_version == '' && inputs.daft_wheel_url == '' }}
+    with:
+      arch: x86
+      python_version: ${{ inputs.python_version }}
+    secrets:
+      ACTIONS_AWS_ROLE_ARN: ${{ secrets.ACTIONS_AWS_ROLE_ARN }}
+
   run-command:
     runs-on: [self-hosted, linux, x64, ci-dev]
-    timeout-minutes: 15 # Remove for ssh debugging
+    # If both the `daft-version` and `daft-wheel-url` parameters are not specified, the `build-commit` job is entirely skipped.
+    # We still want to run this job, even if `build-commit` is skipped.
+    # The `always()` guarantees that this job is always run.
+    if: always()
     permissions:
       id-token: write
       contents: read
+    needs: build-commit
     steps:
     - name: Log workflow inputs
       run: echo "${{ toJson(github.event.inputs) }}"
@@ -80,7 +93,7 @@ jobs:
             --python 3.12 \
             .github/ci-scripts/templatize_ray_config.py \
             --cluster-name="ray-ci-run-${{ github.run_id }}_${{ github.run_attempt }}" \
-            --daft-wheel-url='${{ inputs.daft_wheel_url }}' \
+            --daft-wheel-url='${{ needs.build-commit.outputs.wheel_url || inputs.daft_wheel_url || '' }}' \
             --daft-version='${{ inputs.daft_version }}' \
             --python-version='${{ inputs.python_version }}' \
             --cluster-profile='${{ inputs.cluster_profile }}' \