Test CI scripts and workflows #245
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Build JAX Artifacts | |
| on: | |
| pull_request: | |
| branches: | |
| - main | |
| workflow_dispatch: | |
| inputs: | |
| halt-for-connection: | |
| description: 'Should this workflow run wait for a remote connection?' | |
| type: choice | |
| required: true | |
| default: 'no' | |
| options: | |
| - 'yes' | |
| - 'no' | |
| workflow_call: | |
| inputs: | |
| build_jax: | |
| description: "Should the jax artifact be built? (1 to enable, 0 to disable)" | |
| type: string | |
| required: true | |
| default: "1" | |
| build_jaxlib: | |
| description: "Should the jaxlib artifact be built? (1 to enable, 0 to disable)" | |
| type: string | |
| required: true | |
| default: "1" | |
| build_jax_cuda_plugin: | |
| description: "Should the jax-cuda-plugin artifact be built? (1 to enable, 0 to disable)" | |
| type: string | |
| required: true | |
| default: "1" | |
| build_jax_cuda_pjrt: | |
| description: "Should the jax-cuda-pjrt artifact be built? (1 to enable, 0 to disable)" | |
| type: string | |
| required: true | |
| default: "1" | |
| clone_main_xla: | |
| description: "Should latest XLA be used? (1 to enable, 0 to disable)" | |
| type: string | |
| required: true | |
| default: "0" | |
| upload_artifacts: | |
| description: "Should the artifacts be uploaded to a GCS bucket?" | |
| required: true | |
| default: false | |
| type: boolean | |
| upload_destination: | |
| description: "GCS location to where the artifacts should be uploaded" | |
| required: true | |
| default: '${{ github.workflow }}/${{ github.run_number }}/${{ github.run_attempt }}' | |
| type: string | |
| is_workflow_call: | |
| description: "Metadata variable to know whether a workflow call was made" | |
| type: string | |
| required: false | |
| default: "1" | |
| jobs: | |
| determine_artifact_matrix: | |
| runs-on: "linux-x86-n2-16" | |
| container: "us-central1-docker.pkg.dev/tensorflow-sigs/tensorflow/ml-build:latest" | |
| outputs: | |
| artifact_matrix: ${{ steps.set-matrix.outputs.artifact_matrix }} | |
| defaults: | |
| run: | |
| shell: bash | |
| steps: | |
| # Halt for testing | |
| - name: Wait For Connection | |
| uses: google-ml-infra/actions/ci_connection@main | |
| with: | |
| halt-dispatch-input: ${{ inputs.halt-for-connection }} | |
| - id: set-matrix | |
| run: | | |
| artifacts=() | |
| # Define inputs as bash variables to be able to parse them in | |
| # if conditions | |
| is_workflow_call=${{ inputs.is_workflow_call }} | |
| build_jax=${{ inputs.build_jax }} | |
| build_jaxlib=${{ inputs.build_jaxlib }} | |
| build_jax_cuda_pjrt=${{ inputs.build_jax_cuda_pjrt }} | |
| build_jax_cuda_plugin=${{ inputs.build_jax_cuda_plugin }} | |
| # Build every package if not a workflow call | |
| if [[ ${is_workflow_call:-"0"} == "0" ]]; then | |
| artifacts=("'jax'" ", 'jaxlib'" ", 'jax-cuda-pjrt'" ", 'jax-cuda-plugin'") | |
| else | |
| if [[ ${build_jax:-"0"} == "1" ]]; then | |
| artifacts+="'jax'," | |
| fi | |
| if [[ ${build_jaxlib:-"0"} == "1" ]]; then | |
| artifacts+="'jaxlib'," | |
| fi | |
| if [[ ${build_jax_cuda_pjrt:-"0"} == "1" ]]; then | |
| artifacts+="'jax-cuda-pjrt'," | |
| fi | |
| if [[ ${build_jax_cuda_plugin:-"0"} == "1" ]]; then | |
| artifacts+="'jax-cuda-plugin'" | |
| fi | |
| fi | |
| echo "artifact_matrix=[${artifacts[@]}]" >> $GITHUB_OUTPUT | |
| build_artifacts: | |
| needs: determine_artifact_matrix | |
| continue-on-error: true | |
| defaults: | |
| run: | |
| # Explicitly set the shell to bash to override the default Windows environment, i.e, cmd. | |
| shell: bash | |
| strategy: | |
| matrix: | |
| runner: ["windows-x86-n2-64", "linux-x86-n2-16", "linux-arm64-t2a-48"] | |
| artifact: ${{ fromJSON(needs.determine_artifact_matrix.outputs.artifact_matrix) }} | |
| python: ["3.10", "3.11", "3.12", "3.13"] | |
| # jax-cuda-pjrt and jax are pure Python packages so they do not need to be built for each | |
| # Python version. | |
| exclude: | |
| # jax-cuda-pjrt and jax are Python version independent and do not need to be built for | |
| # each Python version | |
| # Exclude jax-cuda-pjrt | |
| - artifact: "jax-cuda-pjrt" | |
| python: "3.11" | |
| - artifact: "jax-cuda-pjrt" | |
| python: "3.12" | |
| - artifact: "jax-cuda-pjrt" | |
| python: "3.13" | |
| # Exclude jax | |
| - artifact: "jax" | |
| python: "3.11" | |
| - artifact: "jax" | |
| python: "3.12" | |
| - artifact: "jax" | |
| python: "3.13" | |
| # jax also only needs to be built once per runner | |
| - artifact: "jax" | |
| runner: "windows-x86-n2-64" | |
| - artifact: "jax" | |
| runner: "linux-arm64-t2a-48" | |
| # jax-cuda-plugin and jax-cuda-pjrt are not supported on Windows. | |
| - artifact: "jax-cuda-plugin" | |
| runner: "windows-x86-n2-64" | |
| - artifact: "jax-cuda-pjrt" | |
| runner: "windows-x86-n2-64" | |
| runs-on: ${{ matrix.runner }} | |
| container: ${{ (contains(matrix.runner, 'linux-x86') && 'us-central1-docker.pkg.dev/tensorflow-sigs/tensorflow/ml-build:latest') || | |
| (contains(matrix.runner, 'linux-arm64') && 'us-central1-docker.pkg.dev/tensorflow-sigs/tensorflow/ml-build-arm64:latest') || | |
| (contains(matrix.runner, 'windows-x86') && null) }} | |
| env: | |
| JAXCI_HERMETIC_PYTHON_VERSION: "${{ matrix.python }}" | |
| JAXCI_CLONE_MAIN_XLA: "${{ inputs.clone_main_xla }}" | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - name: Enable RBE on platforms where its supported | |
| run: | | |
| os=$(uname -s | awk '{print tolower($0)}') | |
| arch=$(uname -m) | |
| if [[ ($os == "linux" || $os =~ "msys_nt" ) && $arch == "x86_64" ]]; then | |
| echo "JAXCI_BUILD_ARTIFACT_WITH_RBE=1" >> $GITHUB_ENV | |
| fi | |
| # Halt for testing | |
| - name: Wait For Connection | |
| uses: google-ml-infra/actions/ci_connection@main | |
| with: | |
| halt-dispatch-input: ${{ inputs.halt-for-connection }} | |
| - name: Build ${{ matrix.artifact }} | |
| run: ./ci/build_artifacts.sh "${{ matrix.artifact }}" | |
| - name: Set Platform | |
| run: | | |
| echo "PLATFORM=$(uname)_$(uname -m)" >> $GITHUB_ENV | |
| - name: Upload artifacts to GCS bucket | |
| # Upload if requested and one of the artifacts was built | |
| if: inputs.upload_artifacts | |
| run: gsutil -m cp -r $(pwd)/dist gs://general-ml-ci-transient/jax-github-actions/"${{ inputs.upload_destination }}"/$PLATFORM | |