build_and_release.yml

name: build_and_release

on:
  push:
    branches:
      - trunk
    tags:
      - v*
    paths-ignore:
      - '**/*.md'
      - 'docs/**'
      - 'README.md'
      - 'Makefile'
      - 'CONTRIBUTING.md'
      - 'SECURITY.md'
      - 'LICENSE'
      - '.github/**'
      - 'version.txt'
      - '.schema/**'
      - '.vscode/**'
      - 'deploy/**'
      - 'install/**'
      - 'media/**'
      - 'monitoring/**'
      - 'acknowledgements.md'
      - 'Dockerfile*'

  workflow_dispatch:
    inputs:
      platform_option:
        description: 'Which option do you want to run on? (Default is ALL)'
        required: false
        type: choice
        options:
          - 'all'
          - 'Linux x64'
          - 'Linux aarch64'
          - 'macOS aarch64 (Apple Silicon)'
          - 'Windows x64'
          - 'macOS x64 (Intel)'
        default: 'all'
      tag_option:
        description: 'Which build tag to include? (Default is all available tags per platform)'
        required: false
        type: choice
        options:
          - 'all'
          - 'default'
          - 'odbc'
          - 'metal'
        default: 'all'
      profile_option:
        description: 'Which Rust build profile to use? (Default is release)'
        required: false
        type: choice
        options:
          - 'release'
          - 'release-lto'
        default: 'release'

env:
  # CI performance optimizations
  CARGO_REGISTRIES_CRATES_IO_PROTOCOL: sparse
  CARGO_NET_RETRY: 10
  CARGO_HTTP_TIMEOUT: 60
  CARGO_INCREMENTAL: 0
  CARGO_NET_GIT_FETCH_WITH_CLI: true

jobs:
  setup-matrix:
    name: Setup strategy matrix
    runs-on: ubuntu-24.04
    outputs:
      matrix: ${{ steps.setup-matrix.outputs.result }}

    steps:
      - name: Set up matrix
        uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
        id: setup-matrix
        with:
          script: |
            // Check if this is a tagged release (uses GitHub hosted runners, no minio)
            const isTaggedRelease = context.ref.startsWith('refs/tags/v');

            // Check if this is a release branch (uses spiceai runners with minio)
            const isReleaseBranch = context.ref.startsWith('refs/heads/release-') ||
                                    context.ref.startsWith('refs/heads/release/');

            // Check if this is a manual workflow dispatch
            const isManualDispatch = context.eventName === 'workflow_dispatch';

            // Include Windows and Linux ARM only for tagged releases, release branches, or manual dispatch (exclude trunk pushes)
            const includeWindowsAndLinuxArm = isTaggedRelease || isReleaseBranch || isManualDispatch;

            // Tagged releases use GitHub hosted runners, others use spiceai runners
            const macosRunner = isTaggedRelease ? "macos-15" : "spiceai-macos";
            const linuxRunner = isTaggedRelease ? "ubuntu-24.04" : "spiceai-dev-runners";

            // Use minio for spiceai runners (trunk, release branches, manual dispatch on non-tagged)
            const useMinio = !isTaggedRelease;

            const matrix = [
              {
                name: "Linux x64",
                runner: linuxRunner,
                target_os: "linux",
                target_arch: "x86_64",
                tags: ["default", "odbc"],
                use_minio: useMinio,
                is_tagged_release: isTaggedRelease,
              }, {
                name: "macOS aarch64 (Apple Silicon)",
                runner: macosRunner,
                target_os: "darwin",
                target_arch: "aarch64",
                tags: ["default", "metal"],
                use_minio: false,
                is_tagged_release: isTaggedRelease,
              }
            ];

            // Add Linux ARM and Windows only for tagged releases, release branches, or manual dispatch
            if (includeWindowsAndLinuxArm) {
              matrix.push({
                name: "Linux aarch64",
                runner: isTaggedRelease ? "hosted-linux-arm-runner" : "hosted-linux-arm-runner",
                target_os: "linux",
                target_arch: "aarch64",
                tags: ["default"],
                use_minio: false,
                is_tagged_release: isTaggedRelease,
              });
              matrix.push({
                name: "Windows x64",
                runner: "windows-latest",
                target_os: "windows",
                target_arch: "x86_64",
                tags: ["default"],
                use_minio: false,
                is_tagged_release: isTaggedRelease,
              });
            }

            // If running via workflow_dispatch, filter based on inputs
            if (isManualDispatch) {
              const platform_option = context.payload.inputs.platform_option;
              const tag_option = context.payload.inputs.tag_option;

              // Filter by platform
              let filtered = matrix;
              if (platform_option !== 'all') {
                filtered = filtered.filter(m => m.name === platform_option);
              }

              // Filter by tag
              if (tag_option !== 'all') {
                filtered = filtered.map(m => ({
                  ...m,
                  tags: m.tags.filter(t => t === tag_option)
                }));
                // Remove entries that have no matching tags
                filtered = filtered.filter(m => m.tags.length > 0);
              }

              return filtered;
            }

            return matrix;

  build:
    name: Build ${{ matrix.target.name }} binaries
    runs-on: ${{ matrix.target.runner }}
    needs: setup-matrix
    env:
      RUST_PROFILE: ${{ github.event.inputs.profile_option || 'release' }}
      CARGO_TERM_COLOR: always
      CARGO_INCREMENTAL: 0
      CARGO_NET_GIT_FETCH_WITH_CLI: true

    strategy:
      matrix:
        target: ${{ fromJson(needs.setup-matrix.outputs.matrix) }}

    steps:
      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
        with:
          persist-credentials: false

      - name: Set REL_VERSION from version.txt
        run: python3 ./.github/scripts/get_release_version.py

      - name: Install Visual Studio Build Tools
        if: matrix.target.target_os == 'windows'
        uses: ./.github/actions/install-vs-buildtools

      - name: Set up Rust
        uses: ./.github/actions/setup-rust

      - name: Set up make
        if: matrix.target.target_os == 'linux'
        uses: ./.github/actions/setup-make
        with:
          os: ${{ matrix.target.target_os }}

      - name: Set up cc
        if: matrix.target.target_os == 'linux'
        uses: ./.github/actions/setup-cc

      # The aarch64 runner does not have any tools pre-installed
      - name: Install missing tools (Linux aarch64)
        if: matrix.target.target_os == 'linux' && matrix.target.target_arch == 'aarch64'
        run: |
          sudo apt-get update
          sudo apt-get install build-essential libssl-dev pkg-config cmake unixodbc unixodbc-dev unzip -y

      # The x86_64 runner does not unixodbc pre-installed
      - name: Install missing tools (Linux x86_64)
        if: matrix.target.target_os == 'linux' && matrix.target.target_arch == 'x86_64'
        run: |
          sudo apt-get install unixodbc unixodbc-dev -y

      - name: Install missing tools (Mac)
        if: matrix.target.target_os == 'darwin'
        run: |
          brew update
          brew list cmake || brew install cmake
          brew install unixodbc
          echo "RUSTFLAGS=-L /opt/homebrew/lib" >> $GITHUB_ENV

      - name: Install protoc
        uses: ./.github/actions/install-protoc

      # Default flavor - build both spiced and spice CLI together (runtime not built on Windows - use WSL)
      # Non-release builds include postgres-accel for benchmark testing
      - name: Build spiced and spice CLI
        if: contains(matrix.target.tags, 'default') && matrix.target.target_os != 'windows' && !matrix.target.is_tagged_release
        working-directory: bin/spiced
        run: cargo build --profile ${{ env.RUST_PROFILE }} --features release,models,postgres-accel -p spiced -p spice --target-dir ../../target

      # Release builds exclude optional accelerators from the distributed binary
      - name: Build spiced and spice CLI (release)
        if: contains(matrix.target.tags, 'default') && matrix.target.target_os != 'windows' && matrix.target.is_tagged_release
        working-directory: bin/spiced
        run: cargo build --profile ${{ env.RUST_PROFILE }} --features release,models -p spiced -p spice --target-dir ../../target

      - name: tar binary
        if: matrix.target.target_os != 'windows' && contains(matrix.target.tags, 'default')
        run: |
          mv target/release/spiced spiced
          chmod +x spiced
          tar czf spiced_${{ matrix.target.target_os }}_${{ matrix.target.target_arch }}.tar.gz spiced

      - name: Print version
        if: matrix.target.target_os != 'windows' && contains(matrix.target.tags, 'default')
        run: ./spiced --version

      - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
        if: matrix.target.target_os != 'windows' && contains(matrix.target.tags, 'default')
        with:
          name: spiced_${{ matrix.target.target_os }}_${{ matrix.target.target_arch }}
          path: spiced_${{ matrix.target.target_os }}_${{ matrix.target.target_arch }}.tar.gz

      - name: Upload spiced to Minio
        if: matrix.target.use_minio && matrix.target.target_os == 'linux' && matrix.target.target_arch == 'x86_64' && contains(matrix.target.tags, 'default')
        uses: ./.github/actions/upload-to-minio
        with:
          minio_endpoint: ${{ secrets.TEST_MINIO_ENDPOINT }}
          minio_access_key: ${{ secrets.TEST_MINIO_ACCESS_KEY }}
          minio_secret_key: ${{ secrets.TEST_MINIO_SECRET_KEY }}
          source_path: spiced_linux_x86_64.tar.gz
          destination_path: spice-minio/artifacts/spiced/linux-x86_64/${{ github.sha }}/

      ## ODBC variant - for internal testing
      - name: Build spiced (odbc)
        if: matrix.target.target_os != 'windows' && contains(matrix.target.tags, 'odbc')
        working-directory: bin/spiced
        run: cargo build --release --features release,odbc,models --target-dir ../../target

      - name: tar binary (odbc)
        if: matrix.target.target_os != 'windows' && contains(matrix.target.tags, 'odbc')
        run: |
          mv target/release/spiced spiced
          chmod +x spiced
          tar czf spiced_odbc_${{ matrix.target.target_os }}_${{ matrix.target.target_arch }}.tar.gz spiced

      - name: Print version (odbc)
        if: matrix.target.target_os != 'windows' && contains(matrix.target.tags, 'odbc')
        run: ./spiced --version

      - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
        if: matrix.target.target_os != 'windows' && contains(matrix.target.tags, 'odbc')
        with:
          name: spiced_odbc_${{ matrix.target.target_os }}_${{ matrix.target.target_arch }}
          path: spiced_odbc_${{ matrix.target.target_os }}_${{ matrix.target.target_arch }}.tar.gz

      - name: Upload spiced_odbc to Minio
        if: matrix.target.use_minio && matrix.target.target_os == 'linux' && matrix.target.target_arch == 'x86_64' && contains(matrix.target.tags, 'odbc')
        uses: ./.github/actions/upload-to-minio
        with:
          minio_endpoint: ${{ secrets.TEST_MINIO_ENDPOINT }}
          minio_access_key: ${{ secrets.TEST_MINIO_ACCESS_KEY }}
          minio_secret_key: ${{ secrets.TEST_MINIO_SECRET_KEY }}
          source_path: spiced_odbc_linux_x86_64.tar.gz
          destination_path: spice-minio/artifacts/spiced/linux-x86_64/${{ github.sha }}/

      ## Metal variant (macOS with Metal GPU acceleration, models included by default)
      - name: Build spiced (metal)
        if: matrix.target.target_os == 'darwin'
        working-directory: bin/spiced
        run: cargo build --release --features release,models,metal --target-dir ../../target

      - name: tar binary (metal)
        if: matrix.target.target_os == 'darwin'
        run: |
          mv target/release/spiced spiced
          chmod +x spiced
          tar czf spiced_metal_${{ matrix.target.target_os }}_${{ matrix.target.target_arch }}.tar.gz spiced

      - name: Print version (metal)
        if: matrix.target.target_os == 'darwin'
        run: ./spiced --version

      - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
        if: matrix.target.target_os == 'darwin'
        with:
          name: spiced_metal_${{ matrix.target.target_os }}_${{ matrix.target.target_arch }}
          path: spiced_metal_${{ matrix.target.target_os }}_${{ matrix.target.target_arch }}.tar.gz

      ## CLI - package the CLI binary that was built above
      - name: tar CLI binary
        if: matrix.target.target_os != 'windows' && contains(matrix.target.tags, 'default')
        run: |
          mv target/release/spice spice
          chmod +x spice
          tar czf spice_${{ matrix.target.target_os }}_${{ matrix.target.target_arch }}.tar.gz spice

      ## Windows CLI - build and package (runtime not supported on Windows, CLI only)
      - name: Build spice CLI (Windows)
        if: matrix.target.target_os == 'windows' && contains(matrix.target.tags, 'default')
        run: cargo build --profile ${{ env.RUST_PROFILE }} -p spice --target-dir target

      - name: tar CLI binary (Windows)
        if: matrix.target.target_os == 'windows' && contains(matrix.target.tags, 'default')
        run: |
          mv target/release/spice.exe spice.exe
          tar czf spice.exe_${{ matrix.target.target_os }}_${{ matrix.target.target_arch }}.tar.gz spice.exe

      - name: Print CLI version
        if: matrix.target.target_os != 'windows' && contains(matrix.target.tags, 'default')
        run: ./spice version

      - name: Print CLI version (Windows)
        if: matrix.target.target_os == 'windows' && contains(matrix.target.tags, 'default')
        run: ./spice.exe version

      - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
        if: matrix.target.target_os != 'windows' && contains(matrix.target.tags, 'default')
        with:
          name: spice_${{ matrix.target.target_os }}_${{ matrix.target.target_arch }}
          path: spice_${{ matrix.target.target_os }}_${{ matrix.target.target_arch }}.tar.gz

      - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
        if: matrix.target.target_os == 'windows' && contains(matrix.target.tags, 'default')
        with:
          name: spice.exe_${{ matrix.target.target_os }}_${{ matrix.target.target_arch }}
          path: spice.exe_${{ matrix.target.target_os }}_${{ matrix.target.target_arch }}.tar.gz

      - name: Upload spice CLI to Minio
        if: matrix.target.use_minio && matrix.target.target_os == 'linux' && matrix.target.target_arch == 'x86_64' && contains(matrix.target.tags, 'default')
        uses: ./.github/actions/upload-to-minio
        with:
          minio_endpoint: ${{ secrets.TEST_MINIO_ENDPOINT }}
          minio_access_key: ${{ secrets.TEST_MINIO_ACCESS_KEY }}
          minio_secret_key: ${{ secrets.TEST_MINIO_SECRET_KEY }}
          source_path: spice_linux_x86_64.tar.gz
          destination_path: spice-minio/artifacts/spice/linux-x86_64/${{ github.sha }}/

  publish:
    name: Publish ${{ matrix.target_os }}-${{ matrix.target_arch }} binaries
    needs: build
    if: startswith(github.ref, 'refs/tags/v') && github.event_name != 'pull_request'
    env:
      ARTIFACT_DIR: ./release

    strategy:
      matrix:
        include:
          - target_os: linux
            target_arch: x86_64
          - target_os: linux
            target_arch: aarch64
          - target_os: darwin
            target_arch: aarch64
          - target_os: windows
            target_arch: x86_64

    runs-on: ubuntu-24.04
    steps:
      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
        with:
          persist-credentials: false

      - name: Set REL_VERSION from version.txt
        run: python3 ./.github/scripts/get_release_version.py

      - name: download artifacts - spice_${{ matrix.target_os }}_${{ matrix.target_arch }}
        if: matrix.target_os != 'windows'
        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8
        with:
          name: spice_${{ matrix.target_os }}_${{ matrix.target_arch }}
          path: ${{ env.ARTIFACT_DIR }}

      - name: download artifacts - spice_${{ matrix.target_os }}_${{ matrix.target_arch }}
        if: matrix.target_os == 'windows'
        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8
        with:
          name: spice.exe_${{ matrix.target_os }}_${{ matrix.target_arch }}
          path: ${{ env.ARTIFACT_DIR }}

      - name: download artifacts - spiced_${{ matrix.target_os }}_${{ matrix.target_arch }}
        if: matrix.target_os != 'windows'
        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8
        with:
          name: spiced_${{ matrix.target_os }}_${{ matrix.target_arch }}
          path: ${{ env.ARTIFACT_DIR }}

      # macOS metal variant
      - name: download artifacts - spiced_metal_${{ matrix.target_os }}_${{ matrix.target_arch }}
        if: matrix.target_os == 'darwin'
        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8
        with:
          name: spiced_metal_${{ matrix.target_os }}_${{ matrix.target_arch }}
          path: ${{ env.ARTIFACT_DIR }}

      - name: lists artifacts
        run: ls -l ${{ env.ARTIFACT_DIR }}

      - name: publish ${{ matrix.target_os }}/${{ matrix.target_arch }} binaries to github
        run: |
          # Parse repository to get owner and repo names
          OWNER_NAME="${GITHUB_REPOSITORY%%/*}"
          REPO_NAME="${GITHUB_REPOSITORY#*/}"
          export GITHUB_TOKEN=${{ secrets.GITHUB_TOKEN }}
          # Get the list of files
          RELEASE_ARTIFACT=(${ARTIFACT_DIR}/*)
          # Delete existing release artifact
          python ./.github/scripts/github_release.py delete \
            --owner $OWNER_NAME --repo $REPO_NAME \
            --tag "v${{ env.REL_VERSION }}" \
            ${RELEASE_ARTIFACT[*]}
          if [ "$LATEST_RELEASE" = "true" ]; then
            export RELEASE_BODY=`cat ./docs/release_notes/v${{ env.REL_VERSION }}.md`
          else
            export RELEASE_BODY="This is the release candidate ${{ env.REL_VERSION }}"
          fi
          echo "Uploading Spice.ai Binaries to GitHub Release"
          python ./.github/scripts/github_release.py upload \
            --owner $OWNER_NAME --repo $REPO_NAME \
            --tag "v${{ env.REL_VERSION }}" \
            --release-name "v${{ env.REL_VERSION }}" \
            --body "${RELEASE_BODY}" \
            --prerelease "$PRE_RELEASE" \
            ${RELEASE_ARTIFACT[*]}