release.yml

name: Release

on:
  release:
    types: [published]
  workflow_dispatch:
    inputs:
      release_tag:
        description: 'Release tag to upload artifacts to (e.g. v0.1.0)'
        required: false

# Default-deny top-level permissions; each job grants only what it needs.
permissions: {}

jobs:
  # ============================================================================
  # macOS ARM64 (Apple Silicon) build
  # ============================================================================
  build-macos:
    name: Build macOS (Apple Silicon)
    runs-on: self-hosted-macos-26-arm64 # Self-hosted Apple Silicon runner with macOS 26 SDK + Metal 4
    environment: packaging
    timeout-minutes: 90
    # Block forks from invoking this workflow against the canonical self-hosted
    # runner pool. `workflow_dispatch` can be triggered from any branch the
    # actor has push access to, so this guard is the last line of defense
    # against a forked copy of the repo dispatching a build with the
    # canonical-org runner labels.
    if: github.repository == 'lablup/mlxcel'
    permissions:
      contents: write

    env:
      MACOSX_DEPLOYMENT_TARGET: "14.0"

    steps:
      - name: Checkout code
        uses: actions/checkout@v6
        with:
          # Don't leave GITHUB_TOKEN in .git/config; release jobs only fetch.
          persist-credentials: false

      # Use persistent cache paths for self-hosted runner (outside workspace)
      - name: Setup persistent cache paths (self-hosted)
        run: |
          # Set persistent cargo target directory to enable incremental builds
          # on self-hosted runner (avoids recompiling from scratch every run)
          CARGO_TARGET="$HOME/.cargo-target/mlxcel"
          mkdir -p "$CARGO_TARGET"
          echo "CARGO_TARGET_DIR=$CARGO_TARGET" >> $GITHUB_ENV

          # Prune stale cargo target cache if older than 7 days
          # Prevents unbounded disk growth from accumulated build artifacts
          MARKER="$CARGO_TARGET/.last-clean"
          SHOULD_CLEAN=false
          if [ ! -f "$MARKER" ]; then
            SHOULD_CLEAN=true
          elif [ "$(find "$MARKER" -mtime +7 2>/dev/null)" ]; then
            SHOULD_CLEAN=true
          fi
          if [ "$SHOULD_CLEAN" = true ]; then
            echo "Cargo target cache older than 7 days — cleaning"
            rm -rf "$CARGO_TARGET"
            mkdir -p "$CARGO_TARGET"
            date -u +"%Y-%m-%dT%H:%M:%SZ" > "$MARKER"
          else
            AGE_DAYS=$(( ($(date +%s) - $(stat -f %m "$MARKER")) / 86400 ))
            echo "Cargo target cache age: ${AGE_DAYS}d — keeping"
          fi

      - name: Ensure build tools
        run: |
          # cmake is required by mlxcel-core and sentencepiece-sys build scripts
          if ! command -v cmake &>/dev/null; then
            echo "cmake not found, installing via Homebrew..."
            brew install cmake
          fi
          echo "cmake: $(cmake --version | head -1)"
          echo "git: $(git --version)"

      - name: Verify build environment
        run: |
          echo "=== macOS ==="
          sw_vers
          echo ""
          echo "=== Xcode SDK ==="
          xcrun --show-sdk-version
          echo ""
          echo "=== Metal version ==="
          echo "__METAL_VERSION__" | xcrun -sdk macosx metal -E -x metal -P - | tail -1
          echo ""
          echo "=== Metal compiler ==="
          xcrun -sdk macosx metal --version

      - name: Install Rust toolchain
        uses: dtolnay/rust-toolchain@stable
        with:
          targets: aarch64-apple-darwin
          components: clippy, rustfmt

      # Quality gate — fmt / clippy / test. Used to live in ci.yml's
      # `clippy + test (macOS ARM64)` job, but consumed ~15 min of the shared
      # self-hosted runner on every PR push for limited PR-time signal. Moved
      # here so the cost is paid once per release. Local devs should run
      # `make verify` before pushing to catch regressions earlier.
      - name: Format check
        run: cargo fmt --all -- --check

      - name: Clippy
        run: cargo clippy --all-targets --features metal,accelerate -- -D warnings

      - name: Test
        run: cargo test --release --features metal,accelerate

      - name: Build release binaries
        run: cargo build --release --target aarch64-apple-darwin --locked
        env:
          RUSTFLAGS: "-C target-cpu=apple-m1"

      - name: Prepare signing certificate (rcodesign)
        env:
          APPLE_CERTIFICATE: ${{ secrets.DEV_ID_CERT_P12 }}
          APPLE_CERTIFICATE_PASSWORD: ${{ secrets.DEV_ID_CERT_PASSWORD }}
        run: |
          # Use rcodesign instead of Apple's codesign to bypass macOS 26 keychain
          # session issues on self-hosted runners. rcodesign reads PEM directly —
          # no keychain, no SecurityAgent, no GUI session required.
          echo "$APPLE_CERTIFICATE" | base64 --decode > $RUNNER_TEMP/original.p12

          # Extract to unencrypted PEM (cert + private key) using -legacy flag
          # for p12 files with legacy RC2-40-CBC encryption.
          openssl pkcs12 -in $RUNNER_TEMP/original.p12 \
            -passin "pass:$APPLE_CERTIFICATE_PASSWORD" \
            -legacy -nodes -out $RUNNER_TEMP/signing.pem 2>/dev/null

          rm -f $RUNNER_TEMP/original.p12

          if [ ! -s "$RUNNER_TEMP/signing.pem" ]; then
            echo "::error::Failed to extract certificate from p12"
            exit 1
          fi

          echo "=== Certificate details ==="
          openssl x509 -in $RUNNER_TEMP/signing.pem -noout -subject -dates 2>/dev/null || true

          echo "PEM_FILE=$RUNNER_TEMP/signing.pem" >> $GITHUB_ENV

      - name: Install rcodesign
        run: |
          RCODESIGN_VERSION="0.29.0"
          RCODESIGN_DIR="$HOME/.rcodesign"
          RCODESIGN_BIN="$RCODESIGN_DIR/rcodesign"

          if [ -f "$RCODESIGN_BIN" ] && "$RCODESIGN_BIN" --version 2>/dev/null | grep -q "$RCODESIGN_VERSION"; then
            echo "rcodesign $RCODESIGN_VERSION already installed"
          else
            echo "Installing rcodesign $RCODESIGN_VERSION..."
            mkdir -p "$RCODESIGN_DIR"
            curl -sL "https://github.com/indygreg/apple-platform-rs/releases/download/apple-codesign%2F${RCODESIGN_VERSION}/apple-codesign-${RCODESIGN_VERSION}-aarch64-apple-darwin.tar.gz" \
              | tar xz -C "$RCODESIGN_DIR" --strip-components=1
          fi
          echo "$RCODESIGN_DIR" >> $GITHUB_PATH

      - name: Code sign macOS binaries
        run: |
          BIN_DIR="$CARGO_TARGET_DIR/aarch64-apple-darwin/release"
          for BIN in mlxcel mlxcel-server; do
            echo "Signing $BIN..."
            rcodesign sign \
              --pem-file "$PEM_FILE" \
              --code-signature-flags runtime \
              "$BIN_DIR/$BIN"
          done

      - name: Locate mlx.metallib
        run: |
          METALLIB=$(find "$CARGO_TARGET_DIR/aarch64-apple-darwin/release/build/mlxcel-core-"*/out/build/lib -name "mlx.metallib" 2>/dev/null | head -1)
          if [ -z "$METALLIB" ]; then
            echo "::error::mlx.metallib not found in build output"
            exit 1
          fi
          echo "METALLIB_PATH=$METALLIB" >> $GITHUB_ENV
          echo "Found mlx.metallib at: $METALLIB ($(du -h "$METALLIB" | cut -f1))"

      - name: Package binaries
        run: |
          BIN_DIR="$CARGO_TARGET_DIR/aarch64-apple-darwin/release"
          mkdir -p package
          cp "$BIN_DIR/mlxcel" package/
          cp "$BIN_DIR/mlxcel-server" package/
          cp "$METALLIB_PATH" package/mlx.metallib
          ditto -c -k --sequesterRsrc package mlxcel-macos-aarch64.zip

      - name: Generate checksum
        run: shasum -a 256 mlxcel-macos-aarch64.zip > mlxcel-macos-aarch64.zip.sha256

      - name: Upload release artifacts
        if: github.event_name == 'release' || github.event_name == 'workflow_dispatch'
        uses: softprops/action-gh-release@v3
        with:
          tag_name: ${{ github.event.release.tag_name || github.event.inputs.release_tag }}
          files: |
            mlxcel-macos-aarch64.zip
            mlxcel-macos-aarch64.zip.sha256

  # ============================================================================
  # Linux ARM64 + CUDA builds (self-hosted GB10 runner)
  # ============================================================================
  build-linux-cuda:
    name: Build Linux ARM64 CUDA (${{ matrix.variant }})
    runs-on: GB10
    environment: packaging
    if: github.repository == 'lablup/mlxcel'
    permissions:
      contents: write

    strategy:
      fail-fast: false
      matrix:
        include:
          # NVIDIA DGX Spark / GB10 (Blackwell, SM 121)
          - variant: gb10
            cuda_arch: "121"
            asset_name: mlxcel-linux-aarch64-cuda13-gb10

          # NVIDIA GH200 (Grace Hopper, SM 90a) — cross-compiled on GB10 runner
          - variant: gh200
            cuda_arch: "90a"
            asset_name: mlxcel-linux-aarch64-cuda13-gh200

    timeout-minutes: 120

    steps:
      - name: Checkout code
        uses: actions/checkout@v6
        with:
          # Don't leave GITHUB_TOKEN in .git/config; release jobs only fetch.
          persist-credentials: false

      # Use persistent cache paths for self-hosted runner (outside workspace)
      - name: Setup persistent cache paths (self-hosted)
        run: |
          # Set persistent cargo target directory to enable incremental builds
          # on self-hosted runner (avoids recompiling from scratch every run)
          CARGO_TARGET="$HOME/.cargo-target/mlxcel-cuda-${{ matrix.variant }}"
          mkdir -p "$CARGO_TARGET"
          echo "CARGO_TARGET_DIR=$CARGO_TARGET" >> $GITHUB_ENV

          # Prune stale cargo target cache if older than 7 days
          MARKER="$CARGO_TARGET/.last-clean"
          SHOULD_CLEAN=false
          if [ ! -f "$MARKER" ]; then
            SHOULD_CLEAN=true
          elif [ "$(find "$MARKER" -mtime +7 2>/dev/null)" ]; then
            SHOULD_CLEAN=true
          fi
          if [ "$SHOULD_CLEAN" = true ]; then
            echo "Cargo target cache older than 7 days — cleaning"
            rm -rf "$CARGO_TARGET"
            mkdir -p "$CARGO_TARGET"
            date -u +"%Y-%m-%dT%H:%M:%SZ" > "$MARKER"
          else
            AGE_DAYS=$(( ($(date +%s) - $(stat -c %Y "$MARKER")) / 86400 ))
            echo "Cargo target cache age: ${AGE_DAYS}d — keeping"
          fi

      - name: Validate MLX build cache
        env:
          # Must match GIT_TAG in src/lib/mlx-cpp/CMakeLists.txt and
          # MLX_EXPECTED_COMMIT in src/lib/mlxcel-core/build.rs
          MLX_EXPECTED_COMMIT: "84961223c02925bef6bef95d3a0a046779bde935"
        run: |
          # Check every _deps directory for a valid .mlx-build-commit marker.
          # If the marker is missing or doesn't match, purge that _deps/ entirely.
          # This ensures stale MLX source/objects never survive across MLX upgrades.
          FOUND=0
          PURGED=0
          for deps_dir in $(find "$CARGO_TARGET_DIR" -path "*/_deps" -type d 2>/dev/null); do
            FOUND=$((FOUND + 1))
            MARKER="$deps_dir/.mlx-build-commit"
            if [ -f "$MARKER" ] && [ "$(cat "$MARKER")" = "$MLX_EXPECTED_COMMIT" ]; then
              echo "Valid cache: $deps_dir"
            else
              echo "Stale cache (marker=$(cat "$MARKER" 2>/dev/null || echo 'missing')): purging $deps_dir"
              rm -rf "$deps_dir"
              PURGED=$((PURGED + 1))
            fi
          done
          # Also clean workspace target/ from any previous actions/cache restore
          for deps_dir in $(find target -path "*/_deps" -type d 2>/dev/null); do
            rm -rf "$deps_dir"
            PURGED=$((PURGED + 1))
          done
          echo "Checked $FOUND cache(s), purged $PURGED"

      - name: Install Rust toolchain
        uses: dtolnay/rust-toolchain@stable

      - name: Build release binaries
        run: cargo build --release --features cuda --locked
        env:
          MLX_CUDA_ARCHITECTURES: ${{ matrix.cuda_arch }}

      - name: Package binaries
        run: |
          BIN_DIR="$CARGO_TARGET_DIR/release"
          mkdir -p package
          cp "$BIN_DIR/mlxcel" package/
          cp "$BIN_DIR/mlxcel-server" package/
          cd package && zip -r "../${{ matrix.asset_name }}.zip" . && cd ..

      - name: Generate checksum
        run: sha256sum "${{ matrix.asset_name }}.zip" > "${{ matrix.asset_name }}.zip.sha256"

      - name: Upload release artifacts
        if: github.event_name == 'release' || github.event_name == 'workflow_dispatch'
        uses: softprops/action-gh-release@v3
        with:
          tag_name: ${{ github.event.release.tag_name || github.event.inputs.release_tag }}
          files: |
            ${{ matrix.asset_name }}.zip
            ${{ matrix.asset_name }}.zip.sha256

  # ============================================================================
  # Microsoft Teams release notification (Power Automate Workflows webhook)
  # ============================================================================
  notify-teams:
    name: Notify Teams on release
    needs: [build-macos, build-linux-cuda]
    if: github.repository == 'lablup/mlxcel' && github.event_name == 'release'
    runs-on: ubuntu-latest
    permissions: {}

    steps:
      - name: Build Adaptive Card payload
        env:
          TAG:  ${{ github.event.release.tag_name }}
          NAME: ${{ github.event.release.name }}
          URL:  ${{ github.event.release.html_url }}
          BODY: ${{ github.event.release.body }}
          REPO: ${{ github.repository }}
        run: |
          TRIMMED=$(printf '%s' "$BODY" | head -c 2000)
          jq -n \
            --arg tag "$TAG" --arg name "$NAME" \
            --arg url "$URL" --arg body "$TRIMMED" --arg repo "$REPO" '
          {
            type: "message",
            attachments: [{
              contentType: "application/vnd.microsoft.card.adaptive",
              content: {
                "$schema": "http://adaptivecards.io/schemas/adaptive-card.json",
                type: "AdaptiveCard",
                version: "1.5",
                body: [
                  { type: "TextBlock", size: "Large", weight: "Bolder",
                    text: ("🚀 " + $repo + " " + $tag + " released") },
                  { type: "TextBlock", text: $name, wrap: true, isSubtle: true },
                  { type: "TextBlock", text: $body, wrap: true }
                ],
                actions: [
                  { type: "Action.OpenUrl", title: "View release", url: $url }
                ]
              }
            }]
          }' > card.json

      - name: POST to Teams workflow
        if: env.WEBHOOK_URL != ''
        env:
          WEBHOOK_URL: ${{ secrets.TEAMS_RELEASE_NOTIFICATION_WORKFLOW_URL }}
        run: |
          curl -sSf -X POST \
            -H "Content-Type: application/json" \
            --data-binary @card.json \
            "$WEBHOOK_URL"

  # ============================================================================
  # Promote pre-release to full release after all jobs succeed
  # ============================================================================
  promote-release:
    name: Promote to full release
    needs: [build-macos, build-linux-cuda]
    if: always() && github.repository == 'lablup/mlxcel' && github.event_name == 'release' && github.event.release.prerelease && needs.build-macos.result == 'success' && needs.build-linux-cuda.result == 'success'
    runs-on: ubuntu-latest
    permissions:
      contents: write

    steps:
      - name: Mark release as non-prerelease
        env:
          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
        run: |
          gh release edit "${{ github.event.release.tag_name }}" \
            --repo "${{ github.repository }}" \
            --prerelease=false