macos_benchmarks.yml

name: macOS benchmarks

permissions:
  contents: read

on:
  workflow_call:
    inputs:
      benchmark_package_path:
        type: string
        description: "Path to the directory containing the benchmarking package. Defaults to ."
        default: "."
      swift_package_arguments:
        type: string
        description: "Arguments to the switch package command invocation e.g. `--disable-sandbox`"

      macos_xcode_16_3_enabled:
        type: boolean
        description: "Boolean to enable the macOS Xcode 16.3 benchmark job. Defaults to false."
        default: false
      macos_xcode_16_4_enabled:
        type: boolean
        description: "Boolean to enable the macOS Xcode 16.4 benchmark job. Defaults to false."
        default: false
      macos_xcode_26_0_enabled:
        type: boolean
        description: "Boolean to enable the macOS Xcode 26.0 benchmark job. Defaults to false."
        default: false
      macos_xcode_26_1_enabled:
        type: boolean
        description: "Boolean to enable the macOS Xcode 26.1 benchmark job. Defaults to false."
        default: false
      macos_xcode_26_2_enabled:
        type: boolean
        description: "Boolean to enable the macOS Xcode 26.2 benchmark job. Defaults to false."
        default: false
      macos_xcode_latest_beta_enabled:
        type: boolean
        description: "Boolean to enable the macOS Xcode latest beta benchmark job. Defaults to false."
        default: false

      macos_swift_6_1_enabled:
        type: boolean
        description: "Boolean to enable the macOS Swift 6.1 benchmark job. Defaults to false."
        default: false
      macos_swift_6_2_enabled:
        type: boolean
        description: "Boolean to enable the macOS Swift 6.2 benchmark job. Defaults to false."
        default: false
      macos_swift_6_3_enabled:
        type: boolean
        description: "Boolean to enable the macOS Swift 6.3 benchmark job. Defaults to false."
        default: false

      macos_runner_pool:
        type: string
        description: "The runner pool which will be requested for macOS jobs."
        default: "nightly"

      macos_env_vars:
        type: string
        description: "Environment variables for macOS jobs as JSON (e.g., '{\"DEBUG\":\"1\",\"LOG_LEVEL\":\"info\"}')."
        default: "{}"

      minimum_swift_version:
        type: string
        description: "Minimum Swift version to include in the matrix. Leave empty for auto-detection from Package.swift, specify a version like '6.0' to override, or set to 'none' to disable filtering."
        default: ""

jobs:
  construct-matrix-macos:
    name: Construct macOS Benchmarks matrix
    runs-on: ubuntu-latest
    if: |
      inputs.macos_xcode_16_3_enabled ||
      inputs.macos_xcode_16_4_enabled ||
      inputs.macos_xcode_26_0_enabled ||
      inputs.macos_xcode_26_1_enabled ||
      inputs.macos_xcode_26_2_enabled ||
      inputs.macos_xcode_latest_beta_enabled ||
      inputs.macos_swift_6_1_enabled ||
      inputs.macos_swift_6_2_enabled ||
      inputs.macos_swift_6_3_enabled
    outputs:
      macos-matrix: '${{ steps.generate-matrix.outputs.macos-matrix }}'
    steps:
      - name: Checkout repository
        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6.0.2
        with:
          persist-credentials: false
      - id: generate-matrix
        run: |
          # Validate JSON environment variables
          macos_env_vars_json="$INPUT_MACOS_ENV_VARS"

          if ! echo "$macos_env_vars_json" | jq empty 2>/dev/null; then
            echo "Error: macos_env_vars is not valid JSON"
            exit 1
          fi

          runner_pool="${MACOS_RUNNER_POOL}"
          xcode_16_3_enabled="${MACOS_XCODE_16_3_ENABLED}"
          xcode_16_4_enabled="${MACOS_XCODE_16_4_ENABLED}"
          xcode_26_0_enabled="${MACOS_XCODE_26_0_ENABLED}"
          xcode_26_1_enabled="${MACOS_XCODE_26_1_ENABLED}"
          xcode_26_2_enabled="${MACOS_XCODE_26_2_ENABLED}"
          xcode_latest_beta_enabled="${MACOS_XCODE_LATEST_BETA_ENABLED}"
          swift_6_1_enabled="${MACOS_SWIFT_6_1_ENABLED}"
          swift_6_2_enabled="${MACOS_SWIFT_6_2_ENABLED}"
          swift_6_3_enabled="${MACOS_SWIFT_6_3_ENABLED}"

          # Create matrix from inputs
          matrix='{"config": []}'

          if [[ "$xcode_16_3_enabled" == "true" ]]; then
              matrix=$(echo "$matrix" | jq -c \
              --arg runner_pool "$runner_pool" \
              --argjson env_vars "$macos_env_vars_json" \
              '.config[.config| length] |= . + { "name": "macOS (Xcode 16.3)", "xcode_version": "16.3", "xcode_app": "Xcode_16.3.app", "os": "tahoe", "arch": "ARM64", "pool": $runner_pool, "env": $env_vars }')
          fi

          if [[ "$xcode_16_4_enabled" == "true" ]]; then
              matrix=$(echo "$matrix" | jq -c \
              --arg runner_pool "$runner_pool" \
              --argjson env_vars "$macos_env_vars_json" \
              '.config[.config| length] |= . + { "name": "macOS (Xcode 16.4)", "xcode_version": "16.4", "xcode_app": "Xcode_16.4.app", "os": "tahoe", "arch": "ARM64", "pool": $runner_pool, "env": $env_vars }')
          fi

          if [[ "$xcode_26_0_enabled" == "true" ]]; then
              matrix=$(echo "$matrix" | jq -c \
              --arg runner_pool "$runner_pool" \
              --argjson env_vars "$macos_env_vars_json" \
              '.config[.config| length] |= . + { "name": "macOS (Xcode 26.0)", "xcode_version": "26.0", "xcode_app": "Xcode_26.0.app", "os": "tahoe", "arch": "ARM64", "pool": $runner_pool, "env": $env_vars }')
          fi

          if [[ "$xcode_26_1_enabled" == "true" ]]; then
              matrix=$(echo "$matrix" | jq -c \
              --arg runner_pool "$runner_pool" \
              --argjson env_vars "$macos_env_vars_json" \
              '.config[.config| length] |= . + { "name": "macOS (Xcode 26.1)", "xcode_version": "26.1", "xcode_app": "Xcode_26.1.app", "os": "tahoe", "arch": "ARM64", "pool": $runner_pool, "env": $env_vars }')
          fi

          if [[ "$xcode_26_2_enabled" == "true" ]]; then
              matrix=$(echo "$matrix" | jq -c \
              --arg runner_pool "$runner_pool" \
              --argjson env_vars "$macos_env_vars_json" \
              '.config[.config| length] |= . + { "name": "macOS (Xcode 26.2)", "xcode_version": "26.2", "xcode_app": "Xcode_26.2.app", "os": "tahoe", "arch": "ARM64", "pool": $runner_pool, "env": $env_vars }')
          fi

          if [[ "$xcode_latest_beta_enabled" == "true" ]]; then
              matrix=$(echo "$matrix" | jq -c \
              --arg runner_pool "$runner_pool" \
              --argjson env_vars "$macos_env_vars_json" \
              '.config[.config| length] |= . + { "name": "macOS (Xcode latest beta)", "xcode_version": "latest beta", "xcode_app": "Xcode-latest.app", "os": "tahoe", "arch": "ARM64", "pool": $runner_pool, "env": $env_vars }')
          fi

          if [[ "$swift_6_1_enabled" == "true" ]]; then
              matrix=$(echo "$matrix" | jq -c \
              --arg runner_pool "$runner_pool" \
              --argjson env_vars "$macos_env_vars_json" \
              '.config[.config| length] |= . + { "name": "macOS (Swift 6.1)", "xcode_version": "swift 6.1", "xcode_app": "Xcode_swift_6.1.app", "os": "tahoe", "arch": "ARM64", "pool": $runner_pool, "env": $env_vars }')
          fi

          if [[ "$swift_6_2_enabled" == "true" ]]; then
              matrix=$(echo "$matrix" | jq -c \
              --arg runner_pool "$runner_pool" \
              --argjson env_vars "$macos_env_vars_json" \
              '.config[.config| length] |= . + { "name": "macOS (Swift 6.2)", "xcode_version": "swift 6.2", "xcode_app": "Xcode_swift_6.2.app", "os": "tahoe", "arch": "ARM64", "pool": $runner_pool, "env": $env_vars }')
          fi

          if [[ "$swift_6_3_enabled" == "true" ]]; then
              matrix=$(echo "$matrix" | jq -c \
              --arg runner_pool "$runner_pool" \
              --argjson env_vars "$macos_env_vars_json" \
              '.config[.config| length] |= . + { "name": "macOS (Swift 6.3)", "xcode_version": "swift 6.3", "xcode_app": "Xcode_swift_6.3.app", "os": "tahoe", "arch": "ARM64", "pool": $runner_pool, "env": $env_vars }')
          fi

          echo "macos-matrix=$matrix" >> "$GITHUB_OUTPUT"
        env:
          INPUT_MACOS_ENV_VARS: ${{ inputs.macos_env_vars }}
          MACOS_RUNNER_POOL: ${{ inputs.macos_runner_pool }}
          MACOS_XCODE_16_3_ENABLED: ${{ inputs.macos_xcode_16_3_enabled }}
          MACOS_XCODE_16_4_ENABLED: ${{ inputs.macos_xcode_16_4_enabled }}
          MACOS_XCODE_26_0_ENABLED: ${{ inputs.macos_xcode_26_0_enabled }}
          MACOS_XCODE_26_1_ENABLED: ${{ inputs.macos_xcode_26_1_enabled }}
          MACOS_XCODE_26_2_ENABLED: ${{ inputs.macos_xcode_26_2_enabled }}
          MACOS_XCODE_LATEST_BETA_ENABLED: ${{ inputs.macos_xcode_latest_beta_enabled }}
          MACOS_SWIFT_6_1_ENABLED: ${{ inputs.macos_swift_6_1_enabled }}
          MACOS_SWIFT_6_2_ENABLED: ${{ inputs.macos_swift_6_2_enabled }}
          MACOS_SWIFT_6_3_ENABLED: ${{ inputs.macos_swift_6_3_enabled }}

  benchmarks-macos:
    name: ${{ matrix.config.name }}
    needs: construct-matrix-macos
    runs-on: [self-hosted, macos, "${{ matrix.config.os }}", "${{ matrix.config.arch }}", "${{ matrix.config.pool }}"]
    timeout-minutes: 30
    strategy:
      fail-fast: false
      matrix: ${{ fromJson(needs.construct-matrix-macos.outputs.macos-matrix) }}
    steps:
      - name: Checkout repository
        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6.0.2
        with:
          persist-credentials: false
          submodules: true
      - name: Install jemalloc
        run: |
          brew install jemalloc
      - name: Export environment variables
        if: ${{ matrix.config.env != '' && matrix.config.env != '{}'}}
        run: |
          echo "Exporting environment variables from matrix configuration..."
          echo "$MATRIX_ENV_JSON" | jq -r 'to_entries[] | "\(.key)=\(.value)"' >> $GITHUB_ENV
          echo "$MATRIX_ENV_JSON" | jq -r 'to_entries[] | "exporting \(.key)=\(.value)"'
        env:
          MATRIX_ENV_JSON: ${{ toJSON(matrix.config.env) }}
      - name: Run benchmarks script
        run: |
          curl -s https://raw.githubusercontent.com/apple/swift-nio/main/scripts/check_benchmark_thresholds.sh | BENCHMARK_PACKAGE_PATH="$INPUT_BENCHMARK_PACKAGE_PATH" bash -s -- --disable-sandbox --allow-writing-to-package-directory
        env:
          INPUT_BENCHMARK_PACKAGE_PATH: ${{ inputs.benchmark_package_path }}
          DEVELOPER_DIR: "/Applications/${{ matrix.config.xcode_app }}"
          SWIFT_VERSION: "Xcode ${{ matrix.config.xcode_version }}"