mac.yml

name: macOS (14, Python 3.11)
on:
  workflow_dispatch:
  pull_request:
  merge_group:
  push:
    branches:
      - master
      - 'releases/**'

permissions: read-all # Required by https://github.com/ossf/scorecard/blob/e23b8ad91fd6a64a0a971ca4fc0a4d1650725615/docs/checks.md#token-permissions

concurrency:
  # github.ref is not unique in post-commit
  group: ${{ github.event_name == 'push' && github.run_id || github.ref }}-mac
  cancel-in-progress: true

env:
  MACOSX_DEPLOYMENT_TARGET: '11.0'
  PYTHON_VERSION: '3.11'
  TARGET_BRANCH: ${{ github.base_ref || github.event.merge_group.base_ref || github.ref }}
  BASE_PRODUCT_TYPE: public_macos_arm64
  CCACHE_MAXSIZE: 500Mi
  HF_HOME: ~/.cache/hf
  OV_CACHE: ~/.cache/ov_cache/
  CLEANUP_CACHE: 1
  OPENVINO_LOG_LEVEL: 4

jobs:
  smart_ci:
    name: Smart CI
    runs-on: ubuntu-latest
    outputs:
      affected_components: "${{ steps.smart_ci.outputs.affected_components }}"
      changed_components: "${{ steps.smart_ci.outputs.changed_components }}"
      skip_workflow: "${{ steps.smart_ci.outputs.skip_workflow }}"
    steps:
      - name: checkout action
        uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5.0.1
        timeout-minutes: 15
        with:
          sparse-checkout: .github

      - name: Get affected components
        id: smart_ci
        uses: openvinotoolkit/openvino/.github/actions/smart-ci@36a8f092d3250e7a2a365f0445e61297d91c358e
        with:
          repository: ${{ github.repository }}
          pr: ${{ github.event.number }}
          commit_sha: ${{ github.sha }}
          ref_name: ${{ github.ref_name }}
          component_pattern: "category: ((?!Python API|CPP API).*)"
          repo_token: ${{ secrets.GITHUB_TOKEN }}
          skip_when_only_listed_labels_set: 'GH Pages Docs'
          skip_when_only_listed_files_changed: '*.md,*.rst,*.png,*.jpg,*.svg,*.gif'

      - name: Show affected components
        run: echo "${{ toJSON(steps.smart_ci.outputs.affected_components) }}"
        shell: bash

  openvino_download:
    needs: smart_ci
    if: ${{ github.event_name != 'merge_group' && needs.smart_ci.outputs.skip_workflow != 'True' }}
    name: Download OpenVINO
    outputs:
      status: ${{ steps.openvino_download.outcome }}
      ov_artifact_name: ${{ steps.openvino_download.outputs.ov_artifact_name }}
      ov_wheel_source: ${{ steps.openvino_download.outputs.ov_wheel_source }}
      ov_version: ${{ steps.openvino_download.outputs.ov_version }}
    timeout-minutes: 10
    defaults:
      run:
        shell: bash
    runs-on: aks-linux-medium
    container:
      image: 'openvinogithubactions.azurecr.io/openvino_provider:0.1.0'
      volumes:
        - /mount:/mount
        - ${{ github.workspace }}:${{ github.workspace }}

    steps:
    - uses: openvinotoolkit/openvino/.github/actions/openvino_provider@master
      id: openvino_download
      with:
        platform: macos_14_7
        arch: 'arm64'
        commit_packages_to_provide: wheels,developer_package.tar.gz,openvino_node_npm_package.tar.gz
        revision: latest_available_commit
        # Set specific revision and uncomment to use OV from its PR build:
        # branch_name: master
        # event_name: pull_request

  genai_build_cmake:
    name: Build cpack - ${{ matrix.build-type }}
    strategy:
      fail-fast: false
      matrix:
        build-type: [Release]
    needs: [ openvino_download ]
    timeout-minutes: 45
    defaults:
      run:
        shell: bash
    runs-on: macos-14
    env:
      CMAKE_GENERATOR: Ninja
      CMAKE_CXX_COMPILER_LAUNCHER: ccache
      CMAKE_C_COMPILER_LAUNCHER: ccache
      CCACHE_DIR: ${{ github.workspace }}/ccache
      OV_INSTALL_DIR: ${{ github.workspace }}/ov
      INSTALL_DIR: ${{ github.workspace }}/install
      INSTALL_TOOLS_DIR: ${{ github.workspace }}/tools
      INSTALL_TESTS_DIR: ${{ github.workspace }}/tests
      BUILD_DIR: ${{ github.workspace }}/build
      SRC_DIR: ${{ github.workspace }}/src
      MANIFEST_PATH: ${{ github.workspace }}/manifest.yml

    steps:
      - name: Clone openvino.genai
        uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5.0.1
        with:
          path: ${{ env.SRC_DIR }}
          submodules: recursive

      - name: Download OpenVINO package
        uses: akashchi/download-artifact@d59a9c15fec3fdb7c9adf09464124d00f9c11415
        with:
          name: ${{ needs.openvino_download.outputs.ov_artifact_name }}
          path: ${{ env.OV_INSTALL_DIR }}
          merge-multiple: true

      - name: Setup Python ${{ env.PYTHON_VERSION }}
        uses: actions/setup-python@83679a892e2d95755f2dac6acb0bfd1e9ac5d548 # v6.1.0
        with:
          python-version: ${{ env.PYTHON_VERSION }}
          cache: 'pip'

      - name: Install build dependencies
        run: brew install coreutils ccache

      - name: Setup ccache
        id: ccache-restore
        uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0
        with:
          key: ${{ runner.os }}-${{ runner.arch }}-ccache-${{ env.TARGET_BRANCH }}-${{ matrix.build-type }}-cpack-${{ github.sha }}
          restore-keys: |
            ${{ runner.os }}-${{ runner.arch }}-ccache-${{ env.TARGET_BRANCH }}-${{ matrix.build-type }}-cpack
          path: ${{ env.CCACHE_DIR }}

      - name: Generate product manifest
        id: create_manifest
        uses: openvinotoolkit/openvino/.github/actions/create_manifest@master
        with:
          repos: ${{ env.SRC_DIR }}
          product_type: ${{ env.BASE_PRODUCT_TYPE }}_${{ matrix.build-type }}
          target_arch: 'arm64'
          build_type: ${{ matrix.build-type }}
          save_to: ${{ env.MANIFEST_PATH }}

      - name: Clean ccache stats
        run: ccache --zero-stats --show-config

      - name: CMake Build
        run: |
          source ${{ env.OV_INSTALL_DIR }}/setupvars.sh
          cmake -DOpenVINODeveloperPackage_DIR=${{ env.OV_INSTALL_DIR }}/developer_package/cmake \
                -DENABLE_PYTHON=ON \
                -DCMAKE_BUILD_TYPE=${{ matrix.build-type }} \
                -S ${{ env.SRC_DIR }} \
                -B ${{ env.BUILD_DIR }}
          cmake --build ${{ env.BUILD_DIR}} --config ${{ matrix.build-type }} --parallel $(nproc) --verbose
          cmake --install ${{ env.BUILD_DIR }} --config ${{ matrix.build-type }} --prefix ${{ env.INSTALL_DIR }}
          cmake --install ${{ env.BUILD_DIR }} --config ${{ matrix.build-type }} --prefix ${{ env.INSTALL_TOOLS_DIR }} --component tools_bin
          cmake --install ${{ env.BUILD_DIR }} --config ${{ matrix.build-type }} --prefix ${{ env.INSTALL_TESTS_DIR }} --component tests

      - name: Show ccache stats
        run: ccache --show-stats

      - name: Save ccache
        if: always() && steps.ccache-restore.outputs.cache-hit != 'true' && github.event_name == 'push'
        uses: actions/cache/save@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0
        with:
          key: ${{ steps.ccache-restore.outputs.cache-primary-key }}
          path: ${{ env.CCACHE_DIR }}
      - name: Upload Archive Distribution Package
        if: always()
        uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v5.0.0
        with:
          name: genai_cpack_${{ matrix.build-type }}
          path: ${{ env.INSTALL_DIR }}
          if-no-files-found: 'error'

      - name: Upload Tools
        if: always()
        uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v5.0.0
        with:
          name: genai_tools_${{ matrix.build-type }}
          path: ${{ env.INSTALL_TOOLS_DIR }}
          if-no-files-found: 'error'

      - name: Upload Tests
        if: always()
        uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v5.0.0
        with:
          name: genai_tests_${{ matrix.build-type }}
          path: ${{ env.INSTALL_TESTS_DIR }}
          if-no-files-found: 'error'

      - name: Upload manifest
        if: always()
        uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v5.0.0
        with:
          name: manifest_${{ matrix.build-type }}
          path: ${{ env.MANIFEST_PATH }}
          if-no-files-found: 'error'

  genai_build_wheel:
    name: Build Wheel
    needs: [ openvino_download ]
    timeout-minutes: 90
    defaults:
      run:
        shell: bash
    runs-on: macos-14-xlarge
    env:
      CCACHE_DIR: ${{ github.workspace }}/ccache
      CMAKE_CXX_COMPILER_LAUNCHER: ccache
      CMAKE_C_COMPILER_LAUNCHER: ccache
      OV_INSTALL_DIR: ${{ github.workspace }}/ov
      INSTALL_DIR: ${{ github.workspace }}/install
      WHEELS_DIR: ${{ github.workspace }}/install/wheels
      SRC_DIR: ${{ github.workspace }}/src
      OpenVINODeveloperPackage_DIR: ${{ github.workspace }}/ov/developer_package/cmake

    steps:
      - name: Clone openvino.genai
        uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5.0.1
        with:
          path: ${{ env.SRC_DIR }}
          submodules: recursive

      - name: Download OpenVINO package
        uses: akashchi/download-artifact@d59a9c15fec3fdb7c9adf09464124d00f9c11415
        with:
          name: ${{ needs.openvino_download.outputs.ov_artifact_name }}
          path: ${{ env.OV_INSTALL_DIR }}
          merge-multiple: true

      - name: Setup Python ${{ env.PYTHON_VERSION }}
        uses: actions/setup-python@83679a892e2d95755f2dac6acb0bfd1e9ac5d548 # v6.1.0
        with:
          python-version: ${{ env.PYTHON_VERSION }}
          cache: 'pip'

      - name: Install build dependencies
        run: brew install coreutils ccache

      - name: Setup ccache
        id: ccache-restore
        uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0
        with:
          key: ${{ runner.os }}-${{ runner.arch }}-ccache-${{ env.TARGET_BRANCH }}-wheel-${{ github.sha }}
          restore-keys: |
            ${{ runner.os }}-${{ runner.arch }}-ccache-${{ env.TARGET_BRANCH }}-wheel
          path: ${{ env.CCACHE_DIR }}\

      - name: Set CI environment
        id: create_manifest
        uses: openvinotoolkit/openvino/.github/actions/create_manifest@master
        with:
          repos: ${{ env.SRC_DIR }}
          product_type: ${{ env.BASE_PRODUCT_TYPE }}_Release
          target_arch: 'arm64'
          build_type: Release
          save_to: ${{ github.workspace }}

      - name: Clean ccache stats
        run: ccache --zero-stats --show-config

      - name: Build Tokenizers Wheel
        run: |
          python -m pip wheel -v --no-deps --wheel-dir ${{ env.WHEELS_DIR }} \
              --config-settings='override=wheel.build_tag="${{ github.run_number }}"' \
              ${{ needs.openvino_download.outputs.ov_wheel_source }} \
              ${{ env.SRC_DIR }}/thirdparty/openvino_tokenizers
        working-directory: ${{ env.OV_INSTALL_DIR }}

      - name: Build GenAI Wheel
        run: |
          python -m pip wheel -v --no-deps --wheel-dir ${{ env.WHEELS_DIR }} \
              ${{ needs.openvino_download.outputs.ov_wheel_source }} \
              ${{ env.SRC_DIR }}
        working-directory: ${{ env.OV_INSTALL_DIR }}

      - name: Build WWB Wheel
        run: python -m pip wheel -v --no-deps --wheel-dir ${{ env.WHEELS_DIR }} ${{ env.SRC_DIR }}/tools/who_what_benchmark
        working-directory: ${{ env.OV_INSTALL_DIR }}

      - name: Save ccache
        if: always() && steps.ccache-restore.outputs.cache-hit != 'true' && github.event_name == 'push'
        uses: actions/cache/save@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0
        with:
          key: ${{ steps.ccache-restore.outputs.cache-primary-key }}
          path: ${{ env.CCACHE_DIR }}

      - name: Upload Wheels
        if: always()
        uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v5.0.0
        with:
          name: genai_wheels
          path: ${{ env.INSTALL_DIR }}
          if-no-files-found: 'error'

  genai_build_samples:
    name: Build Samples - ${{ matrix.build-type }}
    strategy:
      fail-fast: false
      matrix:
        build-type: [Release]
    needs: [ openvino_download, genai_build_cmake ]
    timeout-minutes: 30
    defaults:
      run:
        shell: bash
    runs-on: macos-14
    env:
      OV_INSTALL_DIR: ${{ github.workspace }}/ov
      INSTALL_DIR: ${{ github.workspace }}/install
      BUILD_DIR: ${{ github.workspace }}/build
      SRC_DIR: ${{ github.workspace }}/src
      CMAKE_POLICY_VERSION_MINIMUM: 3.5

    steps:
      - name: Clone openvino.genai
        uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5.0.1
        with:
          path: ${{ env.SRC_DIR }}
          submodules: recursive

      - name: Download Build Artifacts
        uses: akashchi/download-artifact@d59a9c15fec3fdb7c9adf09464124d00f9c11415
        with:
          pattern: "{${{ needs.openvino_download.outputs.ov_artifact_name }},genai_cpack_${{ matrix.build-type }}}"
          path: ${{ env.OV_INSTALL_DIR }}
          merge-multiple: true

      - name: Build Samples (Release)
        if: ${{ 'Release' == matrix.build-type }}
        run: |
          chmod +x ${{ env.OV_INSTALL_DIR }}/samples/cpp/build_samples.sh
          ${{ env.OV_INSTALL_DIR }}/samples/cpp/build_samples.sh -i ${{ env.INSTALL_DIR }}
          chmod +x ${{ env.OV_INSTALL_DIR }}/samples/c/build_samples.sh
          ${{ env.OV_INSTALL_DIR }}/samples/c/build_samples.sh -i ${{ env.INSTALL_DIR }}

      - name: Build Samples (${{ matrix.build-type }})
        if: ${{ 'Release' != matrix.build-type }}
        run: |
          source ${{ env.OV_INSTALL_DIR }}/setupvars.sh
          cmake -DCMAKE_BUILD_TYPE=${{ matrix.build-type }} -S ${{ env.OV_INSTALL_DIR }}/samples/cpp/ -B ${{ env.BUILD_DIR }}
          cmake --build ${{ env.BUILD_DIR }} --config ${{ matrix.build-type }} --parallel $(nproc)
          cmake --install ${{ env.BUILD_DIR }} --config ${{ matrix.build-type }} --component samples_bin --prefix ${{ env.INSTALL_DIR }}

      - name: Upload Samples Build Package
        if: ${{ always() }}
        uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v5.0.0
        with:
          name: genai_samples_${{ matrix.build-type }}
          path: ${{ env.INSTALL_DIR }}
          if-no-files-found: 'error'

  genai_build_nodejs:
    name: Build Node.js bindings
    strategy:
      fail-fast: false
      matrix:
        build-type: [Release]
    needs: [ openvino_download ]
    timeout-minutes: 50
    defaults:
      run:
        shell: bash
    runs-on: macos-14

    env:
      SRC_DIR: ${{ github.workspace }}/openvino.genai
      BUILD_DIR: ${{ github.workspace }}/build
      INSTALL_DIR: ${{ github.workspace }}/openvino.genai/src/js/bin
      OV_INSTALL_DIR: ${{ github.workspace }}/ov

    steps:
      - name: Clone openvino.genai
        uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5.0.1
        with:
          submodules: recursive
          path: ${{ env.SRC_DIR }}

      - name: Download OpenVINO package
        uses: akashchi/download-artifact@d59a9c15fec3fdb7c9adf09464124d00f9c11415
        with:
          name: ${{ needs.openvino_download.outputs.ov_artifact_name }}
          path: ${{ env.OV_INSTALL_DIR }}
          merge-multiple: true

      - name: Install build dependencies
        run: brew install coreutils

      - name: Build GenAI Node.js bindings
        run: |
          source ${{ env.OV_INSTALL_DIR }}/setupvars.sh
          cmake -DCMAKE_BUILD_TYPE=${{ matrix.build-type }} \
            -DENABLE_JS=ON -DCPACK_GENERATOR=NPM \
            -DENABLE_PYTHON=OFF -DENABLE_WHEEL=OFF \
            -S ${{ env.SRC_DIR }} -B ${{ env.BUILD_DIR }}
          cmake --build ${{ env.BUILD_DIR }} --config ${{ matrix.build-type }} --parallel $(nproc) --verbose
          cmake --install ${{ env.BUILD_DIR }} --config ${{ matrix.build-type }} --prefix ${{ env.INSTALL_DIR }}

      - name: Upload Node.js bindings Build Package
        if: always()
        uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v5.0.0
        with:
          name: genai_nodejs_bindings
          path: ${{ env.INSTALL_DIR }}
          if-no-files-found: 'error'

  genai_tests_wheel:
    name: Python (${{ matrix.test.name}}) Tests (wheel)
    needs: [ smart_ci, openvino_download, genai_build_wheel ]
    timeout-minutes: ${{ matrix.test.timeout }}
    strategy:
      fail-fast: false
      matrix:
        test:
          - name: 'Whisper'
            # TODO: skip some tests temporary until https://github.com/huggingface/datasets/issues/7647 dataset is fixed
            cmd: 'python -m pytest -v ./tests/python_tests/test_whisper_pipeline.py -k "not test_smoke[sample_from_dataset0 and not test_whisper_constructors[sample_from_dataset0 and not test_max_new_tokens[sample_from_dataset0 and not test_language_mode[language and not test_task_mode[sample_from_dataset0 and not test_language_autodetect[sample_from_dataset0 and not test_whisper_config_constructor and not test_language_autodetect[sample_from_dataset1 and not test_language_autodetect[sample_from_dataset2 and not test_initial_prompt_hotwords[sample_from_dataset0 and not test_random_sampling[sample_from_dataset0"'
            run_condition: ${{ fromJSON(needs.smart_ci.outputs.affected_components).whisper.test }}
            timeout: 120
          # Only supported on X64 or ARM with SVE support
          # - name: 'Cacheopt E2E (Part 1)'
          #   cmd: 'python -m pytest -v tests/python_tests/test_kv_cache_eviction/test_kv_cache_eviction_1.py'
          #   run_condition: ${{ fromJSON(needs.smart_ci.outputs.affected_components).continuous_batching.test }}
          #   timeout: 180
          # Only supported on X64 or ARM with SVE support
          # - name: 'Cacheopt E2E (Part 2)'
          #   cmd: 'python -m pytest -v tests/python_tests/test_kv_cache_eviction/test_kv_cache_eviction_2.py'
          #   run_condition: ${{ fromJSON(needs.smart_ci.outputs.affected_components).continuous_batching.test }}
          #   timeout: 240
          # Only supported on X64 or ARM with SVE support
          # - name: 'LLM & VLM'
          #   cmd: 'tests/python_tests/test_llm_pipeline.py tests/python_tests/test_llm_pipeline_static.py tests/python_tests/test_vlm_pipeline.py tests/python_tests/test_structured_output.py tests/python_tests/test_image_generation.py'
          #   run_condition: ${{ fromJSON(needs.smart_ci.outputs.affected_components).visual_language.test || fromJSON(needs.smart_ci.outputs.affected_components).LLM.test || fromJSON(needs.smart_ci.outputs.affected_components).Image_generation.test }}
          #   timeout: 180
          - name: 'GGUF Reader tests'
            cmd: 'python -m pytest -v ./tests/python_tests/test_gguf_reader.py'
            run_condition: ${{ fromJSON(needs.smart_ci.outputs.affected_components).GGUF.test || fromJSON(needs.smart_ci.outputs.affected_components).LLM.test }}
            timeout: 360
          - name: 'Tokenizer tests'
            cmd: 'python -m pytest -v ./tests/python_tests/test_tokenizer.py'
            run_condition: ${{ fromJSON(needs.smart_ci.outputs.affected_components).tokenizers.test }}
            timeout: 60
          # Only supported on X64 or ARM with SVE support
          # - name: 'API tests'
          #   cmd: 'tests/python_tests/test_continuous_batching.py tests/python_tests/test_generation_config.py tests/python_tests/test_sampling.py tests/python_tests/test_text_streamer.py'
          #   run_condition: ${{ fromJSON(needs.smart_ci.outputs.affected_components).continuous_batching.test || fromJSON(needs.smart_ci.outputs.affected_components).sampling.test || fromJSON(needs.smart_ci.outputs.affected_components).text_streamer.test }}
          #   timeout: 60
          - name: 'Rag tests'
            cmd: 'python -m pytest -v ./tests/python_tests/test_rag.py'
            run_condition: ${{ fromJSON(needs.smart_ci.outputs.affected_components).RAG.test }}
            timeout: 30
          - name: 'WWB tests'
            cmd: 'python -m pytest -v ./tools/who_what_benchmark/tests -m "not nanollava"'
            run_condition: ${{ fromJSON(needs.smart_ci.outputs.affected_components).WWB.test }}
            timeout: 120
          - name: 'WWB tests (nanollava)'
            cmd: |
              python -m pip install transformers==4.48.0
              python -m pytest -v ./tools/who_what_benchmark/tests -m nanollava
            run_condition: ${{ fromJSON(needs.smart_ci.outputs.affected_components).WWB.test }}
            timeout: 90
    defaults:
      run:
        shell: bash
    runs-on: macos-14
    env:
      INSTALL_DIR: ${{ github.workspace }}/install
      SRC_DIR: ${{ github.workspace }}/src
      BUILD_DIR: ${{ github.workspace }}/build
      HF_HOME: ${{ github.workspace }}/hf_cache

    steps:
      - name: Clone openvino.genai
        if: ${{ matrix.test.run_condition }}
        uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5.0.1
        with:
          path: ${{ env.SRC_DIR }}
          submodules: recursive

      - name: Download Build Artifacts
        if: ${{ matrix.test.run_condition }}
        uses: akashchi/download-artifact@d59a9c15fec3fdb7c9adf09464124d00f9c11415
        with:
          pattern: "{${{ needs.openvino_download.outputs.ov_artifact_name }},genai_wheels}"
          path: ${{ env.INSTALL_DIR }}
          merge-multiple: true

      - name: Setup Python ${{ env.PYTHON_VERSION }}
        if: ${{ matrix.test.run_condition }}
        uses: actions/setup-python@83679a892e2d95755f2dac6acb0bfd1e9ac5d548 # v6.1.0
        with:
          python-version: ${{ env.PYTHON_VERSION }}
          cache: 'pip'

      - name: Install GenAI Wheels
        if: ${{ matrix.test.run_condition }}
        uses: ./src/.github/actions/install_wheel
        with:
          packages: "openvino;openvino_tokenizers[transformers];openvino_genai;whowhatbench"
          requirements_files: "${{ env.SRC_DIR }}/tests/python_tests/requirements.txt"
          local_wheel_dir: ${{ env.INSTALL_DIR }}/wheels

      - name: Tests
        if: ${{ matrix.test.run_condition }}
        run: ${{ matrix.test.cmd }}
        working-directory: ${{ env.SRC_DIR }}

  genai_samples_tests:
    name: Samples ${{ matrix.test.name }} (${{ matrix.build-type }})
    strategy:
      fail-fast: false
      matrix:
        build-type: [Release]
        test:
          - name: 'LLM'
            marker: 'llm'
            cmd: 'tests/python_tests/samples'
            run_condition: ${{ fromJSON(needs.smart_ci.outputs.affected_components).LLM_samples.test }}
          - name: 'Whisper'
            marker: 'whisper'
            cmd: 'tests/python_tests/samples'
            run_condition: ${{ fromJSON(needs.smart_ci.outputs.affected_components).Whisper_samples.test }}
          - name: 'Rag'
            marker: 'rag'
            cmd: 'tests/python_tests/samples'
            run_condition: ${{ fromJSON(needs.smart_ci.outputs.affected_components).RAG_samples.test }}
          - name: 'Speech generation'
            marker: 'speech_generation'
            cmd: 'tests/python_tests/samples'
            run_condition: ${{ fromJSON(needs.smart_ci.outputs.affected_components).Speech_generation_samples.test }}

    needs: [ smart_ci, openvino_download, genai_build_cmake, genai_build_wheel, genai_build_samples, genai_build_nodejs ]
    timeout-minutes: 120
    defaults:
      run:
        shell: bash
    runs-on: macos-14
    env:
      INSTALL_DIR: ${{ github.workspace }}/ov
      SRC_DIR: ${{ github.workspace }}/src
      BUILD_DIR: ${{ github.workspace }}/build
      # The debug logging includes messages about the time it takes to read the GGUF model.
      # These messages differ from run to run, so we cannot compare the results of the CPP, Python, and JavaScript parts.
      OPENVINO_LOG_LEVEL: 1

    steps:
      - name: Clone openvino.genai
        if: ${{ matrix.test.run_condition }}
        uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5.0.1
        with:
          path: ${{ env.SRC_DIR }}
          submodules: recursive

      - name: Download Build Artifacts
        if: ${{ matrix.test.run_condition }}
        uses: akashchi/download-artifact@d59a9c15fec3fdb7c9adf09464124d00f9c11415
        with:
          pattern: "{${{ needs.openvino_download.outputs.ov_artifact_name }},genai_cpack_${{ matrix.build-type }},genai_samples_${{ matrix.build-type }},genai_wheels}"
          path: ${{ env.INSTALL_DIR }}
          merge-multiple: true

      - name: Download GenAI JS Bildings Artifacts
        if: ${{ matrix.test.run_condition }}
        uses: akashchi/download-artifact@d59a9c15fec3fdb7c9adf09464124d00f9c11415
        with:
          name: genai_nodejs_bindings
          path: ${{ env.SRC_DIR }}/src/js/bin
          merge-multiple: true

      - name: Setup Python ${{ env.PYTHON_VERSION }}
        if: ${{ matrix.test.run_condition }}
        uses: actions/setup-python@83679a892e2d95755f2dac6acb0bfd1e9ac5d548 # v6.1.0
        with:
          python-version: ${{ env.PYTHON_VERSION }}
          cache: 'pip'

      - name: Install GenAI wheels
        if: ${{ matrix.test.run_condition }}
        uses: ./src/.github/actions/install_wheel
        with:
          packages: "openvino;openvino_tokenizers[transformers];openvino_genai[testing]"
          requirements_files: "${{ env.SRC_DIR }}/samples/requirements.txt"
          local_wheel_dir: ${{ env.INSTALL_DIR }}/wheels

      # transformers >= 4.52 require torch >= 2.6 and raise an error otherwise:
      # ValueError: Due to a serious vulnerability issue in `torch.load`, even with `weights_only=True`, we now require users to upgrade torch to at least v2.6 in order to use the function. This version restriction does not apply when loading files with safetensors.
      # See the vulnerability report here https://nvd.nist.gov/vuln/detail/CVE-2025-32434

      # x86_64 macOS does not (and will not) support newer versions of torch > 2.2 which are used in the newer transformers versions. It's not possible to lower transformer version in requirements.txt because that triggers vulnerability alert: https://github.com/openvinotoolkit/openvino_tokenizers/security/dependabot/11
      - run: python -m pip install "transformers<4.52"

      - name: Fix C++ samples permissions
        if: ${{ matrix.test.run_condition }}
        run: chmod +x ${{ env.INSTALL_DIR }}/samples_bin/*

      - name: Setup NodeJS
        if: ${{ matrix.test.run_condition }}
        uses: actions/setup-node@2028fbc5c25fe9cf00d9f06a71cc4710d4507903 # v6.0.0
        with:
          node-version: 21

      - name: Install GenAI NPM package
        if: ${{ matrix.test.run_condition }}
        working-directory: ${{ env.SRC_DIR }}/src/js
        run: |
          npm install ${{ env.INSTALL_DIR }}/openvino_node_npm_package/openvino-node-* --ignore-scripts
          cp -R ${{ env.INSTALL_DIR }}/openvino_node_npm_package/bin node_modules/openvino-node/bin
          npm install --verbose

      - name: Install NPM dependencies for samples
        if: ${{ matrix.test.run_condition }}
        working-directory: ${{ env.SRC_DIR }}/samples/js/text_generation
        run: |
          npm install ${{ env.SRC_DIR }}/src/js
          npm install --verbose

      - name: Test Samples (Python and C++)
        if: ${{ matrix.test.run_condition }}
        run: |
          source ${{ env.INSTALL_DIR }}/setupvars.sh
          python -m pytest -vs ${{ env.SRC_DIR }}/${{ matrix.test.cmd }} -m "${{ matrix.test.marker }}"
        env:
          SAMPLES_PY_DIR: "${{ env.INSTALL_DIR }}/samples/python"
          SAMPLES_JS_DIR: "${{ env.SRC_DIR }}/samples/js"
          SAMPLES_CPP_DIR: "${{ env.INSTALL_DIR }}/samples_bin"
          SAMPLES_C_DIR: "${{ env.INSTALL_DIR }}/samples_bin"

  genai_tools_tests:
    name: Tools ${{ matrix.build-type }}
    if: ${{ 'false' }} # Only supported on X64 or ARM with SVE support
    strategy:
      fail-fast: false
      matrix:
        build-type: [Release]
    needs: [ smart_ci, openvino_download, genai_build_cmake, genai_build_wheel ]
    timeout-minutes: 90
    defaults:
      run:
        shell: bash
    runs-on: macos-14
    env:
      INSTALL_DIR: ${{ github.workspace }}/ov
      SRC_DIR: ${{ github.workspace }}/src
      BUILD_DIR: ${{ github.workspace }}/build

    steps:
      - name: Clone openvino.genai
        uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5.0.1
        with:
          path: ${{ env.SRC_DIR }}
          submodules: recursive

      - name: Download Build Artifacts
        uses: akashchi/download-artifact@d59a9c15fec3fdb7c9adf09464124d00f9c11415
        with:
          pattern: "{${{ needs.openvino_download.outputs.ov_artifact_name }},genai_cpack_${{ matrix.build-type }},genai_tools_${{ matrix.build-type }},genai_tests_${{ matrix.build-type }},genai_wheels}"
          path: ${{ env.INSTALL_DIR }}
          merge-multiple: true

      - name: Fix C++ samples permissions
        run: chmod +x ${{ env.INSTALL_DIR }}/samples_bin/*

      - name: Setup Python ${{ env.PYTHON_VERSION }}
        uses: actions/setup-python@83679a892e2d95755f2dac6acb0bfd1e9ac5d548 # v6.1.0
        with:
          python-version: ${{ env.PYTHON_VERSION }}
          cache: 'pip'

      - name: Install GenAI wheels
        uses: ./src/.github/actions/install_wheel
        with:
          packages: "openvino;openvino_tokenizers[transformers];openvino_genai[testing]"
          requirements_files: "${{ env.SRC_DIR }}/samples/requirements.txt"
          local_wheel_dir: ${{ env.INSTALL_DIR }}/wheels

      - name: gtests unit tests
        run: |
          source ${{ env.INSTALL_DIR }}/setupvars.sh
          chmod +x ${{ env.INSTALL_DIR }}/tests/tests_continuous_batching
          ${{ env.INSTALL_DIR }}/tests/tests_continuous_batching --gtest_filter="-AddSecondInputTest.*"

      - name: Test C++ Tools
        run: |
          source ${{ env.INSTALL_DIR }}/setupvars.sh
          python -m pytest -vs ${{ env.SRC_DIR }}/tests/python_tests/samples/test_continuous_batching_tools.py -m "samples"
        env:
          SAMPLES_CPP_DIR: "${{ env.INSTALL_DIR }}/samples_bin"

  genai_nodejs_tests:
    name: Node.js bindings tests
    needs: [ smart_ci, openvino_download, genai_build_nodejs ]
    if: ${{ fromJSON(needs.smart_ci.outputs.affected_components).JS_API }}
    timeout-minutes: 20
    defaults:
      run:
        shell: bash
    runs-on: macos-14

    env:
      SRC_DIR: ${{ github.workspace }}/openvino.genai
      INSTALL_DIR: ${{ github.workspace }}/install
      NODE_VERSION: 21

    steps:
      - name: Clone openvino.genai
        uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5.0.1
        with:
          path: ${{ env.SRC_DIR }}
          submodules: recursive

      - name: Download OpenVINO Artifacts
        uses: akashchi/download-artifact@d59a9c15fec3fdb7c9adf09464124d00f9c11415
        with:
          name: ${{ needs.openvino_download.outputs.ov_artifact_name }}
          path: ${{ env.INSTALL_DIR }}
          merge-multiple: true

      - name: Download GenAI JS Bildings Artifacts
        uses: akashchi/download-artifact@d59a9c15fec3fdb7c9adf09464124d00f9c11415
        with:
          name: genai_nodejs_bindings
          path: ${{ env.SRC_DIR }}/src/js/bin
          merge-multiple: true

      - name: Setup Node ${{ env.NODE_VERSION }}
        uses: actions/setup-node@2028fbc5c25fe9cf00d9f06a71cc4710d4507903 # v6.0.0
        with:
          node-version: ${{ env.NODE_VERSION }}

      # JS pacakges uses the OpenVINO and OpenVINO GenAI libraries from the bin directory.
      # Here we emulate the installation of the openvino-node package from NPM. The latest
      # release of the openvino-node package is installed, and we need to update the binaries
      # in the node_modules/openvino-node/bin directory to work correctly with GenAI
      - name: Install npm package tests dependencies
        working-directory: ${{ env.SRC_DIR }}/src/js
        run: |
          npm install ${{ env.INSTALL_DIR }}/openvino_node_npm_package/openvino-node-* --ignore-scripts
          cp -R ${{ env.INSTALL_DIR }}/openvino_node_npm_package/bin node_modules/openvino-node/bin
          npm install --verbose

      - name: Run npm package tests
        working-directory: ${{ env.SRC_DIR }}/src/js
        run: npm test

  Overall_Status:
    name: ci/gha_overall_status_macos
    needs: [smart_ci, openvino_download, genai_build_cmake, genai_build_wheel, genai_build_samples, genai_tests_wheel, genai_samples_tests, genai_tools_tests, genai_build_nodejs, genai_nodejs_tests]
    if: ${{ always() }}
    runs-on: ubuntu-latest
    steps:
      - name: Check status of all jobs
        if: >-
          ${{
            contains(needs.*.result, 'failure') ||
            contains(needs.*.result, 'cancelled')
          }}
        run: exit 1