job_pytorch_models_tests.yml

name: PyTorch Models tests

on:
  workflow_call:
    inputs:
      runner:
        description: 'Machine on which the tests would run'
        type: string
        required: true
      image:
        description: 'Docker image to use for the job'
        type: string
        required: false
        default: null
      model_scope:
        description: 'Scope of models for testing.'
        type: string
        required: true

permissions: read-all

env:
  HF_HUB_CACHE_LIN: /mount/caches/huggingface
  HF_HUB_CACHE_WIN: "C:\\mount\\caches\\huggingface"

jobs:
  PyTorch_Models_Tests:
    name: PyTorch Models tests
    timeout-minutes: ${{ inputs.model_scope == 'precommit' && 90 || 400 }}
    runs-on: ${{ inputs.runner }}
    container:
      image: ${{ inputs.image }}
      volumes:
        - /mount:/mount
        - ${{ github.workspace }}:${{ github.workspace }} # Needed as ${{ github.workspace }} is not working correctly when using Docker
      options: "-e HF_TOKEN" # Needed for HuggingFace cache authentication
    defaults:
      run:
        shell: bash
    env:
      DEBIAN_FRONTEND: noninteractive # to prevent apt-get from waiting user input
      INSTALL_DIR: ${{ github.workspace }}/install
      INSTALL_TEST_DIR: ${{ github.workspace }}/install/tests
      MODEL_HUB_TESTS_INSTALL_DIR: ${{ github.workspace }}/install/tests/model_hub_tests
      USE_SYSTEM_CACHE: False # Using remote HuggingFace cache
      HF_HUB_VERBOSITY: debug
      TRANSFORMERS_VERBOSITY: debug
    steps:
      - name: Fetch setup_python and install wheels actions
        uses: ababushk/checkout@dd591a6a2ac25618db4eda86e7e0d938f88cf01b # cherry_pick_retries
        timeout-minutes: 15
        with:
          sparse-checkout: |
            .github/actions
          sparse-checkout-cone-mode: false
          submodules: 'false'

      - name: Download OpenVINO artifacts (wheels)
        uses: akashchi/download-artifact@d59a9c15fec3fdb7c9adf09464124d00f9c11415
        with:
          name: openvino_wheels
          path: ${{ env.INSTALL_DIR }}

      - name: Download OpenVINO artifacts (tests)
        uses: akashchi/download-artifact@d59a9c15fec3fdb7c9adf09464124d00f9c11415
        with:
          name: openvino_tests
          path: ${{ env.INSTALL_DIR }}

      - name: Download OpenVINO artifacts (tokenizers_wheel)
        uses: akashchi/download-artifact@d59a9c15fec3fdb7c9adf09464124d00f9c11415
        with:
          name: openvino_tokenizers_wheel
          path: ${{ env.INSTALL_DIR }}

      - name: Setup Variables
        run: |
          echo "HF_HUB_CACHE=${{ runner.os == 'Linux' && env.HF_HUB_CACHE_LIN || env.HF_HUB_CACHE_WIN }}" >> "$GITHUB_ENV"
          echo "HUGGINGFACE_HUB_CACHE=${{ runner.os == 'Linux' && env.HF_HUB_CACHE_LIN || env.HF_HUB_CACHE_WIN }}" >> "$GITHUB_ENV"

      - name: Extract OpenVINO artifacts
        run: pigz -dc openvino_tests.tar.gz | tar -xf - -v
        working-directory: ${{ env.INSTALL_DIR }}

      - name: Setup Python 3.11
        uses: ./.github/actions/setup_python
        with:
          version: '3.11'
          should-setup-pip-paths: 'false'
          self-hosted-runner: ${{ contains(inputs.runner, 'aks') }}

      - name: Install OpenVINO Python wheels
        uses: ./.github/actions/install_ov_wheels
        with:
          wheels-dir-path: ${{ env.INSTALL_DIR }}
          wheels-to-install: 'openvino openvino_tokenizers'

      - name: Install PyTorch tests requirements
        run: |
          python3 -m pip install -r ${INSTALL_TEST_DIR}/requirements_pytorch
        env:
          CPLUS_INCLUDE_PATH: ${{ env.Python_ROOT_DIR }}/include/python${{ env.PYTHON_VERSION }}

      - name: PyTorch Models Tests Timm and Torchvision
        if: ${{ inputs.model_scope == 'precommit' || inputs.model_scope == 'nightly_scope1' }}
        run: |
          export PYTHONPATH=${MODEL_HUB_TESTS_INSTALL_DIR}:$PYTHONPATH
          python3 -m pytest ${MODEL_HUB_TESTS_INSTALL_DIR}/pytorch/ -m ${TYPE} --html=${INSTALL_TEST_DIR}/TEST-torch_model_timm_tv_"${TYPE}"_tests.html --self-contained-html -v -n 4 -k "TestTimmConvertModel or TestTorchHubConvertModel or TestEdsrConvertModel"
        env:
          TYPE: ${{ inputs.model_scope == 'precommit' && 'precommit' || 'nightly' }}
          TEST_DEVICE: CPU
          OP_REPORT_FILE: ${{ env.INSTALL_TEST_DIR }}/TEST-torch_unsupported_ops.log

      - name: PyTorch Models Tests Not Timm or Torchvision
        if: ${{ inputs.model_scope == 'precommit' || inputs.model_scope == 'nightly_scope2' }}
        run: |
          export PYTHONPATH=${MODEL_HUB_TESTS_INSTALL_DIR}:$PYTHONPATH
          python3 -m pytest ${MODEL_HUB_TESTS_INSTALL_DIR}/pytorch -m ${TYPE} --html=${INSTALL_TEST_DIR}/TEST-torch_model_"${TYPE}"_tests.html --self-contained-html -v -k "not (TestTimmConvertModel or TestTorchHubConvertModel or TestEdsrConvertModel)"
        env:
          TYPE: ${{ inputs.model_scope == 'precommit' && 'precommit' || 'nightly' }}
          TEST_DEVICE: CPU
          OP_REPORT_FILE: ${{ env.INSTALL_TEST_DIR }}/TEST-torch_unsupported_ops.log

      - name: PagedAttention Test
        if: ${{ inputs.model_scope == 'precommit' }}
        run: |
          export PYTHONPATH=${MODEL_HUB_TESTS_INSTALL_DIR}:$PYTHONPATH
          python3 -m pytest ${MODEL_HUB_TESTS_INSTALL_DIR}/transformation_tests/test_pa_transformation.py -m precommit --html=${INSTALL_TEST_DIR}/TEST-torch_pagedattention_tests.html --self-contained-html -vvv -s --tb=short -n 2
        env:
          TEST_DEVICE: CPU

      - name: RoPE Test
        if: ${{ inputs.model_scope == 'precommit' }}
        run: |
          export PYTHONPATH=${MODEL_HUB_TESTS_INSTALL_DIR}:$PYTHONPATH
          python3 -m pytest ${MODEL_HUB_TESTS_INSTALL_DIR}/transformation_tests/test_transformations.py -m precommit --html=${INSTALL_TEST_DIR}/TEST-torch_rope_tests.html --self-contained-html -v --tb=short -n 2
        env:
          TEST_DEVICE: CPU

      - name: StatefulToStateless Test
        if: ${{ inputs.model_scope == 'precommit' }}
        run: |
          export PYTHONPATH=${MODEL_HUB_TESTS_INSTALL_DIR}:$PYTHONPATH
          python3 -m pytest ${MODEL_HUB_TESTS_INSTALL_DIR}/transformation_tests/test_stateful_to_stateless_transformation.py -m precommit --html=${INSTALL_TEST_DIR}/TEST-torch_stateful_to_stateless_tests.html --self-contained-html -v --tb=short
        env:
          TEST_DEVICE: CPU

      - name: TorchFX GPTQ Pattern Test
        if: ${{ inputs.model_scope == 'precommit' }}
        # install torch 2.3.1 as newer is not yet supported by openvino backend
        run: |
          export PYTHONPATH=${MODEL_HUB_TESTS_INSTALL_DIR}:$PYTHONPATH
          python3 -m pip install torch==2.3.1 torchvision==0.18.1 torchaudio==2.3.1 --upgrade --index-url https://download.pytorch.org/whl/cpu
          python3 -m pytest ${MODEL_HUB_TESTS_INSTALL_DIR}/transformation_tests/test_gptq_torchfx_transformations.py -m precommit --html=${INSTALL_TEST_DIR}/TEST-torch_gptqpattern_tests.html --self-contained-html -v --tb=short
        env:
          TEST_DEVICE: CPU

      - name: MoE Fusion Test
        if: ${{ inputs.model_scope == 'precommit' }}
        run: |
          export PYTHONPATH=${MODEL_HUB_TESTS_INSTALL_DIR}:$PYTHONPATH
          python3 -m pip install optimum-intel==1.25.2 transformers==4.53.3 --upgrade
          python3 -m pytest ${MODEL_HUB_TESTS_INSTALL_DIR}/transformation_tests/test_moe_transformation.py -m precommit --html=${INSTALL_TEST_DIR}/TEST-torch_moe_tests.html --self-contained-html -vvv -s --tb=short -n 2
        env:
          TEST_DEVICE: CPU

      - name: Reformat unsupported ops file
        if: ${{ inputs.model_scope != 'precommit' && !cancelled()}}
        run: |
          python3 ${MODEL_HUB_TESTS_INSTALL_DIR}/pytorch/scripts/process_op_report.py ${INSTALL_TEST_DIR}/TEST-torch_unsupported_ops.log

      - name: Available storage after tests
        run: |
          echo "Available storage:"
          df -h

      - name: Upload Test Results
        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0
        if: ${{ !cancelled() }}
        with:
          name: test-results-torch-models-${{ inputs.model_scope }}
          path: |
            ${{ env.INSTALL_TEST_DIR }}/TEST-torch*
          if-no-files-found: 'error'