openvino/.github/workflows/job_onnx_models_tests.yml at edf89e2dc1f8bda34e2a3e8bc076024702d2b469 · akashchi/openvino · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
name: ONNX Models tests

on:
  workflow_call:
    inputs:
      runner:
        description: 'Machine on which the tests would run'
        type: string
        required: true
      image:
        description: 'Docker image to use for the job'
        type: string
        required: false
        default: null

permissions: read-all

env:
  HF_HUB_CACHE_LIN: /mount/caches/huggingface
  HF_HUB_CACHE_WIN: "C:\\mount\\caches\\huggingface"

jobs:
  ONNX_Models_tests:
    name: ONNX Models tests
    timeout-minutes: 60
    runs-on: ${{ inputs.runner }}
    container:
      image: ${{ inputs.image }}
      volumes:
        - /mount:/mount
        - ${{ github.workspace }}:${{ github.workspace }} # Needed as ${{ github.workspace }} is not working correctly when using Docker
      options: "-e HF_TOKEN" # Needed for HuggingFace cache authentication
    defaults:
      run:
        shell: bash
    env:
      DEBIAN_FRONTEND: noninteractive # to prevent apt-get from waiting user input
      OPENVINO_REPO: ${{ github.workspace }}/openvino
      INSTALL_DIR: ${{ github.workspace }}/install
      INSTALL_TEST_DIR: ${{ github.workspace }}/install/tests
      ONNX_MODELS_PATH: ${{ github.workspace }}/onnx_test_models
      # instead of using static MODELS_SHARE_PATH
      # choose one of the replicas dynamically instead
      # depending on GITHUB_RUN_NUMBER variable
      NUMBER_OF_REPLICAS: 2
      ONNX_MODEL_ZOO_SHA: "5faef4c33eba0395177850e1e31c4a6a9e634c82"
      USE_SYSTEM_CACHE: False # Using remote HuggingFace cache
      LOGS_FOLDER: ${{ github.workspace }}/onnx_models_tests_logs"
      HF_HUB_VERBOSITY: debug
      TRANSFORMERS_VERBOSITY: debug
    if: ${{ github.event_name != 'merge_group' }}
    steps:
      # checkout action cleans up the workspace and have to be the first step
      - name: Fetch custom actions
        uses: ababushk/checkout@dd591a6a2ac25618db4eda86e7e0d938f88cf01b # cherry_pick_retries
        timeout-minutes: 15
        with:
          sparse-checkout: |
            .github/actions
          sparse-checkout-cone-mode: false
          submodules: 'false'

      - name: Download OpenVINO artifacts (wheels)
        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
        with:
          name: openvino_wheels
          path: ${{ env.INSTALL_DIR }}

      - name: Download OpenVINO artifacts (tests)
        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
        with:
          name: openvino_tests
          path: ${{ env.INSTALL_DIR }}

      - name: Extract OpenVINO tests
        run: pigz -dc openvino_tests.tar.gz | tar -xf - -v
        working-directory: ${{ env.INSTALL_DIR }}

      - name: Setup Variables
        run: |
          echo "HF_HUB_CACHE=${{ runner.os == 'Linux' && env.HF_HUB_CACHE_LIN || env.HF_HUB_CACHE_WIN }}" >> "$GITHUB_ENV"
          echo "HUGGINGFACE_HUB_CACHE=$HF_HUB_CACHE" >> "$GITHUB_ENV"
          echo "MODELS_SHARE_PATH=/mount/testdata$((GITHUB_RUN_NUMBER % NUMBER_OF_REPLICAS))" >> "$GITHUB_ENV"

      # Issue 148922
      # Can be a possible root cause for the bug
      # Uncomment only when there's a need
      # to update the models on the shared drive
      # - name: Fetch model_zoo_preprocess script
      #   uses: ababushk/checkout@dd591a6a2ac25618db4eda86e7e0d938f88cf01b # cherry_pick_retries
      #   with:
      #     sparse-checkout: |
      #       src/frontends/onnx/tests/tests_python/model_zoo_preprocess.sh
      #     sparse-checkout-cone-mode: false
      #     path: 'openvino'

      # - name: Update Models
      #   run: bash ${OPENVINO_REPO}/src/frontends/onnx/tests/tests_python/model_zoo_preprocess.sh -d ${MODELS_SHARE_PATH} -o -s "${{ env.ONNX_MODEL_ZOO_SHA }}"

      - name: Install OpenVINO Python wheels
        uses: ./.github/actions/install_ov_wheels
        with:
          wheels-dir-path: ${{ env.INSTALL_DIR }}
          wheels-to-install: 'openvino'

      - name: Install ONNX Models tests requirements
        run: python3 -m pip install -r ${INSTALL_TEST_DIR}/requirements_onnx

      - name: Install Python tests dependencies
        run: |
          # To enable pytest parallel features
          python3 -m pip install pytest-xdist[psutil] pytest-forked

      - name: Find ONNX models
        run: |
          find ${MODELS_SHARE_PATH} -depth -type f -name "*.onnx" -print

      - name: Collect ONNX Models tests
        run: |
          python3 -m pytest --backend="CPU" --model_zoo_dir="${MODELS_SHARE_PATH}" ${INSTALL_TEST_DIR}/onnx/tests/tests_python/test_zoo_models.py -v -n 1 -k 'not _cuda' --model_zoo_xfail --collect-only

      - name: ONNX Models Tests
        run: |
          python3 -m pytest --backend="CPU" --model_zoo_dir="${MODELS_SHARE_PATH}" ${INSTALL_TEST_DIR}/onnx/tests/tests_python/test_zoo_models.py -v -n auto --forked -k 'not _cuda' --model_zoo_xfail

      - name: Upload logs from pytest
        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0
        if: always()
        with:
          name: onnx_models_tests_logs
          path: ${{ env.LOGS_FOLDER }}
          if-no-files-found: 'ignore'