Use imagenet dataset from hf in conformance tests

.github/workflows/conformance_ptq.yml

@@ -0,0 +1,74 @@
+name: Post Training Quantization
+permissions: read-all
+
+on:
+  workflow_call:
+  workflow_dispatch:
+    inputs:
+      pull_request_number:
+        description: 'The pull request number'
+        default: ''
+      pytest_args:
+        description: 'Pytest arguments'
+        default: ''
+
+jobs:
+  examples-cpu:
+    name: Post Training Quantization [${{ matrix.group }}/6]
+    runs-on: ubuntu-latest-16-cores
+    timeout-minutes: 180
+    strategy:
+      fail-fast: false
+      matrix:
+        group: [1, 2, 3, 4, 5, 6]
+    defaults:
+      run:
+        shell: bash
+    steps:
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+        with:
+            lfs: true
+            fetch-depth: 0  # Fetch full history to allow checking out any branch or PR
+      - name: Fetch and Checkout the Pull Request Branch
+        if: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.pull_request_number != '' }}
+        run: |
+          git fetch origin pull/${{ github.event.inputs.pull_request_number }}/head:pr-${{ github.event.inputs.pull_request_number }}
+          git checkout pr-${{ github.event.inputs.pull_request_number }}
+      - uses: actions/setup-python@42375524e23c412d93fb67b49958b491fce71c38 # v5.4.0
+        with:
+          python-version: 3.10.14
+      - name: cpuinfo
+        run: cat /proc/cpuinfo
+      - name: Install NNCF and test requirements
+        run: pip install . -r tests/post_training/requirements.txt
+      - name: Print installed modules
+        run: pip list
+      - name: Run examples test scope
+        run: |
+          python -m pytest -s -ra tests/post_training/test_quantize_conformance.py::test_ptq_quantization \
+            --junit-xml=pytest-results.xml \
+            --durations-path=tests/post_training/data/ptq_test_durations.json \
+            --splitting-algorithm=least_duration \
+            --splits 6 \
+            --group ${{ matrix.group }} \
+            ${{ github.event.inputs.pytest_args || '' }}
+          ret=$?
+          [ $ret -eq 5 ] && [ -n "${{ github.event.inputs.pytest_args || '' }}" ]  && exit 0 || exit $ret
+        env:
+          TQDM_DISABLE: 1
+          HOME_HF: "/home/runner/hf_home"
+          HF_TOKEN: ${{ secrets.HF_TOKEN }}
+      - name: Print results.csv
+        if: ${{ !cancelled() }}
+        run: column -s, -t < tmp/results.csv || echo "no file"
+      - name: Upload artifact
+        if: ${{ !cancelled() }}
+        uses: actions/upload-artifact@65c4c4a1ddee5b72f698fdd19549f0f0fb45cf08 #v4.6.0
+        with:
+          name: ptq_results_${{ matrix.group }}
+          path: tmp/results.csv
+      - name: Test Summary
+        if: ${{ !cancelled() }}
+        run: |
+          pip install defusedxml==0.7.1
+          python .github/scripts/pytest_md_summary.py pytest-results.xml >> $GITHUB_STEP_SUMMARY

tests/post_training/README.md

@@ -23,17 +23,13 @@ pip install -r requirements.txt
 
 ## Data preparation
 
-## Imagenet
+Using datasets from huggingface, required set HF_TOKEN environment variable.
+For using imagenet-1k need to sign licence https://huggingface.co/datasets/mlx-vision/imagenet-1k.
 
-<data>/imagenet/val - name of path
-Since Torchvision `ImageFolder` class is used to work with data the ImageNet validation dataset should be structured accordingly. Below is an example of the `val` folder:
-
-```text
-n01440764
-n01695060
-n01843383
-...
-```
+> [!IMPORTANT]
+> Used modified version of loader imagenet-1k to download only validation subset.
+> To avoid any conflict with full dataset set another cache directory for this test.
+> https://huggingface.co/docs/datasets/en/cache#cache-directory
 
 ## Usage
 

tests/post_training/data/ptq_reference_data.yaml

@@ -37,7 +37,7 @@ torchvision/resnet18_backend_OV:
 torchvision/resnet18_backend_ONNX:
   metric_value: 0.6948
 torchvision/resnet18_backend_TORCH:
-  metric_value: 0.69152
+  metric_value: 0.69404
 torchvision/resnet18_backend_CUDA_TORCH:
   metric_value: 0.69152
 torchvision/resnet18_backend_FX_TORCH:
@@ -55,9 +55,9 @@ torchvision/resnet18_backend_CUDA_FX_TORCH:
 torchvision/mobilenet_v3_small_BC_backend_FP32:
   metric_value: 0.6766
 torchvision/mobilenet_v3_small_BC_backend_OV:
-  metric_value: 0.6681
+  metric_value: 0.66498
 torchvision/mobilenet_v3_small_BC_backend_ONNX:
-  metric_value: 0.6679
+  metric_value: 0.66556
 torchvision/mobilenet_v3_small_BC_backend_FX_TORCH:
   metric_value: 0.6679
   exception_xfail_reason:
@@ -109,19 +109,19 @@ timm/crossvit_9_240_backend_FP32:
 timm/crossvit_9_240_backend_ONNX:
   metric_value: 0.73484
 timm/crossvit_9_240_backend_OV:
-  metric_value: 0.72788
+  metric_value: 0.73346
 timm/crossvit_9_240_backend_TORCH:
-  metric_value: 0.72744
+  metric_value: 0.73364
 timm/darknet53_backend_CUDA_TORCH:
   metric_value: 0.79176
 timm/darknet53_backend_FP32:
   metric_value: 0.80006
 timm/darknet53_backend_ONNX:
-  metric_value: 0.79176
+  metric_value: 0.79578
 timm/darknet53_backend_OV:
-  metric_value: 0.79216
+  metric_value: 0.79604
 timm/darknet53_backend_TORCH:
-  metric_value: 0.79094
+  metric_value: 0.79578
 timm/deit3_small_patch16_224_backend_CUDA_TORCH:
   metric_value: 0.81246
 timm/deit3_small_patch16_224_backend_FP32:
@@ -141,7 +141,7 @@ timm/dla34_backend_ONNX:
 timm/dla34_backend_OV:
   metric_value: 0.74532
 timm/dla34_backend_TORCH:
-  metric_value: 0.74256
+  metric_value: 0.74362
 timm/dpn68_backend_CUDA_TORCH:
   metric_value: 0.75786
 timm/dpn68_backend_FP32:
@@ -151,59 +151,59 @@ timm/dpn68_backend_ONNX:
 timm/dpn68_backend_OV:
   metric_value: 0.75972
 timm/dpn68_backend_TORCH:
-  metric_value: 0.75868
+  metric_value: 0.7597
 timm/efficientnet_b0_BC_backend_FP32:
   metric_value: 0.77698
 timm/efficientnet_b0_BC_backend_ONNX:
-  metric_value: 0.77132
+  metric_value: 0.77328
 timm/efficientnet_b0_BC_backend_OV:
-  metric_value: 0.77166
+  metric_value: 0.77374
 timm/efficientnet_b0_backend_CUDA_TORCH:
   metric_value: 0.77124
 timm/efficientnet_b0_backend_FP32:
   metric_value: 0.77698
 timm/efficientnet_b0_backend_ONNX:
-  metric_value: 0.7719
+  metric_value: 0.77352
 timm/efficientnet_b0_backend_OV:
-  metric_value: 0.77104
+  metric_value: 0.77342
 timm/efficientnet_b0_backend_TORCH:
-  metric_value: 0.77042
+  metric_value: 0.77196
 timm/efficientnet_lite0_backend_CUDA_TORCH:
   metric_value: 0.75162
 timm/efficientnet_lite0_backend_FP32:
   metric_value: 0.75496
 timm/efficientnet_lite0_backend_ONNX:
   metric_value: 0.75184
 timm/efficientnet_lite0_backend_OV:
-  metric_value: 0.75176
+  metric_value: 0.75284
 timm/efficientnet_lite0_backend_TORCH:
   metric_value: 0.7517
 timm/hrnet_w18_backend_CUDA_TORCH:
   metric_value: 0.77178
 timm/hrnet_w18_backend_FP32:
   metric_value: 0.78124
 timm/hrnet_w18_backend_ONNX:
-  metric_value: 0.7743
+  metric_value: 0.77666
 timm/hrnet_w18_backend_OV:
-  metric_value: 0.7743
+  metric_value: 0.77636
 timm/hrnet_w18_backend_TORCH:
-  metric_value: 0.7722
+  metric_value: 0.77558
 timm/inception_resnet_v2_backend_CUDA_TORCH:
   metric_value: 0.80334
 timm/inception_resnet_v2_backend_FP32:
   metric_value: 0.80448
 timm/inception_resnet_v2_backend_ONNX:
   metric_value: 0.80396
 timm/inception_resnet_v2_backend_OV:
-  metric_value: 0.80422
+  metric_value: 0.80304
 timm/inception_resnet_v2_backend_TORCH:
   metric_value: 0.80334
 timm/mobilenetv2_050_BC_backend_FP32:
   metric_value: 0.6594
 timm/mobilenetv2_050_BC_backend_ONNX:
   metric_value: 0.65486
 timm/mobilenetv2_050_BC_backend_OV:
-  metric_value: 0.65332
+  metric_value: 0.65454
 timm/mobilenetv2_050_backend_CUDA_TORCH:
   metric_value: 0.6534
 timm/mobilenetv2_050_backend_FP32:
@@ -219,27 +219,27 @@ timm/mobilenetv3_small_050_backend_CUDA_TORCH:
 timm/mobilenetv3_small_050_backend_FP32:
   metric_value: 0.57906
 timm/mobilenetv3_small_050_backend_ONNX:
-  metric_value: 0.41828
+  metric_value: 0.42322
 timm/mobilenetv3_small_050_backend_OV:
-  metric_value: 0.41874
+  metric_value: 0.42372
 timm/mobilenetv3_small_050_backend_TORCH:
-  metric_value: 0.4267
+  metric_value: 0.4309
 timm/mobilenetv3_small_050_BC_backend_FP32:
   metric_value: 0.57906
 timm/mobilenetv3_small_050_BC_backend_ONNX:
   metric_value: 0.56556
 timm/mobilenetv3_small_050_BC_backend_OV:
-  metric_value: 0.5655
+  metric_value: 0.56658
 timm/regnetx_002_backend_CUDA_TORCH:
   metric_value: 0.68596
 timm/regnetx_002_backend_FP32:
   metric_value: 0.68756
 timm/regnetx_002_backend_ONNX:
-  metric_value: 0.6854
+  metric_value: 0.68422
 timm/regnetx_002_backend_OV:
   metric_value: 0.6852
 timm/regnetx_002_backend_TORCH:
-  metric_value: 0.68576
+  metric_value: 0.6845
 timm/resnest14d_backend_CUDA_TORCH:
   metric_value: 0.74898
 timm/resnest14d_backend_FP32:
@@ -253,7 +253,7 @@ timm/resnest14d_backend_TORCH:
 timm/swin_base_patch4_window7_224_backend_FP32:
   metric_value: 0.85274
 timm/swin_base_patch4_window7_224_backend_OV:
-  metric_value: 0.83566
+  metric_value: 0.8454
 timm/swin_base_patch4_window7_224_no_sq_backend_FP32:
   metric_value: 0.85274
 timm/swin_base_patch4_window7_224_no_sq_backend_CUDA_TORCH:
@@ -271,7 +271,7 @@ timm/tf_inception_v3_backend_ONNX:
 timm/tf_inception_v3_backend_OV:
   metric_value: 0.77748
 timm/tf_inception_v3_backend_TORCH:
-  metric_value: 0.77586
+  metric_value: 0.77732
 timm/vgg11_backend_CUDA_TORCH:
   metric_value: 0.688
 timm/vgg11_backend_FP32: