Merge branch 'bhamehta/track-actions' of github.com:bmehta001/Olive into bhamehta/track-actions

Author: bmehta001

.azure_pipelines/job_templates/olive-test-cpu-template.yaml

@@ -7,6 +7,8 @@ parameters:
   onnxruntime_nightly: false
   torch: 'torch'
   requirements_file: 'requirements-test-cpu.txt'
+  pytest_marker: ''
+  test_path: 'test'
 
 jobs:
 - job: ${{parameters.name}}
@@ -17,6 +19,8 @@ jobs:
     WINDOWS: ${{ parameters.windows}}
     python_version: ${{ parameters.python_version }}
     requirements_file: ${{ parameters.requirements_file }}
+    pytest_marker: ${{ parameters.pytest_marker }}
+    test_path: ${{ parameters.test_path }}
     PIP_CACHE_DIR: $(Pipeline.Workspace)/.cache/pip
     HF_HOME: $(Pipeline.Workspace)/.cache/huggingface
     PYTEST_BASETEMP: $(Pipeline.Workspace)/.pytest_basetemp
@@ -39,7 +43,7 @@ jobs:
         python -m pip install pytest
         python -m pip install -r $(Build.SourcesDirectory)/test/$(requirements_file)
         python -m pip list
-        coverage run --source=$(Build.SourcesDirectory)/olive -m pytest -v -s -p no:warnings --disable-warnings --log-cli-level=WARNING --junitxml=$(Build.SourcesDirectory)/logs/test-TestOlive.xml $(Build.SourcesDirectory)/test --basetemp $(PYTEST_BASETEMP)
+        coverage run --source=$(Build.SourcesDirectory)/olive -m pytest -v -s -p no:warnings --disable-warnings --log-cli-level=WARNING --junitxml=$(Build.SourcesDirectory)/logs/test-TestOlive.xml -m "$(pytest_marker)" $(Build.SourcesDirectory)/$(test_path) --basetemp $(PYTEST_BASETEMP)
         coverage xml
     displayName: Test Olive
 

.azure_pipelines/job_templates/olive-test-linux-gpu-template.yaml

@@ -13,6 +13,8 @@ parameters:
   requirements_file: 'requirements-test-gpu.txt'
   test_script: 'run_test.sh'
   onnxruntime_nightly: false
+  pytest_marker: ''
+  test_path: 'test'
 
 jobs:
 - job: ${{parameters.name}}
@@ -68,9 +70,10 @@ jobs:
       ${{ parameters.onnxruntime }} \
       ${{ parameters.onnxruntime_nightly }} \
       test/${{ parameters.requirements_file }} \
-      test \
+      ${{ parameters.test_path }} \
       false \
-      $(hf_token)
+      $(hf_token) \
+      "${{ parameters.pytest_marker }}"
     displayName: Run Tests in Docker
 
   - task: CredScan@3

.azure_pipelines/olive-ci-amd.yaml

@@ -0,0 +1,49 @@
+trigger:
+  branches:
+    include:
+    - main
+  paths:
+    include:
+    - olive/passes/quark_quantizer/*
+    - test/passes/quark_quantizer/*
+
+pr:
+  branches:
+    include:
+    - main
+  paths:
+    include:
+    - olive/passes/quark_quantizer/*
+    - test/passes/quark_quantizer/*
+
+variables:
+  runCodesignValidationInjection: false
+  skipComponentGovernanceDetection: true
+
+jobs:
+# Linux AMD unit tests
+- template: job_templates/olive-test-cpu-template.yaml
+  parameters:
+    name: Linux_CPU_AMD_Unit_Test
+    pool: $(OLIVE_POOL_UBUNTU2004)
+    pytest_marker: 'amd'
+    requirements_file: 'requirements-test-amd.txt'
+    test_path: 'test/passes/quark_quantizer'
+
+- template: job_templates/olive-test-linux-gpu-template.yaml
+  parameters:
+    name: Linux_GPU_AMD_Unit_Test
+    pool: $(OLIVE_POOL_UBUNTU2004_GPU_V100)
+    pytest_marker: 'amd'
+    requirements_file: 'requirements-test-amd.txt'
+    test_path: 'test/passes/quark_quantizer'
+
+# Windows OpenVINO unit tests
+- template: job_templates/olive-test-cpu-template.yaml
+  parameters:
+    name: Windows_CPU_AMD_Unit_Test
+    pool: $(OLIVE_POOL_WIN2019)
+    windows: True
+    pytest_marker: 'amd'
+    requirements_file: 'requirements-test-amd.txt'
+    test_path: 'test/passes/quark_quantizer'

.azure_pipelines/olive-ci-openvino.yaml

@@ -0,0 +1,49 @@
+trigger:
+  branches:
+    include:
+    - main
+  paths:
+    include:
+    - olive/passes/openvino/*
+    - test/passes/openvino/*
+
+pr:
+  branches:
+    include:
+    - main
+  paths:
+    include:
+    - olive/passes/openvino/*
+    - test/passes/openvino/*
+
+variables:
+  runCodesignValidationInjection: false
+  skipComponentGovernanceDetection: true
+
+jobs:
+# Linux OpenVINO unit tests
+- template: job_templates/olive-test-cpu-template.yaml
+  parameters:
+    name: Linux_CPU_OpenVINO_Unit_Test
+    pool: $(OLIVE_POOL_UBUNTU2004)
+    pytest_marker: 'openvino'
+    requirements_file: 'requirements-test-openvino.txt'
+    test_path: 'test/passes/openvino'
+
+- template: job_templates/olive-test-linux-gpu-template.yaml
+  parameters:
+    name: Linux_GPU_OpenVINO_Unit_Test
+    pool: $(OLIVE_POOL_UBUNTU2004_GPU_V100)
+    pytest_marker: 'openvino'
+    requirements_file: 'requirements-test-openvino.txt'
+    test_path: 'test/passes/openvino'
+
+# Windows OpenVINO unit tests
+- template: job_templates/olive-test-cpu-template.yaml
+  parameters:
+    name: Windows_CPU_OpenVINO_Unit_Test
+    pool: $(OLIVE_POOL_WIN2019)
+    windows: True
+    pytest_marker: 'openvino'
+    requirements_file: 'requirements-test-openvino.txt'
+    test_path: 'test/passes/openvino'

.azure_pipelines/olive-ci.yaml

@@ -32,15 +32,18 @@ jobs:
   parameters:
     name: Linux_CPU_CI_Unit_Test
     pool: $(OLIVE_POOL_UBUNTU2004)
+    pytest_marker: 'not openvino and not amd'
 
 - template: job_templates/olive-test-linux-gpu-template.yaml
   parameters:
     name: Linux_GPU_CI_Unit_Test
     pool: $(OLIVE_POOL_UBUNTU2004_GPU_V100)
+    pytest_marker: 'not openvino and not amd'
 
 # Windows unit tests
 - template: job_templates/olive-test-cpu-template.yaml
   parameters:
     name: Windows_CPU_CI_Unit_Test
     pool: $(OLIVE_POOL_WIN2019)
     windows: True
+    pytest_marker: 'not openvino and not amd'

.azure_pipelines/olive-ort-nightly.yaml

@@ -21,6 +21,7 @@ jobs:
     pool: $(OLIVE_POOL_UBUNTU2004)
     windows: False
     onnxruntime_nightly: true
+    pytest_marker: 'not openvino'
 
 # Linux GPU unit test
 - template: job_templates/olive-test-linux-gpu-template.yaml
@@ -29,6 +30,7 @@ jobs:
     pool: $(OLIVE_POOL_UBUNTU2004_GPU_V100)
     windows: False
     onnxruntime_nightly: true
+    pytest_marker: 'not openvino'
 
 # Windows unit test
 - template: job_templates/olive-test-cpu-template.yaml
@@ -37,3 +39,4 @@ jobs:
     pool: $(OLIVE_POOL_WIN2019)
     windows: True
     onnxruntime_nightly: true
+    pytest_marker: 'not openvino'

.azure_pipelines/scripts/run_test.sh

@@ -7,10 +7,14 @@
 # $5: Path to the test file to run
 # $6: Whether to use coverage tracking (true/false)
 # $7: HF Token
+# $8: Pytest marker (optional, e.g., "not openvino")
 
 # activate venv
 source olive-venv/bin/activate
 
+# Upgrade pip to avoid dependency resolution bugs
+pip install --upgrade pip
+
 # Step 1: Install PyTorch
 pip install "$1"
 
@@ -37,14 +41,19 @@ pip list
 
 # Step 4: Run tests with or without coverage tracking
 XML_PATH="/logs/TestOlive.xml"
+PYTEST_MARKER_ARGS=()
+if [ -n "$8" ]; then
+    PYTEST_MARKER_ARGS=(-m "$8")
+fi
+
 if [ "$6" = "true" ]; then
     echo "Running tests with coverage tracking..."
-    coverage run -m pytest -vv -s --junitxml="$XML_PATH" -p no:warnings --disable-warnings --log-cli-level=WARNING "$5"
+    coverage run -m pytest -vv -s --junitxml="$XML_PATH" -p no:warnings --disable-warnings --log-cli-level=WARNING "${PYTEST_MARKER_ARGS[@]}" "$5"
     coverage xml -o /logs/coverage.xml
 else
     echo "Starting pytest at $(date)"
     echo "Running tests without coverage tracking..."
-    timeout 1100 python -m pytest -vv -s --junitxml="$XML_PATH" -p no:warnings --disable-warnings --log-cli-level=WARNING "$5"
+    timeout 1100 python -m pytest -vv -s --junitxml="$XML_PATH" -p no:warnings --disable-warnings --log-cli-level=WARNING "${PYTEST_MARKER_ARGS[@]}" "$5"
     exit_code=$?
     echo "pytest exited with code $exit_code"
 

olive/cli/benchmark.py

@@ -85,7 +85,8 @@ def _get_run_config(self, tempdir: str) -> dict:
         assert input_model_config["type"].lower() in {
             "hfmodel",
             "pytorchmodel",
-        }, "Only HfModel and PyTorchModel are supported in benchmark command."
+            "onnxmodel",
+        }, "Only HfModel, PyTorchModel and OnnxModel are supported in benchmark command."
 
         to_replace = [
             ("input_model", input_model_config),

olive/passes/onnx/model_builder.py

@@ -25,6 +25,7 @@
 from olive.passes import Pass
 from olive.passes.olive_pass import PassConfigParam
 from olive.passes.pass_config import BasePassConfig
+from olive.search.search_parameter import Boolean, Categorical
 
 logger = logging.getLogger(__name__)
 
@@ -83,11 +84,15 @@ def _default_config(cls, accelerator_spec: AcceleratorSpec) -> dict[str, PassCon
             "int4_block_size": PassConfigParam(
                 type_=ModelBuilder.BlockSize,
                 required=False,
+                search_defaults=Categorical(
+                    [ModelBuilder.BlockSize.B32, ModelBuilder.BlockSize.B64, ModelBuilder.BlockSize.B128]
+                ),
                 description="Specify the block_size for int4 quantization. Acceptable values: 16/32/64/128/256.",
             ),
             "int4_is_symmetric": PassConfigParam(
                 type_=bool,
                 required=False,
+                search_defaults=Boolean(),
                 description="Specify whether symmetric or asymmetric INT4 quantization needs to be used.",
             ),
             "int4_op_types_to_quantize": PassConfigParam(
@@ -106,6 +111,14 @@ def _default_config(cls, accelerator_spec: AcceleratorSpec) -> dict[str, PassCon
             "int4_algo_config": PassConfigParam(
                 type_=str,
                 required=False,
+                search_defaults=Categorical(
+                    [
+                        "default",
+                        "rtn",
+                        "k_quant_mixed",
+                        "k_quant_last",
+                    ]
+                ),
                 description="Specify the INT4 quantization algorithm to use in GenAI Model Builder",
             ),
             "use_qdq": PassConfigParam(

olive/passes/pytorch/selective_mixed_precision.py

@@ -20,6 +20,7 @@
 from olive.passes import Pass
 from olive.passes.pass_config import BasePassConfig, PassConfigParam
 from olive.passes.pytorch.train_utils import get_calibration_dataset, kl_div_loss, load_hf_base_model
+from olive.search.search_parameter import Categorical
 
 if TYPE_CHECKING:
     from olive.hardware.accelerator import AcceleratorSpec
@@ -65,7 +66,14 @@ def _default_config(cls, accelerator_spec: AcceleratorSpec) -> dict[str, PassCon
         return {
             "algorithm": PassConfigParam(
                 type_=SelectiveMixedPrecision.Algorithm,
-                required=True,
+                required=False,
+                search_defaults=Categorical(
+                    [
+                        SelectiveMixedPrecision.Algorithm.K_QUANT_DOWN,
+                        SelectiveMixedPrecision.Algorithm.K_QUANT_MIXED,
+                        SelectiveMixedPrecision.Algorithm.K_QUANT_LAST,
+                    ]
+                ),
                 description="The algorithm to use for mixed precision.",
             ),
             "bits": PassConfigParam(