enable multi-arch cluster support (#3755)

##### Short description:
This PR updates o-virt tests infra to:
1. unblock mono-arch test runs on heterogeneous clusters
2. update tests infra to support multi-arch testing

Assisted-by: Cursor

##### More details:
 - new cli param `--cpu-arch` to set arch type for the run
param can take single arch or multiple arch as comma-separated str (for
e.g. `--cpu-arch=amd64` || `--cpu-arch=arm64,amd64`)
 - new config file for multi-arch clusters
- `get_cluster_architecture` returns set of cluster nodes arch and is
cached
 - `get_nodes_cpu_architecture` removed
- added validation for cpu config (based on cluster nodes arch and
--cpu-arch param)
 - modified fedora container disk manifest to set arch in spec

##### What this PR does / why we need it:

##### Which issue(s) this PR fixes:

##### Special notes for reviewer:
The PR is using some hacks/work-arounds to avoid circular imports
breaking pytest run. This is due to `utilities` modules having tons of
mashed imports between each other. Resolving it inside this PR is way
out of scope and will be handled in future

##### jira-ticket:
https://issues.redhat.com/browse/CNV-74481


<!-- This is an auto-generated comment: release notes by coderabbit.ai
-->
## Summary by CodeRabbit

## Release Notes

* **New Features**
* Added multi-architecture (multiarch) testing support for clusters with
multiple CPU architectures.
* Introduced `--cpu-arch` command-line option to specify target CPU
architecture for test execution.
* Added `multiarch` test marker to identify and manage
architecture-specific tests.
* Implemented architecture validation to enforce CPU architecture
parameters during test collection and execution.

* **Tests**
* Added multi-architecture test configuration and validation framework.
<!-- end of auto-generated comment: release notes by coderabbit.ai -->

Author: vsibirsk

conftest.py

@@ -53,7 +53,6 @@
     config_default_storage_class,
     deploy_run_in_progress_config_map,
     deploy_run_in_progress_namespace,
-    generate_os_matrix_dicts,
     get_artifactory_server_url,
     get_base_matrix_name,
     get_cnv_version_explorer_url,
@@ -65,7 +64,9 @@
     separator,
     skip_if_pytest_flags_exists,
     stop_if_run_in_progress,
+    update_cpu_arch_related_config,
     update_latest_os_config,
+    validate_collected_tests_arch_params,
 )
 
 LOGGER = logging.getLogger(__name__)
@@ -86,6 +87,7 @@
     "numa",
     "cclm",
     "mtv",
+    "multiarch",
 ]
 
 TEAM_MARKERS = {
@@ -114,6 +116,7 @@
 def pytest_addoption(parser):
     matrix_group = parser.getgroup(name="Matrix")
     os_group = parser.getgroup(name="OS")
+    arch_group = parser.getgroup(name="Architecture")
     install_upgrade_group = parser.getgroup(name="Upgrade")
     storage_group = parser.getgroup(name="Storage")
     cluster_sanity_group = parser.getgroup(name="ClusterSanity")
@@ -217,6 +220,15 @@ def pytest_addoption(parser):
         help="Run matrix tests with latest CentOS",
     )
 
+    arch_group.addoption(
+        "--cpu-arch",
+        help="""
+             CPU architecture to use when running tests on heterogeneous clusters.
+             Single arch (e.g. amd64) or comma-separated combination (e.g. amd64,arm64).
+             Defines what OS matrix params to use and what CPU architecture to use for VMs.
+             """,
+    )
+
     # Storage addoption
     storage_group.addoption(
         "--default-storage-class",
@@ -783,7 +795,7 @@ def pytest_sessionstart(session):
     # with runtime storage_class_matrix value(s)
     py_config["system_storage_class_matrix"] = py_config.get("storage_class_matrix", [])
 
-    generate_os_matrix_dicts(os_dict=py_config)
+    update_cpu_arch_related_config(cpu_arch_option=session.config.getoption("--cpu-arch") or "")
     update_latest_os_config(session_config=session.config)
 
     matrix_addoptions = [matrix for matrix in session.config.invocation_params.args if "-matrix=" in matrix]
@@ -834,6 +846,7 @@ def pytest_sessionstart(session):
 
 
 def pytest_collection_finish(session):
+    validate_collected_tests_arch_params(session=session)
     if session.config.getoption("--collect-tests-markers"):
         get_tests_cluster_markers(items=session.items, filepath=session.config.getoption("--tests-markers-file"))
         pytest.exit(reason="Run with --collect-tests-markers. no tests are executed", returncode=0)

docs/ARCHITECTURE_SUPPORT.md

@@ -1,16 +1,19 @@
 # Test Images Architecture Support
 
 The tests can dynamically select test images based on the system's architecture.
-By default, the architecture is extracted from the node's `arch` label.
+By default, the architecture is extracted from the cluster nodes' `arch` label.
 For CI, or to run `--collect-only` without cluster access, this is controlled by the environment variable `OPENSHIFT_VIRTUALIZATION_TEST_IMAGES_ARCH`.
 Note: to run on the default architecture `amd64`, there's no need to set the environment variable.
 
-Supported architectures include:
+Supported architectures include (names aligned with Kubernetes/KubeVirt):
 
 - `amd64` (default, also refered to as x86_64)
 - `arm64`
 - `s390x` (currently work in progress)
 
+## Heterogeneous (multi-arch) clusters
+
+See [Multi-Architecture Clusters](MULTIARCH.md).
 
 ## Test markers
 To run tests on a specific architecture, add `-m <architecture>` to the pytest command.

docs/MULTIARCH.md

@@ -0,0 +1,13 @@
+# Multi-Architecture (Heterogeneous) Clusters
+
+Currently supported architectures for multi-arch runs: `amd64` and `arm64`.
+
+On clusters where nodes have different CPU architectures, you must pass `--cpu-arch` to select the architecture for the run. Use a single value (e.g. `--cpu-arch=amd64`) or, for tests marked with `multiarch`, a comma-separated list (e.g. `--cpu-arch=amd64,arm64`). Use the config file `tests/global_config_multiarch.py` and the `multiarch` marker for tests that run across multiple architectures. Do not pass `--cpu-arch` on homogeneous clusters.
+
+```bash
+uv run pytest --tc-file=tests/global_config_multiarch.py --cpu-arch=amd64 ...
+```
+
+## Limitations
+
+`*_os_matrix` variables are not created for multi-arch runs (when `--cpu-arch` contains multiple architectures, e.g. `--cpu-arch=amd64,arm64`).

docs/RUNNING_TESTS.md

@@ -92,6 +92,10 @@ Example for SNO cluster:
 
 `--tc-file=tests/global_config_sno.py --storage-class-matrix=lvms-vg1`
 
+### Running tests on multi-arch / heterogeneous clusters
+
+See [Multi-Architecture Clusters](MULTIARCH.md).
+
 #### Running tests with an admin client instead of an unprivileged client
 To run tests with an admin client only, pass `--tc=no_unprivileged_client:True` to pytest.
 
@@ -184,6 +188,7 @@ There are other parameters that can be passed to the test suite if needed.
 ```bash
 --tc-file=tests/global_config.py
 --tc-format=python
+--cpu-arch=amd64
 --junitxml /tmp/xunit_results.xml
 --jira
 ```

libs/vm/vm.py

@@ -2,23 +2,24 @@
 
 import uuid
 from dataclasses import asdict
-from typing import Any
+from typing import TYPE_CHECKING, Any
 
 from dacite import from_dict
-from kubernetes.dynamic import DynamicClient
-from ocp_resources.node import Node
 from ocp_resources.resource import ResourceEditor
-from ocp_resources.virtual_machine import VirtualMachine, VirtualMachineInstance
+from ocp_resources.virtual_machine import VirtualMachine
+from ocp_resources.virtual_machine_instance import VirtualMachineInstance
 from pytest_testconfig import config as py_config
 
 from libs.vm.spec import CloudInitNoCloud, ContainerDisk, Devices, Disk, Metadata, SpecDisk, VMISpec, VMSpec, Volume
 from tests.network.libs import cloudinit
 from utilities import infra
 from utilities.constants import CLOUD_INIT_DISK_NAME
-from utilities.cpu import get_nodes_cpu_architecture
 from utilities.network import IfaceNotFound
 from utilities.virt import get_oc_image_info, vm_console_run_commands
 
+if TYPE_CHECKING:
+    from kubernetes.dynamic import DynamicClient
+
 
 class BaseVirtualMachine(VirtualMachine):
     """
@@ -174,7 +175,7 @@ def container_image(base_image: str) -> str:
     image_info = get_oc_image_info(
         image=base_image,
         pull_secret=pull_secret,
-        architecture=get_nodes_cpu_architecture(nodes=list(Node.get())),
+        architecture=py_config["cpu_arch"],
     )
     return f"{base_image}@{image_info['digest']}"
 

pytest.ini

@@ -49,6 +49,7 @@ markers =
     amd64: Tests that can run on amd64-based cluster
     arm64: Tests that can run on ARM-based cluster
     s390x: Tests that can run on s390x-based cluster
+    multiarch: Tests that can run on multi-arch cluster
 
     ## Hardware requirements
     special_infra: Tests that requires special infrastructure. e.g. sriov, gpu etc.

tests/conftest.py

@@ -130,7 +130,6 @@
     find_common_cpu_model_for_live_migration,
     get_common_cpu_from_nodes,
     get_host_model_cpu,
-    get_nodes_cpu_architecture,
     get_nodes_cpu_model,
 )
 from utilities.data_utils import base64_encode_str, name_prefix
@@ -1034,8 +1033,8 @@ def skip_access_mode_rwo_scope_function(storage_class_matrix__function__):
 
 
 @pytest.fixture(scope="session")
-def nodes_cpu_architecture(nodes):
-    return get_nodes_cpu_architecture(nodes=nodes)
+def nodes_cpu_architecture():
+    return py_config["cpu_arch"]
 
 
 @pytest.fixture(scope="session")

tests/global_config.py

@@ -24,6 +24,7 @@
     HPP_CAPABILITIES,
     LINUX_BRIDGE,
     MONITORING_METRICS,
+    MULTIARCH,
     OS_FLAVOR_FEDORA,
     OVS_BRIDGE,
     PRODUCTION_CATALOG_SOURCE,
@@ -45,9 +46,10 @@
 )
 from utilities.storage import HppCsiStorageClass
 
-arch = get_cluster_architecture()
 global config
-global_config = pytest_testconfig.load_python(py_file=f"tests/global_config_{arch}.py", encoding="utf-8")
+cluster_arch = get_cluster_architecture()
+cluster_type = MULTIARCH if len(cluster_arch) > 1 else next(iter(cluster_arch))
+global_config = pytest_testconfig.load_python(py_file=f"tests/global_config_{cluster_type}.py", encoding="utf-8")
 
 
 def _get_default_storage_class(sc_list):
@@ -250,7 +252,7 @@ def _get_default_storage_class(sc_list):
     if not config:  # noqa: F821
         config: dict[str, Any] = {}
     val = locals()[_dir]
-    if type(val) not in [bool, list, dict, str, int]:
+    if type(val) not in [bool, list, dict, str, int, set]:
         continue
 
     if _dir in ["encoding", "py_file"]:

tests/global_config_multiarch.py

@@ -0,0 +1,66 @@
+from typing import Any
+
+from ocp_resources.datavolume import DataVolume
+
+from utilities.constants import (
+    AMD_64,
+    ARM_64,
+    CENTOS_STREAM9_PREFERENCE,
+    CENTOS_STREAM10_PREFERENCE,
+    OS_FLAVOR_FEDORA,
+    RHEL8_PREFERENCE,
+    RHEL9_PREFERENCE,
+    RHEL10_PREFERENCE,
+    StorageClassNames,
+)
+
+global config
+
+
+storage_class_matrix = [
+    {
+        StorageClassNames.IO2_CSI: {
+            "volume_mode": DataVolume.VolumeMode.BLOCK,
+            "access_mode": DataVolume.AccessMode.RWX,
+            "snapshot": True,
+            "online_resize": True,
+            "wffc": True,
+            "default": True,
+        }
+    },
+]
+
+storage_class_a = StorageClassNames.IO2_CSI
+storage_class_b = StorageClassNames.IO2_CSI
+
+os_matrix = {
+    AMD_64: {
+        "rhel_os_list": ["rhel-8-10", "rhel-9-6"],
+        "fedora_os_list": ["fedora-43"],
+        "centos_os_list": ["centos-stream-9"],
+        "windows_os_list": ["win-10", "win-2019", "win-11", "win-2022", "win-2025"],
+        "instance_type_rhel_os_list": [RHEL8_PREFERENCE, RHEL9_PREFERENCE, RHEL10_PREFERENCE],
+        "instance_type_fedora_os_list": [OS_FLAVOR_FEDORA],
+        "instance_type_centos_os_list": [CENTOS_STREAM9_PREFERENCE, CENTOS_STREAM10_PREFERENCE],
+    },
+    ARM_64: {
+        "rhel_os_list": ["rhel-9-6"],
+        "fedora_os_list": ["fedora-42"],
+        "centos_os_list": ["centos-stream-9"],
+        "instance_type_rhel_os_list": [RHEL10_PREFERENCE],
+        "instance_type_fedora_os_list": [OS_FLAVOR_FEDORA],
+    },
+}
+
+
+for _dir in dir():
+    if not config:  # noqa: F821
+        config: dict[str, Any] = {}
+    val = locals()[_dir]
+    if type(val) not in [bool, list, dict, str]:
+        continue
+
+    if _dir in ["encoding", "py_file"]:
+        continue
+
+    config[_dir] = locals()[_dir]  # noqa: F821

utilities/architecture.py

@@ -1,35 +1,38 @@
 import os
+from functools import cache
 
 from ocp_resources.node import Node
 
 from utilities.cluster import cache_admin_client
+from utilities.exceptions import UnsupportedCPUArchitectureError
 
 
-def get_cluster_architecture() -> str:
+@cache
+def get_cluster_architecture() -> set[str]:
     """
     Returns cluster architecture.
 
     To run in CI, where a cluster is not available, set `OPENSHIFT_VIRTUALIZATION_TEST_IMAGES_ARCH` env variable.
 
     Returns:
-        str: cluster architecture.
+        set[str]: cluster architectures.
 
     Raises:
-        ValueError: if architecture is not supported.
+        UnsupportedCPUArchitectureError: If unable to determine architecture.
     """
-    from utilities.constants import AMD_64, ARM_64, KUBERNETES_ARCH_LABEL, S390X
+    # Lazy import to avoid circular dependency
+    # TODO: remove when/if utilities modules are refactored
+    from utilities.constants import KUBERNETES_ARCH_LABEL
 
     # Needed for CI
-    arch = os.environ.get("OPENSHIFT_VIRTUALIZATION_TEST_IMAGES_ARCH")
-
-    if not arch:
-        # TODO: merge with `get_nodes_cpu_architecture`
-        # cache_admin_client is used here as this function is used to get the architecture when initialing pytest config
-        nodes: list[Node] = list(Node.get(client=cache_admin_client()))
-        nodes_cpu_arch = {node.labels[KUBERNETES_ARCH_LABEL] for node in nodes}
-        arch = next(iter(nodes_cpu_arch))
-
-    if arch not in (AMD_64, ARM_64, S390X):
-        raise ValueError(f"{arch} architecture in not supported")
-
-    return arch
+    if arch := os.environ.get("OPENSHIFT_VIRTUALIZATION_TEST_IMAGES_ARCH"):
+        return {arch}
+
+    # cache_admin_client is used here as this function is used to get the architecture when initialing pytest config
+    nodes: list[Node] = list(Node.get(client=cache_admin_client()))
+    cluster_archs = {node.labels[KUBERNETES_ARCH_LABEL] for node in nodes}
+    if not cluster_archs:
+        raise UnsupportedCPUArchitectureError(
+            "Cluster architecture could not be determined (no nodes found and env var unset)."
+        )
+    return cluster_archs