Test VM with MIG VGPU

tests/install_upgrade_operators/conftest.py

@@ -20,9 +20,9 @@
 )
 from tests.install_upgrade_operators.utils import (
     get_network_addon_config,
-    get_resource_by_name,
     get_resource_from_module_name,
 )
+from tests.utils import get_resource_by_name
 from utilities.constants import HOSTPATH_PROVISIONER_CSI, HPP_POOL
 from utilities.hco import ResourceEditorValidateHCOReconcile, get_hco_version
 from utilities.infra import (

tests/install_upgrade_operators/crypto_policy/utils.py

@@ -21,9 +21,9 @@
     TLS_INTERMEDIATE_CIPHERS_IANA_OPENSSL_SYNTAX,
 )
 from tests.install_upgrade_operators.utils import (
-    get_resource_by_name,
     get_resource_key_value,
 )
+from tests.utils import get_resource_by_name
 from utilities.constants import (
     CLUSTER,
     TIMEOUT_2MIN,

tests/install_upgrade_operators/feature_gates/test_default_featuregates.py

@@ -19,9 +19,9 @@
     RESOURCE_TYPE_STR,
 )
 from tests.install_upgrade_operators.utils import (
-    get_resource_by_name,
     get_resource_key_value,
 )
+from tests.utils import get_resource_by_name
 from utilities.constants import CDI_KUBEVIRT_HYPERCONVERGED, KUBEVIRT_HCO_NAME
 
 pytestmark = [pytest.mark.post_upgrade, pytest.mark.sno, pytest.mark.s390x, pytest.mark.skip_must_gather_collection]

tests/install_upgrade_operators/utils.py

@@ -270,19 +270,6 @@ def get_resource_from_module_name(related_obj, ocp_resources_submodule_list, adm
     )
 
 
-def get_resource_by_name(
-    resource_kind: Resource, name: str, admin_client: DynamicClient, namespace: str | None = None
-) -> Resource:
-    kwargs = {"name": name}
-    if namespace:
-        kwargs["namespace"] = namespace
-    kwargs["client"] = admin_client
-    resource = resource_kind(**kwargs)
-    if resource.exists:
-        return resource
-    raise ResourceNotFoundError(f"{resource_kind} {name} not found.")
-
-
 def get_resource_key_value(resource: Resource, key_name: str) -> Any:
     return benedict(
         resource.instance.to_dict()["spec"],

tests/utils.py

@@ -16,7 +16,7 @@
 from ocp_resources.datavolume import DataVolume
 from ocp_resources.kubevirt import KubeVirt
 from ocp_resources.node import Node
-from ocp_resources.resource import ResourceEditor
+from ocp_resources.resource import Resource, ResourceEditor
 from ocp_resources.storage_profile import StorageProfile
 from ocp_resources.virtual_machine import VirtualMachine
 from ocp_resources.virtual_machine_instance_migration import VirtualMachineInstanceMigration
@@ -689,3 +689,16 @@ def verify_rwx_default_storage(client: DynamicClient) -> None:
             f"Default storage class '{storage_class}' doesn't support RWX mode "
             f"(required: RWX, found: {found_mode or 'none'})"
         )
+
+
+def get_resource_by_name(
+    resource_kind: Resource, name: str, admin_client: DynamicClient, namespace: str | None = None
+) -> Resource:
+    kwargs = {"name": name}
+    if namespace:
+        kwargs["namespace"] = namespace
+    kwargs["client"] = admin_client
+    resource = resource_kind(**kwargs)
+    if resource.exists:
+        return resource
+    raise ResourceNotFoundError(f"{resource_kind} {name} not found.")

tests/virt/conftest.py

@@ -8,7 +8,6 @@
 from ocp_resources.deployment import Deployment
 from ocp_resources.infrastructure import Infrastructure
 from ocp_resources.performance_profile import PerformanceProfile
-from timeout_sampler import TimeoutExpiredError, TimeoutSampler
 
 from tests.utils import (
     verify_cpumanager_workers,
@@ -26,6 +25,7 @@
 )
 from tests.virt.node.gpu.utils import (
     apply_node_labels,
+    assert_mdev_bus_exists_on_nodes,
     toggle_vgpu_deploy_labels,
     wait_for_ds_ready,
 )
@@ -41,12 +41,10 @@
 from utilities.constants import (
     AMD,
     INTEL,
-    TIMEOUT_1MIN,
-    TIMEOUT_5SEC,
     NamespacesNames,
 )
 from utilities.exceptions import ResourceValueError, UnsupportedGPUDeviceError
-from utilities.infra import ExecCommandOnPod, get_nodes_with_label, get_resources_by_name_prefix, label_nodes
+from utilities.infra import get_nodes_with_label, get_resources_by_name_prefix, label_nodes
 from utilities.pytest_utils import exit_pytest_execution
 from utilities.virt import get_nodes_gpu_info, vm_instance_from_template
 
@@ -306,28 +304,7 @@ def non_existent_mdev_bus_nodes(workers_utility_pods, vgpu_ready_nodes):
     If it's not available, this means the nvidia-vgpu-manager-daemonset
     Pod might not be in running state in the nvidia-gpu-operator namespace.
     """
-    desired_bus = "mdev_bus"
-    non_existent_mdev_bus_nodes = []
-    for node in vgpu_ready_nodes:
-        pod_exec = ExecCommandOnPod(utility_pods=workers_utility_pods, node=node)
-        try:
-            for sample in TimeoutSampler(
-                wait_timeout=TIMEOUT_1MIN,
-                sleep=TIMEOUT_5SEC,
-                func=pod_exec.exec,
-                command=f"ls /sys/class | grep {desired_bus} || true",
-            ):
-                if sample:
-                    return
-        except TimeoutExpiredError:
-            non_existent_mdev_bus_nodes.append(node.name)
-    if non_existent_mdev_bus_nodes:
-        pytest.fail(
-            reason=(
-                f"On these nodes: {non_existent_mdev_bus_nodes} {desired_bus} is not available."
-                "Ensure that in 'nvidia-gpu-operator' namespace nvidia-vgpu-manager-daemonset Pod is Running."
-            )
-        )
+    assert_mdev_bus_exists_on_nodes(workers_utility_pods=workers_utility_pods, nodes=vgpu_ready_nodes)
 
 
 @pytest.fixture(scope="session")

tests/virt/node/gpu/constants.py

@@ -62,4 +62,18 @@
         MDEV_GRID_AVAILABLE_INSTANCES_STR: "4",
         MDEV_GRID_TYPE_STR: "nvidia-746",
     },
+    "10de:20b7": {
+        DEVICE_ID_STR: "10de:20b7",
+        GPU_DEVICE_NAME_STR: f"{GPU_DEVICE_MANUFACTURER}/NVIDIA_A30-1-6C",
+        VGPU_DEVICE_NAME_STR: f"{GPU_DEVICE_MANUFACTURER}/NVIDIA_A30-1-6C",
+        GPU_PRETTY_NAME_STR: "NVIDIA A30-1-6C",
+        VGPU_PRETTY_NAME_STR: "NVIDIA A30-1-6C",
+        MDEV_NAME_STR: "NVIDIA A30-1-6C",
+        MDEV_AVAILABLE_INSTANCES_STR: "4",
+        MDEV_TYPE_STR: "nvidia-689",
+        VGPU_GRID_NAME_STR: f"{GPU_DEVICE_MANUFACTURER}/A30_1_6C",
+        MDEV_GRID_NAME_STR: "NVIDIA A30-1-6C",
+        MDEV_GRID_AVAILABLE_INSTANCES_STR: "4",
+        MDEV_GRID_TYPE_STR: "nvidia-689",
+    },
 }

tests/virt/node/gpu/utils.py

@@ -1,9 +1,10 @@
 import logging
 import shlex
 
+import pytest
 from ocp_resources.resource import ResourceEditor
 from pyhelper_utils.shell import run_ssh_commands
-from timeout_sampler import TimeoutSampler
+from timeout_sampler import TimeoutExpiredError, TimeoutSampler
 
 from tests.virt.node.gpu.constants import (
     SANDBOX_VALIDATOR_DEPLOY_LABEL,
@@ -13,10 +14,12 @@
 from tests.virt.utils import fetch_gpu_device_name_from_vm_instance, verify_gpu_device_exists_in_vm
 from utilities.constants import (
     TCP_TIMEOUT_30SEC,
+    TIMEOUT_1MIN,
     TIMEOUT_2MIN,
     TIMEOUT_3MIN,
     TIMEOUT_5SEC,
 )
+from utilities.infra import ExecCommandOnPod
 from utilities.virt import restart_vm_wait_for_running_vm, running_vm
 
 LOGGER = logging.getLogger(__name__)
@@ -111,3 +114,37 @@ def toggle_vgpu_deploy_labels(gpu_nodes, nodes_with_supported_gpus, sandbox_vali
         apply_node_labels(nodes=nodes_with_supported_gpus, labels={VGPU_DEVICE_MANAGER_DEPLOY_LABEL: "true"})
     wait_for_ds_ready(ds=sandbox_validator_ds, expected=len(gpu_nodes))
     wait_for_ds_ready(ds=vgpu_device_manager_ds, expected=len(nodes_with_supported_gpus))
+
+
+def assert_mdev_bus_exists_on_nodes(workers_utility_pods, nodes):
+    """Assert that mdev_bus is present on every node in nodes.
+
+    Args:
+        workers_utility_pods: Utility pods used to run commands on nodes.
+        nodes: GPU nodes to check.
+
+    Raises:
+        pytest.fail: If mdev_bus is absent on any node after TIMEOUT_1MIN.
+    """
+    desired_bus = "mdev_bus"
+    missing_nodes = []
+    for node in nodes:
+        pod_exec = ExecCommandOnPod(utility_pods=workers_utility_pods, node=node)
+        try:
+            for sample in TimeoutSampler(
+                wait_timeout=TIMEOUT_1MIN,
+                sleep=TIMEOUT_5SEC,
+                func=pod_exec.exec,
+                command=f"ls /sys/class | grep {desired_bus} || true",
+            ):
+                if sample:
+                    break
+        except TimeoutExpiredError:
+            missing_nodes.append(node.name)
+    if missing_nodes:
+        pytest.fail(
+            reason=(
+                f"On these nodes: {missing_nodes} {desired_bus} is not available."
+                "Ensure that in 'nvidia-gpu-operator' namespace nvidia-vgpu-manager-daemonset Pod is Running."
+            )
+        )