[IUO]Remove obsolete installation overhead tests and utilities (#3752)

##### Short description:

Remove the following tests that caused excessive false failures due to
their brittle design based on resource comparison before/after
installation:

- test_cnv_resources_installed_cluster_scoped: Compared cluster-scoped
resources before/after CNV installation. This test was unreliable
because cluster resources change dynamically from other operators.

- test_cnv_resources_installed_namespace_scoped: Compared namespaced
resources before/after CNV installation. This test repeatedly failed due
to ephemeral resources (Pods, ReplicaSets, PodMetrics, Jobs) created by
the cluster that were not expected. The allowlist approach could never
account for all dynamic resource variations across different cluster
configurations.

- test_install_hpp: Validated HostPathProvisioner installation. This
functionality is better validated through other mechanisms.

##### More details:

Also removed:
- validate_hpp_installation() utility function (only used by
test_install_hpp)
- get_all_resources() utility function and related constants (only used
by the removed tests)
- Resource comparison allowlists that were specific to these tests
- Unused fixtures: installation_data_dir,
before_installation_all_resources, after_installation_all_resources,
installed_hpp

The remaining tests (test_cnv_installation,
test_cnv_installation_alert_cleanup,
test_cnv_installation_without_hco_cr_alert,
test_cnv_installation_without_hco_cr_metrics,
test_default_storage_class_set) provide sufficient coverage for CNV
installation validation without the brittleness of resource comparison
tests.

##### What this PR does / why we need it:
Stabalize CI
##### Which issue(s) this PR fixes:
N/A
##### Special notes for reviewer:
N/A
##### jira-ticket:
https://issues.redhat.com/browse/CNV-75791

<!-- This is an auto-generated comment: release notes by coderabbit.ai
-->
## Summary by CodeRabbit

* **Tests**
* Removed resource validation test cases for cluster- and
namespace-scoped installations.
* Eliminated deprecated test configuration and large ignore/allowlist
data previously used by tests.
* Simplified test infrastructure by removing unused fixtures and utility
utilities.
* Cleaned up obsolete test scenarios and dependencies to streamline and
speed up test runs.
<!-- end of auto-generated comment: release notes by coderabbit.ai -->

Co-authored-by: Claude Haiku 4.5 <noreply@anthropic.com>

Author: rlobillo

tests/install_upgrade_operators/product_install/conftest.py

@@ -1,5 +1,4 @@
 import logging
-import os
 
 import bitmath
 import pytest
@@ -16,7 +15,6 @@
     HCO_NOT_INSTALLED_ALERT,
     OPENSHIFT_VIRTUALIZATION,
 )
-from tests.install_upgrade_operators.product_install.utils import get_all_resources
 from utilities.constants import (
     BREW_REGISTERY_SOURCE,
     CRITICAL_STR,
@@ -32,9 +30,6 @@
     TIMEOUT_10MIN,
     StorageClassNames,
 )
-from utilities.data_collector import (
-    get_data_collector_base_directory,
-)
 from utilities.infra import (
     create_ns,
     get_cluster_platform,
@@ -69,21 +64,6 @@
 LOGGER = logging.getLogger(__name__)
 
 
-@pytest.fixture(scope="session")
-def installation_data_dir():
-    return os.path.join(get_data_collector_base_directory(), "resource_information")
-
-
-@pytest.fixture(scope="session")
-def before_installation_all_resources(installation_data_dir):
-    return get_all_resources(file_name="before_installation", base_directory=installation_data_dir)
-
-
-@pytest.fixture(scope="module")
-def after_installation_all_resources(installation_data_dir):
-    return get_all_resources(file_name="after_installation", base_directory=installation_data_dir)
-
-
 @pytest.fixture(scope="module")
 def hyperconverged_directory(tmpdir_factory, is_production_source):
     if is_production_source:

tests/install_upgrade_operators/product_install/constants.py

@@ -1,189 +1,2 @@
-from utilities.constants import WIN_2K22, WIN_2K25, WIN_10, WIN_11
-
-IGNORE_KIND = [
-    "StorageProfile",
-    "ConsoleCLIDownload",
-    "ImageContentSourcePolicy",
-    "VirtualMachineClusterInstancetype",
-    "VirtualMachineClusterPreference",
-    "Event",
-]
-IGNORE_NAMESPACE = [
-    "openshift-marketplace",
-    "openshift-cnv",
-    "openshift-operator-lifecycle-manager",
-    "openshift-virtualization-os-images",
-]
-CLUSTER_RESOURCE_ALLOWLIST = {
-    "ClusterRole": [
-        "openshift-cnv-group-view",
-        "prometheus-k8s-ssp",
-        "cdi-cronjob",
-        "bridge-marker-cr",
-        "kubevirt-hyperconverged-operator",
-        "cdi-uploadproxy",
-        "cdi-apiserver",
-        "kubemacpool-manager-role",
-        "kubevirt-controller",
-        "kubevirt-hyperconverged-operator",
-        "kubevirt-handler",
-        "openshift-cnv-group-edit",
-        "openshift-cnv-group-admin",
-        "cdi",
-        "kubevirt-exportproxy",
-        "kubevirt-apiserver",
-        "template:view",
-        "kubevirt-hyperconverged-",
-        "olm.og.openshift-cnv-",
-        "kubevirt-ipam-controller-manager-role",
-        "kubevirt-synchronization-controller",
-        "kubevirt-migration-controller",
-    ],
-    "ClusterRoleBinding": [
-        "hostpath-provisioner-operator-service-system:auth-delegator",
-        "prometheus-k8s-ssp",
-        "cdi-cronjob",
-        "kubemacpool-manager-rolebinding",
-        "cdi-sa",
-        "kubevirt-hyperconverged-operator",
-        "bridge-marker-crb",
-        "ssp-operator-service-system:auth-delegator",
-        "cdi-uploadproxy",
-        "cdi-apiserver",
-        "hco-webhook-service-system:auth-delegator",
-        "kubevirt-controller",
-        "kubevirt-apiserver-auth-delegator",
-        "kubevirt-handler",
-        "kubevirt-exportproxy",
-        "kubevirt-apiserver",
-        "template-validator",
-        "kubevirt-hyperconverged-",
-        "olm.og.openshift-cnv-kubevirt-ipam-controller-manager-rolebinding",
-        "kubevirt-synchronization-controller",
-        "kubevirt-ipam-controller-manager-rolebinding",
-        "kubevirt-migration-sa",
-    ],
-    "Namespace": ["openshift-cnv", "openshift-virtualization-os-images"],
-    "Project": ["openshift-cnv", "openshift-virtualization-os-images"],
-    "ConsoleQuickStart": ["creating-virtual-machine", "upload-boot-source", "windows-bootsource-pipeline"],
-    "MachineConfig": ["rendered-worker", "rendered-master"],
-    "ValidatingWebhookConfiguration": [
-        "cdi-api-dataimportcron-validate",
-        "virt-template-validator",
-        "cdi-api-datavolume-validate",
-        "cdi-api-validate",
-        "virt-api-validator",
-        "virt-operator-validator",
-        "objecttransfer-api-validate",
-        "cdi-api-populator-validate",
-    ],
-    "Operator": ["kubevirt-hyperconverged.openshift-cnv"],
-    "MutatingWebhookConfiguration": [
-        "virt-api-mutator",
-        "kubemacpool-mutator",
-        "cdi-api-datavolume-mutate",
-        "cdi-api-pvc-mutate",
-        "kubevirt-ipam-controller-mutating-webhook-configuration",
-    ],
-    "SecurityContextConstraints": [
-        "linux-bridge",
-        "containerized-data-importer",
-        "kubevirt-controller",
-        "bridge-marker",
-        "kubevirt-handler",
-    ],
-    "NetworkAddonsConfig": ["cluster"],
-    "ConsoleCLIDownload": ["virtctl-clidownloads-kubevirt-hyperconverged"],
-    "PriorityClass": ["kubevirt-cluster-critical"],
-    "ConsolePlugin": ["kubevirt-plugin"],
-    "CDI": ["cdi-kubevirt-hyperconverged"],
-    "CDIConfig": ["config"],
-}
-NAMESPACED_IGNORE_KINDS = ["Event", "Template"]
-NAMESPACED_RESOURCE_ALLOWLIST = {
-    "openshift-kube-apiserver": {
-        "Pod": ["installer-", "revision-pruner-"],
-        "PodMetrics": ["installer-", "revision-pruner-"],
-    },
-    "openshift-monitoring": {
-        "Pod": ["metrics-server", "prometheus-k8s", "alertmanager-main"],
-        "PodMetrics": ["metrics-server", "prometheus-k8s", "alertmanager-main"],
-    },
-    "kube-system": {
-        "RoleBinding": [
-            "hostpath-provisioner-operator-service-auth-reader",
-            "hco-webhook-service-auth-reader",
-            "ssp-operator-service-auth-reader",
-        ]
-    },
-    "openshift-config-managed": {"ConfigMap": ["grafana-dashboard-kubevirt-top-consumers"]},
-    "openshift-storage": {
-        "Pod": ["csi-addons", "storageclient"],
-        "EndpointSlice": ["csi-addons"],
-        "PodMetrics": ["csi-addons"],
-        "ReplicaSet": ["csi-addons"],
-        "Job": ["storageclient"],
-    },
-    "openshift-console": {
-        "ReplicaSet": ["console"],
-        "Pod": ["console"],
-        "PodMetrics": ["console"],
-    },
-    "openshift-virtualization-os-images": {
-        "ServiceAccount": ["builder", "default", "deployer"],
-        "RoleBinding": [
-            "system:image-builders",
-            "system:image-pullers",
-            "system:deployers",
-        ],
-        "ConfigMap": ["openshift-service-ca.crt", "kube-root-ca.crt"],
-        "Role": ["os-images.kubevirt.io:view"],
-        "DataImportCron": [
-            "centos-stream10-image-cron",
-            "centos-stream9-image-cron",
-            "rhel9-image-cron",
-            "rhel10-image-cron",
-            "rhel8-image-cron",
-            "fedora-image-cron",
-        ],
-        "ImageTag": ["rhel8-guest:latest", "rhel9-guest:latest"],
-        "ImageStreamTag": ["rhel8-guest:latest", "rhel9-guest:latest"],
-        "DataSource": [
-            "centos-stream10",
-            "centos-stream8",
-            "centos-stream9",
-            "centos7",
-            "fedora",
-            "rhel10",
-            "rhel9",
-            "rhel8",
-            "rhel7",
-            WIN_10,
-            WIN_11,
-            "win2k16",
-            "win2k19",
-            WIN_2K22,
-            WIN_2K25,
-        ],
-        "ImageStream": ["rhel9-guest", "rhel8-guest"],
-        "Secret": [
-            "builder-token",
-            "deployer-token",
-            "default-dockercfg",
-            "builder-dockercfg",
-            "default-token",
-            "deployer-dockercfg",
-        ],
-        "ClusterServiceVersion": ["openshift-pipelines-operator"],
-        "DataVolume": [
-            "rhel8",
-            "rhel9",
-            "rhel10",
-            "fedora",
-            "centos-stream9",
-            "centos-stream10",
-        ],
-    },
-}
 OPENSHIFT_VIRTUALIZATION = "openshift-virtualization"
 HCO_NOT_INSTALLED_ALERT = "HCOInstallationIncomplete"

tests/install_upgrade_operators/product_install/test_install_openshift_virtualization.py

@@ -3,12 +3,7 @@
 import pytest
 
 from tests.install_upgrade_operators.product_install.constants import (
-    CLUSTER_RESOURCE_ALLOWLIST,
     HCO_NOT_INSTALLED_ALERT,
-    IGNORE_KIND,
-    IGNORE_NAMESPACE,
-    NAMESPACED_IGNORE_KINDS,
-    NAMESPACED_RESOURCE_ALLOWLIST,
 )
 from tests.install_upgrade_operators.product_install.utils import (
     validate_hpp_installation,
@@ -18,7 +13,6 @@
     PENDING_STR,
     TIMEOUT_10MIN,
 )
-from utilities.exceptions import ResourceMismatch
 from utilities.hco import wait_for_hco_conditions
 from utilities.infra import wait_for_pods_running
 from utilities.monitoring import (
@@ -42,7 +36,6 @@
 def test_cnv_installation_without_hco_cr_alert(
     prometheus,
     cnv_version_to_install_info,
-    before_installation_all_resources,
     installed_openshift_virtualization,
     alert_dictionary_hco_not_installed,
 ):
@@ -85,96 +78,6 @@ def test_cnv_installation_alert_cleanup(prometheus):
     wait_for_firing_alert_clean_up(prometheus=prometheus, alert_name=HCO_NOT_INSTALLED_ALERT)
 
 
-# TODO: check if this test is still required
-@pytest.mark.polarion("CNV-10076")
-@pytest.mark.order(after=CNV_ALERT_CLEANUP_TEST)
-@pytest.mark.dependency(
-    name="test_cnv_resources_installed_cluster_scoped",
-    depends=[CNV_INSTALLATION_TEST],
-)
-def test_cnv_resources_installed_cluster_scoped(
-    before_installation_all_resources,
-    after_installation_all_resources,
-):
-    mismatch_cluster_scoped = []
-    for kind in after_installation_all_resources["cluster-scoped"]:
-        if kind in IGNORE_KIND:
-            continue
-
-        diff_values = [
-            value
-            for value in list(
-                set(after_installation_all_resources["cluster-scoped"][kind])
-                - (set(before_installation_all_resources["cluster-scoped"].get(kind, [])))
-            )
-            if "kubevirt.io" not in value
-        ]
-        allowlisted_values = CLUSTER_RESOURCE_ALLOWLIST.get(kind, [])
-        if allowlisted_values and diff_values:
-            LOGGER.info(f"Expected allowlisted values for kind: {kind}: {allowlisted_values}")
-            LOGGER.warning(
-                f"Current difference in resources between before and after installation of cnv: {diff_values}"
-            )
-            diff_values = [value for value in diff_values if not value.startswith(tuple(allowlisted_values))]
-            if diff_values:
-                LOGGER.warning(f"After removing allowlisted resources, the diff is: {diff_values}")
-                mismatch_cluster_scoped.append({kind: diff_values})
-    if mismatch_cluster_scoped:
-        LOGGER.error(f"Mismatched cluster resources: {mismatch_cluster_scoped}")
-        raise ResourceMismatch(f"Unexpected cluster resources found post cnv installation: {mismatch_cluster_scoped}")
-
-
-# TODO: check if this test is still required
-@pytest.mark.order(after=CNV_ALERT_CLEANUP_TEST)
-@pytest.mark.dependency(
-    depends=[CNV_INSTALLATION_TEST],
-)
-@pytest.mark.polarion("CNV-10075")
-def test_cnv_resources_installed_namespace_scoped(
-    before_installation_all_resources,
-    after_installation_all_resources,
-):
-    namespaced_resource_before = before_installation_all_resources["namespaced"]
-    namespaced_resource_after = after_installation_all_resources["namespaced"]
-    mismatch_namespaced = {}
-    for namespace in namespaced_resource_after:
-        if namespace in IGNORE_NAMESPACE:
-            continue
-        LOGGER.info(f"Checking resources in {namespace}")
-        mismatch_namespaced[namespace] = {}
-        allowlisted_kinds = NAMESPACED_RESOURCE_ALLOWLIST.get(namespace, [])
-
-        for kind in after_installation_all_resources["namespaced"][namespace]:
-            if kind in NAMESPACED_IGNORE_KINDS:
-                continue
-            allowlisted_values = []
-            if allowlisted_kinds:
-                LOGGER.info(f"Currently allowlisted resources: {allowlisted_kinds}")
-                allowlisted_values = allowlisted_kinds.get(kind, [])
-            diff_value_ns = [
-                value
-                for value in list(
-                    set(namespaced_resource_after[namespace].get(kind, []))
-                    - (set(namespaced_resource_before.get(namespace, {}).get(kind, [])))
-                )
-                if "kubevirt.io" not in value
-            ]
-
-            if allowlisted_values and diff_value_ns:
-                LOGGER.warning(
-                    f"For namespace: {namespace}, kind: {kind} difference in before and after installation "
-                    f"resource(s): {diff_value_ns}"
-                )
-                diff_value_ns = [value for value in diff_value_ns if not value.startswith(tuple(allowlisted_values))]
-            if diff_value_ns:
-                LOGGER.warning(f"After removing allowlisted elements from the difference, mismatch is: {diff_value_ns}")
-                mismatch_namespaced[namespace][kind] = diff_value_ns
-    mismatch_namespaced = {key: value for key, value in mismatch_namespaced.items() if value}
-    if mismatch_namespaced:
-        LOGGER.error(f"Mismatched namespaced resources: {mismatch_namespaced}")
-        raise ResourceMismatch(f"Unexpected namespaced resources found post cnv installation: {mismatch_namespaced}")
-
-
 @pytest.mark.polarion("CNV-12453")
 @pytest.mark.order(after=CNV_INSTALLATION_TEST)
 # Dependency: CNV must be installed before storage class configuration can be verified