[4.18 cherry-pick of #9912] [RDR] [Co-situated hub] Hub recovery after site-failure and failover/relocate of DR workloads (#11964)

Signed-off-by: am-agrawa <[email protected]>

Author: am-agrawa

ocs_ci/deployment/deployment.py

@@ -3443,14 +3443,20 @@ def validate_secret_creation_oadp(self):
         except CommandFailed:
             raise ResourceNotFoundError("Secret Not found")
 
+    @retry(
+        exception_to_check=ResourceWrongStatusException,  # or a specific one
+        tries=8,
+        delay=15,
+        backoff=2,
+    )
     def validate_policy_compliance_status(
         self, resource_name, resource_namespace, compliance_state
     ):
         """
         Validate policy status for given resource
 
-        Raises:
-            ResourceWrongStatusException: Raised when resource state does not match
+        Returns: True if compliance check passes else raises ResourceWrongStatusException when resource state
+        does not match
 
         """
 
@@ -3462,6 +3468,7 @@ def validate_policy_compliance_status(
         compliance_status = compliance_output.get()
         if compliance_status["status"]["compliant"] == compliance_state:
             logger.info("Compliance status Matches ")
+            return True
         else:
             raise ResourceWrongStatusException("Compliance status does not match")
 

ocs_ci/framework/pytest_customization/marks.py

@@ -594,6 +594,11 @@
     reason="RDR UI failover or relocate config needed",
 )
 
+dr_hub_recovery = pytest.mark.skipif(
+    config.nclusters != 4,
+    reason="DR hub recovery requires 4th OCP cluster to be available for Passive hub",
+)
+
 # Filter warnings
 filter_insecure_request_warning = pytest.mark.filterwarnings(
     "ignore::urllib3.exceptions.InsecureRequestWarning"

ocs_ci/helpers/dr_helpers.py

@@ -29,6 +29,7 @@
     get_passive_acm_index,
     enable_mco_console_plugin,
     set_recovery_as_primary,
+    get_all_acm_indexes,
 )
 from ocs_ci.utility import version, templating
 from ocs_ci.utility.retry import retry
@@ -43,6 +44,7 @@
     run_cmd_verify_cli_output,
     find_cephblockpoolradosnamespace,
     find_cephfilesystemsubvolumegroup,
+    create_unique_resource_name,
 )
 
 logger = logging.getLogger(__name__)
@@ -1103,7 +1105,9 @@ def get_all_drpolicy():
 
 
 def verify_last_group_sync_time(
-    drpc_obj, scheduling_interval, initial_last_group_sync_time=None
+    drpc_obj,
+    scheduling_interval,
+    initial_last_group_sync_time=None,
 ):
     """
     Verifies that the lastGroupSyncTime for a given DRPC object is within the expected range.
@@ -1401,6 +1405,9 @@ def restore_backup():
     restore_index = config.cur_index
     config.switch_ctx(get_passive_acm_index())
     restore_schedule = templating.load_yaml(constants.DR_RESTORE_YAML)
+    restore_schedule["metadata"]["name"] = create_unique_resource_name(
+        resource_description="acm", resource_type="restore"
+    )
     restore_schedule_yaml = tempfile.NamedTemporaryFile(
         mode="w+", prefix="restore", delete=False
     )
@@ -1547,6 +1554,7 @@ def create_klusterlet_config():
     )
     templating.dump_data_to_temp_yaml(klusterlet_config, klusterlet_config_yaml.name)
     run_cmd(f"oc create -f {klusterlet_config_yaml.name}")
+    logger.info("Klusterletconfig is successfully created on the passive hub")
     config.switch_ctx(old_ctx)
 
 
@@ -1564,6 +1572,9 @@ def remove_parameter_klusterlet_config():
     klusterlet_config_obj.patch(
         resource_name=name, params=json.dumps(remove_op), format_type="json"
     )
+    logger.info(
+        "appliedManifestWorkEvictionGracePeriod and it's value is successfully removed from KlusterletConfig"
+    )
     config.switch_ctx(old_ctx)
 
 
@@ -1915,3 +1926,50 @@ def verify_last_kubeobject_protection_time(drpc_obj, kubeobject_sync_interval):
     logger.info("Verified lastKubeObjectProtectionTime value within expected range")
     config.switch_ctx(restore_index)
     return last_kubeobject_protection_time
+
+
+def configure_rdr_hub_recovery():
+    """
+    RDR helper function to create backup schedule on the active hub cluster needed for hub recovery
+    using backup and restore.
+
+    This function ensures all pre-reqs are verified before hub recovery is performed.
+
+    """
+    # Create backup-schedule on active hub
+    config.switch_acm_ctx()
+    logger.info("Create backup schedule on the active hub cluster")
+    create_backup_schedule()
+    wait_time = 420
+    logger.info(f"Wait {wait_time} seconds until backup is taken ")
+    time.sleep(wait_time)
+    logger.info(
+        "Check pre-reqs on both the hub clusters before performing hub recovery"
+    )
+    acm_indexes = get_all_acm_indexes()
+    for _ in acm_indexes:
+        config.switch_ctx(_)
+        verify_backup_is_taken()
+        # To avoid circular import
+        from ocs_ci.deployment.deployment import (
+            Deployment,
+            get_multicluster_dr_deployment,
+        )
+
+        dr_conf = Deployment().get_rdr_conf()
+        rdrclass_obj = get_multicluster_dr_deployment()(dr_conf)
+        rdrclass_obj.validate_dpa()
+        assert rdrclass_obj.validate_policy_compliance_status(
+            resource_name="backup-restore-enabled",
+            resource_namespace="open-cluster-management-backup",
+            compliance_state="Compliant",
+        )
+    config.switch_ctx(get_passive_acm_index())
+    logger.info(
+        "Add label for cluster-monitoring needed to fire VolumeSyncronizationDelay alert on the Hub cluster"
+    )
+    exec_cmd(
+        "oc label namespace openshift-operators openshift.io/cluster-monitoring='true'"
+    )
+    logger.info("All pre-reqs verified for performing hub recovery")
+    return True

ocs_ci/ocs/dr/dr_workload.py

@@ -72,6 +72,7 @@ def __init__(self, **kwargs):
 
         self.workload_type = kwargs.get("workload_type", constants.SUBSCRIPTION)
         self.workload_namespace = kwargs.get("workload_namespace", None)
+        self.pvc_interface = kwargs.get("pvc_interface", None)
         self.app_name = kwargs.get("app_name", None)
         self.workload_pod_count = kwargs.get("workload_pod_count")
         self.workload_pvc_count = kwargs.get("workload_pvc_count")
@@ -466,6 +467,7 @@ def __init__(self, **kwargs):
 
         self.workload_type = kwargs.get("workload_type", constants.APPLICATION_SET)
         self.workload_namespace = kwargs.get("workload_namespace", None)
+        self.pvc_interface = kwargs.get("pvc_interface", None)
         self.workload_pod_count = kwargs.get("workload_pod_count")
         self.workload_pvc_count = kwargs.get("workload_pvc_count")
         self.dr_policy_name = kwargs.get(

ocs_ci/templates/ocs-deployment/multicluster/backupschedule.yaml

@@ -4,7 +4,7 @@ metadata:
   name: schedule-acm
   namespace: open-cluster-management-backup
 spec:
-  veleroSchedule: 0 */1 * * *
-  veleroTtl: 48h
+  veleroSchedule: "*/5 * * * *"
+  veleroTtl: 96h
   useManagedServiceAccount: true
   managedServiceAccountTTL: 96h

ocs_ci/templates/ocs-deployment/multicluster/klusterlet_config.yaml

@@ -3,4 +3,4 @@ kind: KlusterletConfig
 metadata:
   name: global
 spec:
-  appliedManifestWorkEvictionGracePeriod: "24h"
+  appliedManifestWorkEvictionGracePeriod: "168h"

tests/conftest.py

@@ -6886,6 +6886,7 @@ def factory(
                 workload_dir=workload_details["workload_dir"],
                 workload_pod_count=workload_details["pod_count"],
                 workload_pvc_count=workload_details["pvc_count"],
+                pvc_interface=pvc_interface,
             )
             instances.append(workload)
             total_pvc_count += workload_details["pvc_count"]
@@ -6905,6 +6906,7 @@ def factory(
                 ],
                 workload_pvc_selector=workload_details["dr_workload_app_pvc_selector"],
                 appset_model=appset_model,
+                pvc_interface=pvc_interface,
             )
             instances.append(workload)
             total_pvc_count += workload_details["pvc_count"]