Wait for worker MCP to be updated before running RHEL worker upgrade (#65478)

* Wait for worker MCP to be updated before running RHEL worker upgrade

* remove useless lines

Author: gpei

ci-operator/step-registry/cucushift/chainupgrade/toimage/cucushift-chainupgrade-toimage-commands.sh

@@ -310,6 +310,30 @@ EOF
     ansible-playbook -i "${SHARED_DIR}/ansible-hosts" /tmp/repo.yaml -vvv
 }
 
+function rhel_pre_unpause(){
+    echo "Running the workaround step before unpausing worker mcp"
+    local testcase="rhel"
+    cat > /tmp/rhel_pre_unpause.yaml <<-'EOF'
+---
+- name: RHEL pre-unpause playbook
+  hosts: workers
+  any_errors_fatal: true
+  gather_facts: false
+  vars:
+    required_packages:
+      - ose-azure-acr-image-credential-provider
+      - ose-gcp-gcr-image-credential-provider
+  tasks:
+  - name: Install required package on the node
+    dnf:
+      name: "{{ required_packages }}"
+      state: latest
+      disable_gpg_check: true
+EOF
+    ansible-inventory -i "${SHARED_DIR}/ansible-hosts" --list --yaml
+    ansible-playbook -i "${SHARED_DIR}/ansible-hosts" /tmp/rhel_pre_unpause.yaml -vvv
+}
+
 # Do sdn migration to ovn since sdn is not supported from 4.17 version
 function sdn2ovn(){
     oc patch network.operator.openshift.io cluster --type='merge'  -p='{"spec":{"defaultNetwork":{"ovnKubernetesConfig":{"ipv4":{"internalJoinSubnet": "100.65.0.0/16"}}}}}' 
@@ -1154,6 +1178,12 @@ for target in "${TARGET_RELEASES[@]}"; do
         run_command "oc get node -owide"
         if [[ $(oc get machineconfigpools worker -ojson | jq -r '.spec.paused') == "true" ]]; then
             echo "worker mcp are paused, it sounds eus upgrade, skip rhel worker upgrade here, should upgrade them after worker mcp unpaused"
+	    #Temporary workaround for 4.14 to 4.16 cpou test with RHEL workers, would be removed until https://github.com/openshift/openshift-ansible/pull/12531 merged
+	    if [[ "${SOURCE_MINOR_VERSION}" == "14" ]]; then
+	        echo "Running workaround for https://issues.redhat.com/browse/OCPBUGS-32057 in 4.14 to 4.16 cpou test"
+		rhel_repo
+	        rhel_pre_unpause
+	    fi
         else
             rhel_repo
             rhel_upgrade

ci-operator/step-registry/cucushift/upgrade/cpou/unpause-worker-mcp/cucushift-upgrade-cpou-unpause-worker-mcp-commands.sh

@@ -90,8 +90,6 @@ for mcp in "${normal_mcp_arr[@]}"
 do
     check_mcp ${mcp} "True"
 done
-current_minor_ver="$(oc get clusterversion --no-headers | awk '{print $2}' | cut -f2 -d.)"
-echo -e "current minor version: ${current_minor_ver} \n"
 
 IFS=" " read -r -a arr <<<"$PAUSED_MCP_NAME"
 for mcp in "${arr[@]}";
@@ -100,11 +98,5 @@ do
     printf "\n"
     unpause ${mcp}
     printf "\n"
-    if [[ "${current_minor_ver}" -lt "19" ]] && [[ $(oc get nodes -l node.openshift.io/os_id=rhel) != "" ]]; then
-        echo "Found rhel worker, this step is supposed to be used in eus upgrade, skipping mcp checking here, need to check it after rhel worker upgraded..."
-        run_command "oc get machineconfigpools"
-        run_command "oc get node -owide"
-    else
-        check_mcp ${mcp} "True"
-    fi
-done
+    check_mcp ${mcp} "True"
+done