add the workaround for 4.14.to 4.16

Author: gpei

ci-operator/config/openshift/openshift-tests-private/openshift-openshift-tests-private-release-4.16__amd64-nightly-4.16-cpou-upgrade-from-4.14.yaml

@@ -86,6 +86,16 @@ tests:
     - ref: cucushift-upgrade-setedge-2hops
     - chain: openshift-upgrade-qe-test-cpou
     workflow: cucushift-installer-rehearse-vsphere-upi-encrypt
+- as: azure-ipi-workers-rhel8-f28
+  cron: 21 8 24 * *
+  steps:
+    cluster_profile: azure-qe
+    env:
+      BASE_DOMAIN: qe.azure.devcluster.openshift.com
+    test:
+    - ref: cucushift-upgrade-setedge-2hops
+    - chain: openshift-upgrade-qe-sanity-cpou
+    workflow: cucushift-installer-rehearse-azure-ipi-workers-rhel8
 zz_generated_metadata:
   branch: release-4.16
   org: openshift

ci-operator/jobs/openshift/openshift-tests-private/openshift-openshift-tests-private-release-4.16-periodics.yaml

@@ -1,4 +1,85 @@
 periodics:
+- agent: kubernetes
+  cluster: build09
+  cron: 21 8 24 * *
+  decorate: true
+  decoration_config:
+    skip_cloning: true
+  extra_refs:
+  - base_ref: release-4.16
+    org: openshift
+    repo: openshift-tests-private
+  labels:
+    ci-operator.openshift.io/cloud: azure4
+    ci-operator.openshift.io/cloud-cluster-profile: azure-qe
+    ci-operator.openshift.io/variant: amd64-nightly-4.16-cpou-upgrade-from-4.14
+    ci.openshift.io/generator: prowgen
+    pj-rehearse.openshift.io/can-be-rehearsed: "true"
+  name: periodic-ci-openshift-openshift-tests-private-release-4.16-amd64-nightly-4.16-cpou-upgrade-from-4.14-azure-ipi-workers-rhel8-f28
+  spec:
+    containers:
+    - args:
+      - --gcs-upload-secret=/secrets/gcs/service-account.json
+      - --image-import-pull-secret=/etc/pull-secret/.dockerconfigjson
+      - --lease-server-credentials-file=/etc/boskos/credentials
+      - --oauth-token-path=/usr/local/github-credentials/oauth
+      - --report-credentials-file=/etc/report/credentials
+      - --secret-dir=/secrets/ci-pull-credentials
+      - --target=azure-ipi-workers-rhel8-f28
+      - --variant=amd64-nightly-4.16-cpou-upgrade-from-4.14
+      command:
+      - ci-operator
+      image: ci-operator:latest
+      imagePullPolicy: Always
+      name: ""
+      resources:
+        requests:
+          cpu: 10m
+      volumeMounts:
+      - mountPath: /etc/boskos
+        name: boskos
+        readOnly: true
+      - mountPath: /secrets/ci-pull-credentials
+        name: ci-pull-credentials
+        readOnly: true
+      - mountPath: /secrets/gcs
+        name: gcs-credentials
+        readOnly: true
+      - mountPath: /usr/local/github-credentials
+        name: github-credentials-openshift-ci-robot-private-git-cloner
+        readOnly: true
+      - mountPath: /secrets/manifest-tool
+        name: manifest-tool-local-pusher
+        readOnly: true
+      - mountPath: /etc/pull-secret
+        name: pull-secret
+        readOnly: true
+      - mountPath: /etc/report
+        name: result-aggregator
+        readOnly: true
+    serviceAccountName: ci-operator
+    volumes:
+    - name: boskos
+      secret:
+        items:
+        - key: credentials
+          path: credentials
+        secretName: boskos-credentials
+    - name: ci-pull-credentials
+      secret:
+        secretName: ci-pull-credentials
+    - name: github-credentials-openshift-ci-robot-private-git-cloner
+      secret:
+        secretName: github-credentials-openshift-ci-robot-private-git-cloner
+    - name: manifest-tool-local-pusher
+      secret:
+        secretName: manifest-tool-local-pusher
+    - name: pull-secret
+      secret:
+        secretName: registry-pull-credentials
+    - name: result-aggregator
+      secret:
+        secretName: result-aggregator
 - agent: kubernetes
   cluster: build09
   cron: 58 14 25 * *

ci-operator/step-registry/cucushift/chainupgrade/toimage/cucushift-chainupgrade-toimage-commands.sh

@@ -308,6 +308,30 @@ EOF
     ansible-playbook -i "${SHARED_DIR}/ansible-hosts" /tmp/repo.yaml -vvv
 }
 
+function rhel_pre_unpause(){
+    echo "Running the workaround step before unpausing worker mcp"
+    local testcase="rhel"
+    cat > /tmp/rhel_pre_unpause.yaml <<-'EOF'
+---
+- name: RHEL pre-unpause playbook
+  hosts: workers
+  any_errors_fatal: true
+  gather_facts: false
+  vars:
+    required_packages:
+      - ose-azure-acr-image-credential-provider
+      - ose-gcp-gcr-image-credential-provider
+  tasks:
+  - name: Install required package on the node
+    dnf:
+      name: "{{ required_packages }}"
+      state: latest
+      disable_gpg_check: true
+EOF
+    ansible-inventory -i "${SHARED_DIR}/ansible-hosts" --list --yaml
+    ansible-playbook -i "${SHARED_DIR}/ansible-hosts" /tmp/rhel_pre_unpause.yaml -vvv
+}
+
 # Do sdn migration to ovn since sdn is not supported from 4.17 version
 function sdn2ovn(){
     oc patch network.operator.openshift.io cluster --type='merge'  -p='{"spec":{"defaultNetwork":{"ovnKubernetesConfig":{"ipv4":{"internalJoinSubnet": "100.65.0.0/16"}}}}}' 
@@ -1152,6 +1176,11 @@ for target in "${TARGET_RELEASES[@]}"; do
         run_command "oc get node -owide"
         if [[ $(oc get machineconfigpools worker -ojson | jq -r '.spec.paused') == "true" ]]; then
             echo "worker mcp are paused, it sounds eus upgrade, skip rhel worker upgrade here, should upgrade them after worker mcp unpaused"
+	    if [[ "${SOURCE_MINOR_VERSION}" == "14" ]]; then
+	        echo "Workaround for https://issues.redhat.com/browse/OCPBUGS-32057 in 4.14 to 4.16 cpou test"
+		rhel_repo
+	        rhel_pre_unpause
+	    fi
         else
             rhel_repo
             rhel_upgrade

ci-operator/step-registry/cucushift/upgrade/cpou/unpause-worker-mcp/cucushift-upgrade-cpou-unpause-worker-mcp-commands.sh

@@ -100,11 +100,5 @@ do
     printf "\n"
     unpause ${mcp}
     printf "\n"
-    if [[ "${current_minor_ver}" -lt "19" ]] && [[ $(oc get nodes -l node.openshift.io/os_id=rhel) != "" ]]; then
-        echo "Found rhel worker, this step is supposed to be used in eus upgrade, skipping mcp checking here, need to check it after rhel worker upgraded..."
-        run_command "oc get machineconfigpools"
-        run_command "oc get node -owide"
-    else
-        check_mcp ${mcp} "True"
-    fi
-done
+    check_mcp ${mcp} "True"
+done