chore(ci): fix rsync interference & droute re-enable (#2414)

Author: zdrapela

.ibm/pipelines/utils.sh

@@ -95,26 +95,42 @@ droute_send() {
       .targets.reportportal.processing.tfa.auto_finalization_threshold = ($auto_finalization_treshold | tonumber)
       ' data_router/data_router_metadata_template.json > "${ARTIFACT_DIR}/${project}/${metadata_output}"
 
-    oc rsync --progress=true --include="${metadata_output}" --include="${JUNIT_RESULTS}" --exclude="*" -n "${droute_project}" "${ARTIFACT_DIR}/${project}/" "${droute_project}/${droute_pod_name}:${temp_droute}/"
+    # Send test by rsync to bastion pod.
+    local max_attempts=5
+    local wait_seconds=4
+    for ((i = 1; i <= max_attempts; i++)); do
+      echo "Attempt ${i} of ${max_attempts} to rsync test resuls to bastion pod."
+      if output=$(oc rsync --progress=true --include="${metadata_output}" --include="${JUNIT_RESULTS}" --exclude="*" -n "${droute_project}" "${ARTIFACT_DIR}/${project}/" "${droute_project}/${droute_pod_name}:${temp_droute}/" 2>&1); then
+        echo "$output"
+        break
+      fi
+      if ((i == max_attempts)); then
+        echo "Failed to rsync test results after ${max_attempts} attempts."
+        echo "Last rsync error details:"
+        echo "${output}"
+        echo "Troubleshooting steps:"
+        echo "1. Restart $droute_pod_name in $droute_project project/namespace"
+      fi
+    done
 
     # "Install" Data Router
     oc exec -n "${droute_project}" "${droute_pod_name}" -- /bin/bash -c "
-      curl -fsSLk -o /tmp/droute-linux-amd64 'https://${DATA_ROUTER_NEXUS_HOSTNAME}/nexus/repository/dno-raw/droute-client/${droute_version}/droute-linux-amd64' \
-      && chmod +x /tmp/droute-linux-amd64 \
-      && /tmp/droute-linux-amd64 version"
+      curl -fsSLk -o ${temp_droute}/droute-linux-amd64 'https://${DATA_ROUTER_NEXUS_HOSTNAME}/nexus/repository/dno-raw/droute-client/${droute_version}/droute-linux-amd64' \
+      && chmod +x ${temp_droute}/droute-linux-amd64 \
+      && ${temp_droute}/droute-linux-amd64 version"
 
     # Send test results through DataRouter and save the request ID.
     local max_attempts=5
     local wait_seconds=1
     for ((i = 1; i <= max_attempts; i++)); do
       echo "Attempt ${i} of ${max_attempts} to send test results through Data Router."
       if output=$(oc exec -n "${droute_project}" "${droute_pod_name}" -- /bin/bash -c "
-        /tmp/droute-linux-amd64 send --metadata ${temp_droute}/${metadata_output} \
-        --url '${DATA_ROUTER_URL}' \
-        --username '${DATA_ROUTER_USERNAME}' \
-        --password '${DATA_ROUTER_PASSWORD}' \
-        --results '${temp_droute}/${JUNIT_RESULTS}' \
-        --verbose" 2>&1); then
+        ${temp_droute}/droute-linux-amd64 send --metadata ${temp_droute}/${metadata_output} \
+          --url '${DATA_ROUTER_URL}' \
+          --username '${DATA_ROUTER_USERNAME}' \
+          --password '${DATA_ROUTER_PASSWORD}' \
+          --results '${temp_droute}/${JUNIT_RESULTS}' \
+          --verbose" 2>&1); then
         if DATA_ROUTER_REQUEST_ID=$(echo "$output" | grep "request:" | awk '{print $2}') &&
           [ -n "$DATA_ROUTER_REQUEST_ID" ]; then
           echo "Test results successfully sent through Data Router."
@@ -133,12 +149,69 @@ droute_send() {
         echo "3. Ask for help at Slack: #forum-dno-datarouter"
       fi
     done
-    
+
+    # shellcheck disable=SC2317
+    if [[ "$JOB_NAME" == *periodic-* ]]; then
+      local max_attempts=30
+      local wait_seconds=2
+      set +e
+      for ((i = 1; i <= max_attempts; i++)); do
+        # Get DataRouter request information.
+        DATA_ROUTER_REQUEST_OUTPUT=$(oc exec -n "${droute_project}" "${droute_pod_name}" -- /bin/bash -c "
+          ${temp_droute}/droute-linux-amd64 request get \
+          --url ${DATA_ROUTER_URL} \
+          --username ${DATA_ROUTER_USERNAME} \
+          --password ${DATA_ROUTER_PASSWORD} \
+          ${DATA_ROUTER_REQUEST_ID}")
+        # Try to extract the ReportPortal launch URL from the request. This fails if it doesn't contain the launch URL.
+        REPORTPORTAL_LAUNCH_URL=$(echo "$DATA_ROUTER_REQUEST_OUTPUT" | yq e '.targets[0].events[] | select(.component == "reportportal-connector") | .message | fromjson | .[0].launch_url' -)
+        if [[ -n "$REPORTPORTAL_LAUNCH_URL" ]]; then
+          reportportal_slack_alert $release_name $REPORTPORTAL_LAUNCH_URL
+          return 0
+        else
+          echo "Attempt ${i} of ${max_attempts}: ReportPortal launch URL not ready yet."
+          sleep "${wait_seconds}"
+        fi
+      done
+      set -e
+    fi
     oc exec -n "${droute_project}" "${droute_pod_name}" -- /bin/bash -c "rm -rf ${temp_droute}/*"
     if [ -n "${PULL_NUMBER:-}" ]; then
       set -e
     fi
   ) # Close subshell
   rm -f "$temp_kubeconfig" # Destroy temporary KUBECONFIG
   oc whoami --show-server
+}
+
+reportportal_slack_alert() {
+  local release_name=$1
+  local reportportal_launch_url=$2
+
+  if [[ "$release_name" == *rbac* ]]; then
+    RUN_TYPE="rbac-nightly"
+  else
+    RUN_TYPE="nightly"
+  fi
+  if [[ ${RESULT} -eq 0 ]]; then
+    RUN_STATUS_EMOJI=":done-circle-check:"
+    RUN_STATUS="passed"
+  else
+    RUN_STATUS_EMOJI=":failed:"
+    RUN_STATUS="failed"
+  fi
+  jq -n \
+    --arg run_status "$RUN_STATUS" \
+    --arg run_type "$RUN_TYPE" \
+    --arg reportportal_launch_url "$reportportal_launch_url" \
+    --arg job_name "$JOB_NAME" \
+    --arg run_status_emoji "$RUN_STATUS_EMOJI" \
+    '{
+      "RUN_STATUS": $run_status,
+      "RUN_TYPE": $run_type,
+      "REPORTPORTAL_LAUNCH_URL": $reportportal_launch_url,
+      "JOB_NAME": $job_name,
+      "RUN_STATUS_EMOJI": $run_status_emoji
+    }' > /tmp/data_router_slack_message.json
+  curl -X POST -H 'Content-type: application/json' --data @/tmp/data_router_slack_message.json  $SLACK_DATA_ROUTER_WEBHOOK_URL
 }