test(RELEASE-2331): multi-arch manifest list for push-to-addons-registry

elenagerman · elenagerman · commit 1b2c40f2af7c · 2026-06-03T15:41:13.000-04:00
Adds end-to-end coverage for multi-arch (amd64 + arm64) operator index
flows for the push-to-addons-registry pipeline.

Signed-off-by: Elena German &lt;elgerman@redhat.com&gt;
Assisted-by: Claude
diff --git a/integration-tests/push-to-addons-registry/README.md b/integration-tests/push-to-addons-registry/README.md
@@ -2,6 +2,9 @@
 
 This test validates the addon registry push pipeline functionality.
 
+The Konflux build is configured for a **multi-arch manifest list** (amd64 and arm64), matching common OSD addon index images.
+**Note:** This test patches the component’s PaC templates in the PR *before merge* to ensure the `build-platforms` param 
+includes `linux/amd64` and `linux/arm64`, so the push pipeline produces a multi-arch index for verification.
 ## Test-Specific Configuration
 
 ### Files Structure
diff --git a/integration-tests/push-to-addons-registry/resources/tenant/component.yaml b/integration-tests/push-to-addons-registry/resources/tenant/component.yaml
@@ -6,6 +6,7 @@ metadata:
     git-provider: github
     build.appstudio.openshift.io/request: configure-pac
     image.redhat.com/generate: '{"visibility": "public"}'
+    build.appstudio.openshift.io/pipeline: '{"name": "docker-build-multi-platform-oci-ta", "bundle": "latest"}'
   name: ${component_name}
   labels:
     originating-tool: "${originating_tool}"
diff --git a/integration-tests/push-to-addons-registry/test.sh b/integration-tests/push-to-addons-registry/test.sh
@@ -1,6 +1,8 @@
 # --- Global Script Variables (Defaults) ---
 CLEANUP="true"
 NO_CVE="true" # Default to true
+# GitHub API curl retries (override in CI/local: export GITHUB_API_CURL_RETRY=5)
+GITHUB_API_CURL_RETRY="${GITHUB_API_CURL_RETRY:-3}"
 
 # Variables that will be set by functions and used globally:
 # component_branch, component_base_branch, component_repo_name (from test.env or similar)
@@ -15,6 +17,110 @@ NO_CVE="true" # Default to true
 # component_push_plr_name (set by wait_for_plr_to_appear)
 # RELEASE_NAME, RELEASE_NAMESPACE (set and exported by wait_for_release)
 
+patch_component_source_before_merge() {
+  # CI often runs scripts under xtrace (bash -x). Disable tracing only while handling tokens.
+  local xtrace_was_on=0
+  case $- in
+    *x*) xtrace_was_on=1 ;;
+  esac
+  if [ "${xtrace_was_on}" -eq 1 ]; then
+    set +x
+  fi
+
+  echo "Patching component source BEFORE MERGE to ensure multi-arch build..."
+
+  # PaC GitHub token: do not export — scoped env only for the helper that requires GH_TOKEN.
+  # Vault file uses ${component_name} placeholders; envsubst runs at kubectl apply, not on disk.
+  local secret_value
+  secret_value=$(yq '. | select(.metadata.name | contains("pipelines-as-code-secret-")) | .stringData.password' \
+    "${SUITE_DIR}/resources/tenant/secrets/tenant-secrets.yaml" | head -n 1)
+
+  if [ -z "${secret_value}" ] || [ "${secret_value}" = "null" ]; then
+    log_error "PaC token not found in tenant secrets (pipelines-as-code-secret-*)"
+  fi
+
+  local file_names=(
+    ".tekton/${component_name}-pull-request.yaml"
+    ".tekton/${component_name}-push.yaml"
+  )
+  local head_sha api_msg pr_response
+  if ! pr_response=$(curl --retry "${GITHUB_API_CURL_RETRY}" -s --fail-with-body \
+    -H "Authorization: token ${secret_value}" \
+    "https://api.github.com/repos/${component_repo_name}/pulls/${pr_number}"); then
+    if api_msg=$(jq -r '.message // empty' <<< "${pr_response}" 2>/dev/null) && [ -n "${api_msg}" ]; then
+      log_error "GitHub API error fetching PR ${pr_number}: ${api_msg}"
+    else
+      log_error "failed to fetch PR ${pr_number} from ${component_repo_name} (check PaC token and repo access)"
+    fi
+  fi
+  head_sha=$(jq -r -e '.head.sha' <<< "${pr_response}") || {
+    log_error "missing or invalid .head.sha in PR ${pr_number} response"
+  }
+
+  for file_name in "${file_names[@]}"; do
+    local decoded_contents encoded_contents
+    local contents_response encoded_content_field
+    echo "Patching ${file_name}..."
+
+    if ! contents_response=$(curl --retry "${GITHUB_API_CURL_RETRY}" -s --fail-with-body \
+      -H "Authorization: token ${secret_value}" \
+      "https://api.github.com/repos/${component_repo_name}/contents/${file_name}?ref=${head_sha}"); then
+      if api_msg=$(jq -r '.message // empty' <<< "${contents_response}" 2>/dev/null) && [ -n "${api_msg}" ]; then
+        log_error "GitHub API error fetching ${file_name}: ${api_msg}"
+      else
+        log_error "failed to fetch ${file_name} at ref ${head_sha} from ${component_repo_name}"
+      fi
+    else
+      encoded_content_field=$(jq -r -e '.content' <<< "${contents_response}") || {
+        log_error "missing or invalid .content for ${file_name} in PR ${pr_number}"
+      }
+      if ! decoded_contents=$(printf '%s' "${encoded_content_field}" | base64 -d); then
+        log_error "failed to base64-decode ${file_name} from PR ${pr_number}"
+      fi
+      if [ -z "${decoded_contents}" ]; then
+        log_error "decoded contents for ${file_name} are empty"
+      fi
+    fi
+
+    encoded_contents=$(
+      set -eo pipefail
+      work_dir=$(mktemp -d)
+      trap 'rm -rf "${work_dir}"' EXIT
+      nopath_file_name=$(basename "${file_name}")
+      echo "${decoded_contents}" > "${work_dir}/${nopath_file_name}"
+
+      # Ensure linux/amd64 + linux/arm64 are present.
+      if yq -e '(.spec.params[]? | select(.name == "build-platforms") | .value | type) == "!!seq"' \
+        "${work_dir}/${nopath_file_name}" >/dev/null 2>&1; then
+        yq -i '(.spec.params[] | select(.name == "build-platforms") | .value) |= ([.[] | select(. != "linux/arm64")] + ["linux/arm64"])' \
+          "${work_dir}/${nopath_file_name}"
+        yq -i '(.spec.params[] | select(.name == "build-platforms") | .value) |= ([.[] | select(. != "linux/amd64")] + ["linux/amd64"])' \
+          "${work_dir}/${nopath_file_name}"
+      else
+        yq -i '.spec.params += [{"name": "build-platforms", "value": ["linux/amd64", "linux/arm64"]}]' \
+          "${work_dir}/${nopath_file_name}"
+      fi
+
+      base64 -w 0 < "${work_dir}/${nopath_file_name}"
+    ) || {
+      log_error "failed to patch ${file_name} for multi-arch build"
+    }
+
+    GH_TOKEN="${secret_value}" "${SCRIPT_DIR}/scripts/update-file-in-pull-request.sh" \
+      "${component_repo_name}" \
+      "${pr_number}" \
+      "${file_name}" \
+      "Update PaC templates for multi-arch build" \
+      "${encoded_contents}"
+  done
+
+  unset secret_value
+  if [ "${xtrace_was_on}" -eq 1 ]; then
+    set -x
+  fi
+  echo "✅️ Successfully patched component PaC templates for multi-arch."
+}
+
 # Function to verify Release contents
 # Relies on global variables: RELEASE_NAMES, RELEASE_NAMESPACE, SCRIPT_DIR, managed_namespace, managed_sa_name, NO_CVE
 verify_release_contents() {
@@ -29,13 +135,34 @@ verify_release_contents() {
           log_error "Could not retrieve Release JSON for ${RELEASE_NAME}"
       fi
 
-      echo "Release JSON: ${release_json}"
-
       local failures=0
-      local image_url mergerequest_url
+      local image_url mergerequest_url image_arches image_shasum released_status
 
       image_url=$(jq -r '.status.artifacts.images[0].urls[0] // ""' <<< "${release_json}")
       mergerequest_url=$(jq -r '.status.artifacts.merge_requests[0].url // ""' <<< "${release_json}")
+      # Release may list one arch per manifest/index entry (e.g. duplicate amd64); compare distinct sets.
+      # Strip optional linux/ prefix (e.g. linux/amd64 -> amd64). Default null/missing .arches to [].
+      image_arches=$(jq -r '(.status.artifacts.images[0].arches // [])
+        | map((tostring | split("/") | .[-1]))
+        | unique
+        | join(" ")' <<< "${release_json}")
+      image_shasum=$(jq -r '.status.artifacts.images[0].shasum // ""' <<< "${release_json}")
+      released_status=$(jq -r '.status.conditions[]? | select(.type=="Released") | .status // ""' <<< "${release_json}")
+
+      echo "Release fields under validation:"
+      echo "  Released: ${released_status}"
+      echo "  image_url: ${image_url}"
+      echo "  mergerequest_url: ${mergerequest_url}"
+      echo "  image_arches: ${image_arches}"
+      echo "  image_shasum: ${image_shasum}"
+
+      echo "Checking Released=True..."
+      if [ "${released_status}" = "True" ]; then
+        echo "✅️ Released=True"
+      else
+        echo "🔴 Released was not True (found: '${released_status}')"
+        failures=$((failures+1))
+      fi
 
       echo "Checking image_url..."
       if [ -n "${image_url}" ]; then
@@ -52,6 +179,39 @@ verify_release_contents() {
         failures=$((failures+1))
       fi
 
+      echo "Checking image arches include amd64 and arm64..."
+      if [[ " ${image_arches} " == *" amd64 "* && " ${image_arches} " == *" arm64 "* ]]; then
+        echo "✅️ Found required arches: ${image_arches}"
+      else
+        echo "🔴 Missing required arches (need: amd64 and arm64), found: '${image_arches}'"
+        failures=$((failures+1))
+      fi
+
+      echo "Checking image shasum (manifest list digest) is present..."
+      if [[ "${image_shasum}" == sha256:* ]]; then
+        echo "✅️ image_shasum: ${image_shasum}"
+      else
+        echo "🔴 image_shasum missing or invalid: '${image_shasum}'"
+        failures=$((failures+1))
+      fi
+
+      echo "Checking skopeo inspect succeeds for both arches (digest pull + registry auth)..."
+      if [ -n "${image_url}" ] && [[ "${image_shasum}" == sha256:* ]]; then
+        set +e
+        "${SCRIPT_DIR}/scripts/skopeo-verify-image.sh" \
+          "${image_url}" "${image_shasum}" \
+          "${SUITE_DIR}/resources/managed/secrets/managed-secrets.yaml" \
+          "amd64 arm64"
+        skopeo_rc=$?
+        set -e
+        if [ "${skopeo_rc}" -ne 0 ]; then
+          failures=$((failures+1))
+        fi
+      elif [ -n "${image_url}" ]; then
+        echo "🔴 Skipping skopeo multi-arch check: image_shasum missing or not sha256:*"
+        failures=$((failures+1))
+      fi
+
       if [ "${failures}" -gt 0 ]; then
         echo "🔴 Test has FAILED with ${failures} failure(s)!"
         failed_releases="${RELEASE_NAME} ${failed_releases}"
diff --git a/integration-tests/scripts/skopeo-verify-image.sh b/integration-tests/scripts/skopeo-verify-image.sh
@@ -1,12 +1,17 @@
 #!/usr/bin/env bash
 # Verifies that an image is pullable from its target registry using skopeo.
 #
-# Usage: skopeo-verify-image.sh <image_url> <image_shasum> <managed_secrets_yaml>
+# Usage: skopeo-verify-image.sh <image_url> <image_shasum> <managed_secrets_yaml> [arches]
 #
 # Arguments:
 #   image_url            - The published image URL (may include a tag or digest)
 #   image_shasum         - The image digest (e.g. sha256:abc123...)
 #   managed_secrets_yaml - Path to the decrypted managed-secrets.yaml file
+#   arches (optional)    - Space-separated list of GOARCH values (e.g. "amd64 arm64").
+#                          When set, verifies the digest is an OCI image index or Docker
+#                          manifest list (--raw mediaType), then runs skopeo inspect with
+#                          --override-arch for each arch. On failure, re-runs the failing
+#                          inspect without suppressing stderr so logs stay debuggable.
 #
 # Exits with 0 on success, 1 on failure.
 
@@ -15,28 +20,93 @@ set -euo pipefail
 image_url="${1:?image_url argument is required}"
 image_shasum="${2:?image_shasum argument is required}"
 managed_secrets_yaml="${3:?managed_secrets_yaml argument is required}"
+optional_arches="${4:-}"
 
-if [[ "${image_url}" == *"@"* ]]; then
-    STRIPPED_PULLSPEC="${image_url%@*}"
+STRIPPED_PULLSPEC="${image_url}"
+if [[ "${STRIPPED_PULLSPEC}" == *"@"* ]]; then
+    STRIPPED_PULLSPEC="${STRIPPED_PULLSPEC%@*}"
     echo "Stripped digest from: ${image_url} -> ${STRIPPED_PULLSPEC}"
-elif [[ "${image_url}" == *":"* ]]; then
-    STRIPPED_PULLSPEC="${image_url%:*}"
-    echo "Stripped tag from: ${image_url} -> ${STRIPPED_PULLSPEC}"
+fi
+
+# Tags appear only after the last '/'; registry ports (host:port) must not be stripped.
+if [[ "${STRIPPED_PULLSPEC}" == */* ]]; then
+    path_prefix="${STRIPPED_PULLSPEC%/*}"
+    final_segment="${STRIPPED_PULLSPEC##*/}"
 else
-    STRIPPED_PULLSPEC="${image_url}"
+    path_prefix=""
+    final_segment="${STRIPPED_PULLSPEC}"
+fi
+if [[ "${final_segment}" == *":"* ]]; then
+    final_segment="${final_segment%:*}"
+    if [[ -n "${path_prefix}" ]]; then
+        STRIPPED_PULLSPEC="${path_prefix}/${final_segment}"
+    else
+        STRIPPED_PULLSPEC="${final_segment}"
+    fi
+    echo "Stripped tag from: ${image_url} -> ${STRIPPED_PULLSPEC}"
+elif [[ "${STRIPPED_PULLSPEC}" == "${image_url}" ]]; then
     echo "No tag or digest found, using original as is: ${STRIPPED_PULLSPEC}"
 fi
 
 COMPLETE_PULLSPEC="${STRIPPED_PULLSPEC}@${image_shasum}"
 echo "New complete pullspec: ${COMPLETE_PULLSPEC}"
 
+# CI often runs scripts under xtrace (bash -x). Disable tracing while loading registry credentials.
+xtrace_was_on=0
+case $- in
+  *x*) xtrace_was_on=1 ;;
+esac
+if [ "${xtrace_was_on}" -eq 1 ]; then
+  set +x
+fi
+
 DOCKER_CONFIG="$(mktemp -d)"
 export DOCKER_CONFIG
+cleanup_skopeo_verify() {
+  if [ "${xtrace_was_on}" -eq 1 ]; then
+    set -x
+  fi
+  rm -rf "${DOCKER_CONFIG}"
+}
+trap cleanup_skopeo_verify EXIT
+
+dockerconfig_b64="$(
+  yq '. | select(.metadata.name | contains("push-")) | .data.".dockerconfigjson"' \
+    "${managed_secrets_yaml}" | head -n 1
+)"
+if [[ -z "${dockerconfig_b64}" || "${dockerconfig_b64}" == "null" ]]; then
+  echo "🔴 Failed to find push-* dockerconfigjson in managed secrets."
+  exit 1
+fi
+printf '%s' "${dockerconfig_b64}" | base64 -d > "${DOCKER_CONFIG}/config.json"
+unset dockerconfig_b64
 
-yq '. | select(.metadata.name | contains("push-")) | .data.".dockerconfigjson"' \
-    "${managed_secrets_yaml}" | base64 -d > "${DOCKER_CONFIG}/config.json"
+if [[ -n "${optional_arches}" ]]; then
+    raw_manifest="$(
+      skopeo inspect --tls-verify=true --retry-times 3 --raw "docker://${COMPLETE_PULLSPEC}" 2>/dev/null
+    )"
+    if ! media_type="$(jq -r '.mediaType // empty' <<< "${raw_manifest}")" || [[ -z "${media_type}" ]]; then
+      echo "🔴 Unable to determine manifest mediaType from raw inspect output."
+      echo "Raw manifest (first 200 chars): $(head -c 200 <<< "${raw_manifest}")"
+      exit 1
+    fi
+    if [[ "${media_type}" != "application/vnd.oci.image.index.v1+json" ]] &&
+       [[ "${media_type}" != "application/vnd.docker.distribution.manifest.list.v2+json" ]]; then
+        echo "🔴 Expected OCI image index or Docker manifest list, found mediaType: '${media_type}'"
+        exit 1
+    fi
+    echo "✅️ Verified multi-arch index (mediaType: ${media_type})."
 
-if skopeo inspect --tls-verify=true "docker://${COMPLETE_PULLSPEC}" &>/dev/null; then
+    for arch in ${optional_arches}; do
+        if skopeo inspect --tls-verify=true --retry-times 3 --override-arch "${arch}" "docker://${COMPLETE_PULLSPEC}" &>/dev/null; then
+            echo "✅️ Image '${COMPLETE_PULLSPEC}' can be inspected for arch ${arch}."
+        else
+            echo "🔴 Failed skopeo inspect for arch ${arch} on '${COMPLETE_PULLSPEC}'."
+            skopeo inspect --tls-verify=true --retry-times 3 --override-arch "${arch}" "docker://${COMPLETE_PULLSPEC}"
+            exit 1
+        fi
+    done
+elif skopeo inspect --tls-verify=true "docker://${COMPLETE_PULLSPEC}" &>/dev/null; then
     echo "✅️ Image '${COMPLETE_PULLSPEC}' can be pulled using skopeo."
 else
     echo "🔴 Failed to pull or inspect image '${COMPLETE_PULLSPEC}'."
