feat(RELEASE-1989): python for publish-index-images tasks

tasks/internal/publish-index-image-task/publish-index-image-task.yaml

@@ -64,7 +64,8 @@ spec:
       securityContext:
         runAsUser: 1001
       image: >-
-        quay.io/konflux-ci/release-service-utils@sha256:5546fa78d3c88d7b6a2e8cff8902f7757f00541d0bbaf113b9f293133894afa3
+        quay.io/jluza/release-service-utils:RELEASE-1989
+      imagePullPolicy: Always
       computeResources:
         limits:
           memory: 64Mi
@@ -74,52 +75,11 @@ spec:
       script: |
         #!/usr/bin/env bash
         set -euo pipefail
-
-        SOURCE_INDEX_CREDENTIAL="$(cat /mnt/publishingCredentials/sourceIndexCredential)"
-        TARGET_INDEX_CREDENTIAL="$(cat /mnt/publishingCredentials/targetIndexCredential)"
-
-        PATH=/bin:/usr/bin:/usr/local/bin
-        export PATH
-
-        SOURCE_AUTH_ARGS=()
-        if [[ ! "$(params.sourceIndex)" =~ ^registry-proxy(\-stage)?.engineering.redhat.com ]]; then
-            SOURCE_AUTH_ARGS=("--src-creds" "${SOURCE_INDEX_CREDENTIAL}")
-        fi
-
-        TARGET_AUTH_ARGS=("--dest-creds" "${TARGET_INDEX_CREDENTIAL}")
-
-        # Extract digest from pull spec
-        SOURCE_INDEX="$(params.sourceIndex)"
-        SOURCE_DIGEST="${SOURCE_INDEX##*@}"
-
-        echo "Getting target image digest: $(params.targetIndex)"
-        if TARGET_DIGEST=$(skopeo inspect \
-            "docker://$(params.targetIndex)" \
-            --format '{{.Digest}}' \
-            --retry-times "$(params.retries)"); then
-            echo "Target image exists."
-            echo "DEBUG: Source Digest - $SOURCE_DIGEST"
-            echo "DEBUG: Target Digest - $TARGET_DIGEST"
-            if [ "$SOURCE_DIGEST" == "$TARGET_DIGEST" ]; then
-                echo "Image already exists with the same digest, skipping copy." | tee "$(results.requestMessage.path)"
-                exit 0
-            else
-                echo "Image exists in target registry but digests do not match." \
-                    "Proceeding to copy the image."
-            fi
-        else
-            echo "Target image does not exist. Proceeding to copy the image."
-        fi
-
-        # Proceed with copying the image
-        echo "Copying image from $(params.sourceIndex) to $(params.targetIndex)"
-        (skopeo copy \
-        --all \
-        --preserve-digests \
-        --retry-times "$(params.retries)" \
-        --src-tls-verify=false "${SOURCE_AUTH_ARGS[@]}" \
-        "docker://$(params.sourceIndex)" \
-        "${TARGET_AUTH_ARGS[@]}" \
-        "docker://$(params.targetIndex)" && \
-        echo -n "Index Image Published successfully" || \
-        echo -n "Error: Failed publishing Index Image" ) | tee "$(results.requestMessage.path)"
+        python3 -m publish_index_image \
+          --source-index "$(params.sourceIndex)" \
+          --target-index "$(params.targetIndex)" \
+          --retries "$(params.retries)" \
+          --source-credential-path /mnt/publishingCredentials/sourceIndexCredential \
+          --target-credential-path /mnt/publishingCredentials/targetIndexCredential | \
+        awk '{printf "%s", (NR==1 ? "" : ORS) $0}' | \
+        tee "$(results.requestMessage.path)"

tasks/internal/publish-index-image-task/tests/mocks.sh

@@ -3,46 +3,59 @@ set -x
 
 # mocks to be injected into task step scripts
 
-function skopeo() {
-  echo Mock skopeo called with: $* >&2
+export _python3=$(which python3)
+fake_setup=$(mktemp)
+cat <<'EOF' > $fake_setup
+---
+inspect:
+  - match:
+      image: "docker://quay.io/match-target-digest"
+      format: "{{.Digest}}"  # optional
+    return: "sha256:match1234567890"  # string when format specified
+  - match:
+      image: "docker://quay.io/target"
+      format: "{{.Digest}}"  # optional
+    return: "sha256:target1234567890"  # string when format specified
+  - match:
+      image: "docker://registry-proxy.engineering.redhat.com/foo"
+      format: "{{.Digest}}"  # optional
+    return: "sha256:0987654321fedcba"  # string when format specified
+  - match:
+      image: "docker://registry-proxy.engineering.redhat.com/foo"
+      format: "{{.Digest}}"  # optional
+    return: "sha256:0987654321fedcba"  # string when format specified
+  - match:
+      image: "docker://registry-proxy.engineering.redhat.com/fail"
+    return:
+      error: "skopeo inspect failed"  # string when format specified
+      returncode: 1
+copy:
+  - match:
+      source: "docker://registry-proxy.engineering.redhat.com/match@sha256:match1234567890"
+      destination: "docker://quay.io/match-target-digest"
+    # omit return for success
+  - match:
+      source: "docker://registry-proxy.engineering.redhat.com/foo@sha256:0987654321fedcba"
+      destination: "docker://quay.io/target"
+    # omit return for success
+  - match:
+      source: "docker://quay.io/source@sha256:abcdef1234567890"
+      destination: "docker://quay.io/target"
+    # omit return for success
+  - match:
+      source: "docker://registry-proxy.engineering.redhat.com/fail@sha256:0987654321fedcba"
+      destination: "docker://quay.io/target"
+    return:
+      success: false
+      error: "skopeo copy failed"  # string when format specified
+      returncode: 1
+EOF
+export RELEASE_SERVICE_UTILS_FAKE_SKOPEO_SETUP=$fake_setup
 
-  if [[ "$1" == "inspect" ]]; then
-    # Handle `skopeo inspect`
-    if [[ "$*" == *"docker://quay.io/match-target-digest"* ]]; then
-      echo "sha256:match1234567890"  # Mock target digest for idempotency check
-      return 0
-    elif [[ "$*" == *"docker://quay.io/target"* ]]; then
-      echo "sha256:target1234567890"
-      return 0
-    elif [[ "$*" == *"--tls-verify=false --src-creds source docker://quay.io/source"* ]]; then
-      echo "sha256:abcdef1234567890"
-      return 0
-    elif [[ "$*" == *"--tls-verify=false docker://registry-proxy.engineering.redhat.com/foo"* ]]; then
-      echo "sha256:0987654321fedcba"
-      return 0
-    elif [[ "$*" == *"--tls-verify=false docker://registry-proxy.engineering.redhat.com/fail"* ]]; then
-      return 1
-    else
-      echo "Error: Unexpected inspect call"
-      exit 1
-    fi
-  elif [[ "$1" == "copy" ]]; then
-    # Handle `skopeo copy`
-    if [[ "$*" == *"--src-tls-verify=false --src-creds source docker://quay.io/source"* ]]; then
-      return 0
-    elif [[ "$*" == *"--src-tls-verify=false docker://registry-proxy.engineering.redhat.com/foo"* ]]; then
-      return 0
-    elif [[ "$*" == *"--src-tls-verify=false docker://registry-proxy.engineering.redhat.com/fail"* ]]; then
-      return 1
-    elif [[ "$*" == *"--src-tls-verify=false --src-creds source docker://quay.io/match-source-digest"* ]]; then
-      echo "Error: Copy should not be triggered when digests match"
-      exit 1
-    else
-      echo "Error: Unexpected copy call"
-      exit 1
-    fi
-  else
-    echo "Error: Unknown skopeo command"
-    exit 1
-  fi
+function python3() {
+  "$_python3" -c "
+from fake import patch_skopeo_client;
+patch_skopeo_client();
+from publish_index_image import main;
+main();" "${@:3}"
 }

tasks/managed/publish-index-image/publish-index-image.yaml

@@ -125,7 +125,8 @@ spec:
           value: $(params.sourceDataArtifact)
     - name: publish-index-image
       image: >-
-        quay.io/konflux-ci/release-service-utils@sha256:5546fa78d3c88d7b6a2e8cff8902f7757f00541d0bbaf113b9f293133894afa3
+        quay.io/jluza/release-service-utils:RELEASE-1989
+      imagePullPolicy: Always
       computeResources:
         limits:
           memory: 512Mi
@@ -142,71 +143,16 @@ spec:
             exit 1
         fi
 
-        request="publish-index-image-pipeline"
-        credentials=$(jq -r '.fbc.publishingCredentials' "$DATA_FILE")
-        pipelinerun_label="internal-services.appstudio.openshift.io/pipelinerun-uid"
+        credentials="$(jq -r '.fbc.publishingCredentials' "$DATA_FILE")"
 
-        LENGTH="$(jq -r '.components | length' "$(params.dataDir)/$(params.internalRequestResultsFile)")"
-        for((i=0; i<LENGTH; i++)); do
-          targetIndex=$(jq -r --argjson i "$i" \
-            '.components[$i].target_index' "$(params.dataDir)/$(params.internalRequestResultsFile)")
-
-          sourceIndex=$(jq -r  --argjson i "$i" \
-            '.components[$i].index_image' "$(params.dataDir)/$(params.internalRequestResultsFile)")
-
-          buildTimestamp=$(jq -r --argjson i "$i" '.components[$i].completion_time' \
-            "$(params.dataDir)/$(params.internalRequestResultsFile)")
-
-          publishingImages=("$targetIndex")
-          # only publish the extra timestamp-based tag if the targetIndex does not have it already
-          if [[ ! "$targetIndex" =~ .*"$buildTimestamp"$ ]]; then
-            publishingImages+=("${targetIndex}-${buildTimestamp}")
-          fi
-
-          requestTimeout=$(params.requestTimeout)
-          pipelineTimeoutSeconds=$((requestTimeout+300))
-          pipelineTimeout=$(date -u -d @"${pipelineTimeoutSeconds}" +"%Hh%Mm%Ss")
-          taskTimeout=$(date -u -d @"${requestTimeout}" +"%Hh%Mm%Ss")
-
-          IR_RESULT_FILE=$(mktemp)
-
-          for((x=0; x<${#publishingImages[@]}; x++ )); do
-              echo "=== Creating internal request to publish image:"
-              echo ""
-              echo "- from: ${sourceIndex}"
-              echo "- to: ${publishingImages[$x]}"
-
-              internal-request --pipeline "${request}" \
-                  -p sourceIndex="${sourceIndex}" \
-                  -p targetIndex="${publishingImages[$x]}" \
-                  -p publishingCredentials="${credentials}" \
-                  -p retries="$(params.retries)" \
-                  -p taskGitUrl="$(params.taskGitUrl)" \
-                  -p taskGitRevision="$(params.taskGitRevision)" \
-                  -t "$(params.requestTimeout)" \
-                  --pipeline-timeout "${pipelineTimeout}" \
-                  --task-timeout "$taskTimeout" \
-                  -l ${pipelinerun_label}="$(params.pipelineRunUid)" \
-                  | tee "$IR_RESULT_FILE" || \
-                  (grep "^\[" "$IR_RESULT_FILE" | jq . && exit 1)
-
-              internalRequest=$(awk -F"'" '/created/ { print $2 }' "$IR_RESULT_FILE")
-              echo "done (${internalRequest})"
-
-              results=$(kubectl get internalrequest "${internalRequest}" -o=jsonpath='{.status.results}')
-              requestMessage=$(echo "${results}" | jq -r '.requestMessage // ""')
-
-              if echo "${requestMessage}" | grep -qi "error"; then
-                echo "ERROR: Publish to ${publishingImages[$x]} failed"
-                echo "requestMessage: ${requestMessage}"
-                exit 1
-              fi
-              echo "=== published successfully (${publishingImages[$x]})"
-
-              echo ""
-              echo ""
-          done
-        done
+        python3 -m managed_publish_index_image \
+          --request-timeout "$(params.requestTimeout)" \
+          --publishing-credentials "${credentials}" \
+          --retries "$(params.retries)" \
+          --task-git-url "$(params.taskGitUrl)" \
+          --task-git-revision "$(params.taskGitRevision)" \
+          --pipeline-run-id "$(params.pipelineRunUid)" \
+          --ir-results-file "$(params.dataDir)/$(params.internalRequestResultsFile)"
     - name: create-trusted-artifact
       computeResources:
         limits:

tasks/managed/publish-index-image/tests/test-publish-index-image-with-timestamp.yaml

@@ -165,44 +165,65 @@ spec:
               #!/usr/bin/env bash
               set -eux
 
-              internalRequest="$(kubectl get internalrequest --sort-by=.metadata.creationTimestamp --no-headers | \
-                sed 's/[[:space:]]*$//' |head -1)"
-              pipeline="$(kubectl get internalrequest "${internalRequest}" -o \
-              jsonpath="{.spec.pipeline.pipelineRef.params[2].value}")"
+              internalRequests=$(kubectl get internalrequest --sort-by=.metadata.creationTimestamp --no-headers \
+              -o json | jq '.items')
+              irIndex=$(jq '. | length' <<< "${internalRequests}")
+              if [ "${irIndex}" -ne 2 ]; then
+                  echo "Too many internalrequests found. Should be 2"
+                  exit 1
+              fi
+              declare -a expected_target=()
+              expected_target+=("quay.io/scoheb/fbc-target-index-testing:v4.12")
+              expected_target+=("quay.io/scoheb/fbc-target-index-testing:v4.12-2023-03-06T16:39:11.314092Z")
+              for((i=0; i<irIndex; i++)); do
+                  pipeline=$(jq -r ".[$i] | .spec.pipeline.pipelineRef.params[2].value" <<< "${internalRequests}")
 
-              params=$(kubectl get internalrequest "${internalRequest}" -o jsonpath="{.spec.params}")
+                  params=$(jq -r ".[$i] | .spec.params" <<< "${internalRequests}")
 
-              if [ "$pipeline" != \
-                "pipelines/internal/publish-index-image-pipeline/publish-index-image-pipeline.yaml" ]; then
-                echo "pipeline does not match"
-                exit 1
-              fi
+                  if [ "$pipeline" != \
+                    "pipelines/internal/publish-index-image-pipeline/publish-index-image-pipeline.yaml" ]; then
+                    echo "pipeline does not match"
+                    exit 1
+                  fi
 
-              if [ "$(jq -r '.retries' <<< "${params}")" != "2" ]; then
-                echo "number of retries does not match"
-                exit 1
-              fi
+                  if [ "$(jq -r '.retries' <<< "${params}")" != "2" ]; then
+                    echo "number of retries does not match"
+                    exit 1
+                  fi
 
-              if [ "$(jq -r '.sourceIndex' <<< "${params}")" != "redhat.com/rh-stage/iib:01" ]; then
-                echo "sourceIndex image does not match"
-                exit 1
-              fi
+                  if [ "$(jq -r '.sourceIndex' <<< "${params}")" != "redhat.com/rh-stage/iib:01" ]; then
+                    echo "sourceIndex image does not match"
+                    exit 1
+                  fi
 
-              targetIndex=$(jq -r '.targetIndex' <<< "${params}")
-              if [ "$targetIndex" != "quay.io/scoheb/fbc-target-index-testing:v4.12" ]; then
-                echo "targetIndex image does not match"
-                exit 1
-              fi
+                  targetIndex=$(jq -r '.targetIndex' <<< "${params}")
+                  found=
+                  ti=0
+                  for expected in "${expected_target[@]}"; do
+                    if [ "$targetIndex" == "$expected" ]; then
+                      unset "expected_target[ti]"
+                      expected_target=("${expected_target[@]}")
+                      echo "${expected_target[@]}"
+                      found=1
+                      break
+                    fi
+                    ti=$((ti+1))
+                  done
+                  if [ -z "$found" ]; then
+                    echo "targetIndex ${targetIndex} image does not match any expected value"
+                    exit 1
+                  fi
 
-              if [ "$(jq -r '.taskGitUrl' <<< "${params}")" != "http://localhost" ]; then
-                echo "taskGitUrl image does not match"
-                exit 1
-              fi
+                  if [ "$(jq -r '.taskGitUrl' <<< "${params}")" != "http://localhost" ]; then
+                    echo "taskGitUrl image does not match"
+                    exit 1
+                  fi
 
-              if [ "$(jq -r '.taskGitRevision' <<< "${params}")" != "main" ]; then
-                echo "taskGitRevision image does not match"
-                exit 1
-              fi
+                  if [ "$(jq -r '.taskGitRevision' <<< "${params}")" != "main" ]; then
+                    echo "taskGitRevision image does not match"
+                    exit 1
+                  fi
+              done
   finally:
     - name: cleanup
       taskSpec: