Skip to content

Commit fb16139

Browse files
midnighterczqodo-code-review[bot]
midnightercz
and
qodo-code-review[bot]
committed
feat(RELEASE-1989): python for publish-index-images tasks
tasks/internal/publish-index-image and tasks/managed/publish-index-image internal bash scripts are replaced with python scripts from release-service-utils Signed-off-by: Jindrich Luza <jluza@redhat.com> Assisted-By: gemini Co-authored-by: qodo-code-review[bot] <151058649+qodo-code-review[bot]@users.noreply.github.com>
1 parent 79d015d commit fb16139

5 files changed

Lines changed: 144 additions & 198 deletions

File tree

‎tasks/internal/publish-index-image-task/publish-index-image-task.yaml‎

Lines changed: 10 additions & 50 deletions
Original file line numberDiff line numberDiff line change
@@ -64,7 +64,8 @@ spec:
6464
securityContext:
6565
runAsUser: 1001
6666
image: >-
67-
quay.io/konflux-ci/release-service-utils@sha256:5546fa78d3c88d7b6a2e8cff8902f7757f00541d0bbaf113b9f293133894afa3
67+
quay.io/jluza/release-service-utils:RELEASE-1989
68+
imagePullPolicy: Always
6869
computeResources:
6970
limits:
7071
memory: 64Mi
@@ -74,52 +75,11 @@ spec:
7475
script: |
7576
#!/usr/bin/env bash
7677
set -euo pipefail
77-
78-
SOURCE_INDEX_CREDENTIAL="$(cat /mnt/publishingCredentials/sourceIndexCredential)"
79-
TARGET_INDEX_CREDENTIAL="$(cat /mnt/publishingCredentials/targetIndexCredential)"
80-
81-
PATH=/bin:/usr/bin:/usr/local/bin
82-
export PATH
83-
84-
SOURCE_AUTH_ARGS=()
85-
if [[ ! "$(params.sourceIndex)" =~ ^registry-proxy(\-stage)?.engineering.redhat.com ]]; then
86-
SOURCE_AUTH_ARGS=("--src-creds" "${SOURCE_INDEX_CREDENTIAL}")
87-
fi
88-
89-
TARGET_AUTH_ARGS=("--dest-creds" "${TARGET_INDEX_CREDENTIAL}")
90-
91-
# Extract digest from pull spec
92-
SOURCE_INDEX="$(params.sourceIndex)"
93-
SOURCE_DIGEST="${SOURCE_INDEX##*@}"
94-
95-
echo "Getting target image digest: $(params.targetIndex)"
96-
if TARGET_DIGEST=$(skopeo inspect \
97-
"docker://$(params.targetIndex)" \
98-
--format '{{.Digest}}' \
99-
--retry-times "$(params.retries)"); then
100-
echo "Target image exists."
101-
echo "DEBUG: Source Digest - $SOURCE_DIGEST"
102-
echo "DEBUG: Target Digest - $TARGET_DIGEST"
103-
if [ "$SOURCE_DIGEST" == "$TARGET_DIGEST" ]; then
104-
echo "Image already exists with the same digest, skipping copy." | tee "$(results.requestMessage.path)"
105-
exit 0
106-
else
107-
echo "Image exists in target registry but digests do not match." \
108-
"Proceeding to copy the image."
109-
fi
110-
else
111-
echo "Target image does not exist. Proceeding to copy the image."
112-
fi
113-
114-
# Proceed with copying the image
115-
echo "Copying image from $(params.sourceIndex) to $(params.targetIndex)"
116-
(skopeo copy \
117-
--all \
118-
--preserve-digests \
119-
--retry-times "$(params.retries)" \
120-
--src-tls-verify=false "${SOURCE_AUTH_ARGS[@]}" \
121-
"docker://$(params.sourceIndex)" \
122-
"${TARGET_AUTH_ARGS[@]}" \
123-
"docker://$(params.targetIndex)" && \
124-
echo -n "Index Image Published successfully" || \
125-
echo -n "Error: Failed publishing Index Image" ) | tee "$(results.requestMessage.path)"
78+
python3 -m publish_index_image \
79+
--source-index "$(params.sourceIndex)" \
80+
--target-index "$(params.targetIndex)" \
81+
--retries "$(params.retries)" \
82+
--source-credential-path /mnt/publishingCredentials/sourceIndexCredential \
83+
--target-credential-path /mnt/publishingCredentials/targetIndexCredential | \
84+
awk '{printf "%s", (NR==1 ? "" : ORS) $0}' | \
85+
tee "$(results.requestMessage.path)"

‎tasks/internal/publish-index-image-task/tests/mocks.sh‎

Lines changed: 54 additions & 41 deletions
Original file line numberDiff line numberDiff line change
@@ -3,46 +3,59 @@ set -x
33

44
# mocks to be injected into task step scripts
55

6-
function skopeo() {
7-
echo Mock skopeo called with: $* >&2
6+
export _python3=$(which python3)
7+
fake_setup=$(mktemp)
8+
cat <<'EOF' > $fake_setup
9+
---
10+
inspect:
11+
- match:
12+
image: "docker://quay.io/match-target-digest"
13+
format: "{{.Digest}}" # optional
14+
return: "sha256:match1234567890" # string when format specified
15+
- match:
16+
image: "docker://quay.io/target"
17+
format: "{{.Digest}}" # optional
18+
return: "sha256:target1234567890" # string when format specified
19+
- match:
20+
image: "docker://registry-proxy.engineering.redhat.com/foo"
21+
format: "{{.Digest}}" # optional
22+
return: "sha256:0987654321fedcba" # string when format specified
23+
- match:
24+
image: "docker://registry-proxy.engineering.redhat.com/foo"
25+
format: "{{.Digest}}" # optional
26+
return: "sha256:0987654321fedcba" # string when format specified
27+
- match:
28+
image: "docker://registry-proxy.engineering.redhat.com/fail"
29+
return:
30+
error: "skopeo inspect failed" # string when format specified
31+
returncode: 1
32+
copy:
33+
- match:
34+
source: "docker://registry-proxy.engineering.redhat.com/match@sha256:match1234567890"
35+
destination: "docker://quay.io/match-target-digest"
36+
# omit return for success
37+
- match:
38+
source: "docker://registry-proxy.engineering.redhat.com/foo@sha256:0987654321fedcba"
39+
destination: "docker://quay.io/target"
40+
# omit return for success
41+
- match:
42+
source: "docker://quay.io/source@sha256:abcdef1234567890"
43+
destination: "docker://quay.io/target"
44+
# omit return for success
45+
- match:
46+
source: "docker://registry-proxy.engineering.redhat.com/fail@sha256:0987654321fedcba"
47+
destination: "docker://quay.io/target"
48+
return:
49+
success: false
50+
error: "skopeo copy failed" # string when format specified
51+
returncode: 1
52+
EOF
53+
export RELEASE_SERVICE_UTILS_FAKE_SKOPEO_SETUP=$fake_setup
854

9-
if [[ "$1" == "inspect" ]]; then
10-
# Handle `skopeo inspect`
11-
if [[ "$*" == *"docker://quay.io/match-target-digest"* ]]; then
12-
echo "sha256:match1234567890" # Mock target digest for idempotency check
13-
return 0
14-
elif [[ "$*" == *"docker://quay.io/target"* ]]; then
15-
echo "sha256:target1234567890"
16-
return 0
17-
elif [[ "$*" == *"--tls-verify=false --src-creds source docker://quay.io/source"* ]]; then
18-
echo "sha256:abcdef1234567890"
19-
return 0
20-
elif [[ "$*" == *"--tls-verify=false docker://registry-proxy.engineering.redhat.com/foo"* ]]; then
21-
echo "sha256:0987654321fedcba"
22-
return 0
23-
elif [[ "$*" == *"--tls-verify=false docker://registry-proxy.engineering.redhat.com/fail"* ]]; then
24-
return 1
25-
else
26-
echo "Error: Unexpected inspect call"
27-
exit 1
28-
fi
29-
elif [[ "$1" == "copy" ]]; then
30-
# Handle `skopeo copy`
31-
if [[ "$*" == *"--src-tls-verify=false --src-creds source docker://quay.io/source"* ]]; then
32-
return 0
33-
elif [[ "$*" == *"--src-tls-verify=false docker://registry-proxy.engineering.redhat.com/foo"* ]]; then
34-
return 0
35-
elif [[ "$*" == *"--src-tls-verify=false docker://registry-proxy.engineering.redhat.com/fail"* ]]; then
36-
return 1
37-
elif [[ "$*" == *"--src-tls-verify=false --src-creds source docker://quay.io/match-source-digest"* ]]; then
38-
echo "Error: Copy should not be triggered when digests match"
39-
exit 1
40-
else
41-
echo "Error: Unexpected copy call"
42-
exit 1
43-
fi
44-
else
45-
echo "Error: Unknown skopeo command"
46-
exit 1
47-
fi
55+
function python3() {
56+
"$_python3" -c "
57+
from fake import patch_skopeo_client;
58+
patch_skopeo_client();
59+
from publish_index_image import main;
60+
main();" "${@:3}"
4861
}

‎tasks/managed/publish-index-image/publish-index-image.yaml‎

Lines changed: 11 additions & 65 deletions
Original file line numberDiff line numberDiff line change
@@ -125,7 +125,8 @@ spec:
125125
value: $(params.sourceDataArtifact)
126126
- name: publish-index-image
127127
image: >-
128-
quay.io/konflux-ci/release-service-utils@sha256:5546fa78d3c88d7b6a2e8cff8902f7757f00541d0bbaf113b9f293133894afa3
128+
quay.io/jluza/release-service-utils:RELEASE-1989
129+
imagePullPolicy: Always
129130
computeResources:
130131
limits:
131132
memory: 512Mi
@@ -142,71 +143,16 @@ spec:
142143
exit 1
143144
fi
144145
145-
request="publish-index-image-pipeline"
146-
credentials=$(jq -r '.fbc.publishingCredentials' "$DATA_FILE")
147-
pipelinerun_label="internal-services.appstudio.openshift.io/pipelinerun-uid"
146+
credentials="$(jq -r '.fbc.publishingCredentials' "$DATA_FILE")"
148147
149-
LENGTH="$(jq -r '.components | length' "$(params.dataDir)/$(params.internalRequestResultsFile)")"
150-
for((i=0; i<LENGTH; i++)); do
151-
targetIndex=$(jq -r --argjson i "$i" \
152-
'.components[$i].target_index' "$(params.dataDir)/$(params.internalRequestResultsFile)")
153-
154-
sourceIndex=$(jq -r --argjson i "$i" \
155-
'.components[$i].index_image' "$(params.dataDir)/$(params.internalRequestResultsFile)")
156-
157-
buildTimestamp=$(jq -r --argjson i "$i" '.components[$i].completion_time' \
158-
"$(params.dataDir)/$(params.internalRequestResultsFile)")
159-
160-
publishingImages=("$targetIndex")
161-
# only publish the extra timestamp-based tag if the targetIndex does not have it already
162-
if [[ ! "$targetIndex" =~ .*"$buildTimestamp"$ ]]; then
163-
publishingImages+=("${targetIndex}-${buildTimestamp}")
164-
fi
165-
166-
requestTimeout=$(params.requestTimeout)
167-
pipelineTimeoutSeconds=$((requestTimeout+300))
168-
pipelineTimeout=$(date -u -d @"${pipelineTimeoutSeconds}" +"%Hh%Mm%Ss")
169-
taskTimeout=$(date -u -d @"${requestTimeout}" +"%Hh%Mm%Ss")
170-
171-
IR_RESULT_FILE=$(mktemp)
172-
173-
for((x=0; x<${#publishingImages[@]}; x++ )); do
174-
echo "=== Creating internal request to publish image:"
175-
echo ""
176-
echo "- from: ${sourceIndex}"
177-
echo "- to: ${publishingImages[$x]}"
178-
179-
internal-request --pipeline "${request}" \
180-
-p sourceIndex="${sourceIndex}" \
181-
-p targetIndex="${publishingImages[$x]}" \
182-
-p publishingCredentials="${credentials}" \
183-
-p retries="$(params.retries)" \
184-
-p taskGitUrl="$(params.taskGitUrl)" \
185-
-p taskGitRevision="$(params.taskGitRevision)" \
186-
-t "$(params.requestTimeout)" \
187-
--pipeline-timeout "${pipelineTimeout}" \
188-
--task-timeout "$taskTimeout" \
189-
-l ${pipelinerun_label}="$(params.pipelineRunUid)" \
190-
| tee "$IR_RESULT_FILE" || \
191-
(grep "^\[" "$IR_RESULT_FILE" | jq . && exit 1)
192-
193-
internalRequest=$(awk -F"'" '/created/ { print $2 }' "$IR_RESULT_FILE")
194-
echo "done (${internalRequest})"
195-
196-
results=$(kubectl get internalrequest "${internalRequest}" -o=jsonpath='{.status.results}')
197-
requestMessage=$(echo "${results}" | jq -r '.requestMessage // ""')
198-
199-
if echo "${requestMessage}" | grep -qi "error"; then
200-
echo "ERROR: Publish to ${publishingImages[$x]} failed"
201-
echo "requestMessage: ${requestMessage}"
202-
exit 1
203-
fi
204-
echo "=== published successfully (${publishingImages[$x]})"
205-
206-
echo ""
207-
echo ""
208-
done
209-
done
148+
python3 -m managed_publish_index_image \
149+
--request-timeout "$(params.requestTimeout)" \
150+
--publishing-credentials "${credentials}" \
151+
--retries "$(params.retries)" \
152+
--task-git-url "$(params.taskGitUrl)" \
153+
--task-git-revision "$(params.taskGitRevision)" \
154+
--pipeline-run-id "$(params.pipelineRunUid)" \
155+
--ir-results-file "$(params.dataDir)/$(params.internalRequestResultsFile)"
210156
- name: create-trusted-artifact
211157
computeResources:
212158
limits:

‎tasks/managed/publish-index-image/tests/test-publish-index-image-with-timestamp.yaml‎

Lines changed: 52 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -165,44 +165,65 @@ spec:
165165
#!/usr/bin/env bash
166166
set -eux
167167
168-
internalRequest="$(kubectl get internalrequest --sort-by=.metadata.creationTimestamp --no-headers | \
169-
sed 's/[[:space:]]*$//' |head -1)"
170-
pipeline="$(kubectl get internalrequest "${internalRequest}" -o \
171-
jsonpath="{.spec.pipeline.pipelineRef.params[2].value}")"
168+
internalRequests=$(kubectl get internalrequest --sort-by=.metadata.creationTimestamp --no-headers \
169+
-o json | jq '.items')
170+
irIndex=$(jq '. | length' <<< "${internalRequests}")
171+
if [ "${irIndex}" -ne 2 ]; then
172+
echo "Too many internalrequests found. Should be 2"
173+
exit 1
174+
fi
175+
declare -a expected_target=()
176+
expected_target+=("quay.io/scoheb/fbc-target-index-testing:v4.12")
177+
expected_target+=("quay.io/scoheb/fbc-target-index-testing:v4.12-2023-03-06T16:39:11.314092Z")
178+
for((i=0; i<irIndex; i++)); do
179+
pipeline=$(jq -r ".[$i] | .spec.pipeline.pipelineRef.params[2].value" <<< "${internalRequests}")
172180
173-
params=$(kubectl get internalrequest "${internalRequest}" -o jsonpath="{.spec.params}")
181+
params=$(jq -r ".[$i] | .spec.params" <<< "${internalRequests}")
174182
175-
if [ "$pipeline" != \
176-
"pipelines/internal/publish-index-image-pipeline/publish-index-image-pipeline.yaml" ]; then
177-
echo "pipeline does not match"
178-
exit 1
179-
fi
183+
if [ "$pipeline" != \
184+
"pipelines/internal/publish-index-image-pipeline/publish-index-image-pipeline.yaml" ]; then
185+
echo "pipeline does not match"
186+
exit 1
187+
fi
180188
181-
if [ "$(jq -r '.retries' <<< "${params}")" != "2" ]; then
182-
echo "number of retries does not match"
183-
exit 1
184-
fi
189+
if [ "$(jq -r '.retries' <<< "${params}")" != "2" ]; then
190+
echo "number of retries does not match"
191+
exit 1
192+
fi
185193
186-
if [ "$(jq -r '.sourceIndex' <<< "${params}")" != "redhat.com/rh-stage/iib:01" ]; then
187-
echo "sourceIndex image does not match"
188-
exit 1
189-
fi
194+
if [ "$(jq -r '.sourceIndex' <<< "${params}")" != "redhat.com/rh-stage/iib:01" ]; then
195+
echo "sourceIndex image does not match"
196+
exit 1
197+
fi
190198
191-
targetIndex=$(jq -r '.targetIndex' <<< "${params}")
192-
if [ "$targetIndex" != "quay.io/scoheb/fbc-target-index-testing:v4.12" ]; then
193-
echo "targetIndex image does not match"
194-
exit 1
195-
fi
199+
targetIndex=$(jq -r '.targetIndex' <<< "${params}")
200+
found=
201+
ti=0
202+
for expected in "${expected_target[@]}"; do
203+
if [ "$targetIndex" == "$expected" ]; then
204+
unset "expected_target[ti]"
205+
expected_target=("${expected_target[@]}")
206+
echo "${expected_target[@]}"
207+
found=1
208+
break
209+
fi
210+
ti=$((ti+1))
211+
done
212+
if [ -z "$found" ]; then
213+
echo "targetIndex ${targetIndex} image does not match any expected value"
214+
exit 1
215+
fi
196216
197-
if [ "$(jq -r '.taskGitUrl' <<< "${params}")" != "http://localhost" ]; then
198-
echo "taskGitUrl image does not match"
199-
exit 1
200-
fi
217+
if [ "$(jq -r '.taskGitUrl' <<< "${params}")" != "http://localhost" ]; then
218+
echo "taskGitUrl image does not match"
219+
exit 1
220+
fi
201221
202-
if [ "$(jq -r '.taskGitRevision' <<< "${params}")" != "main" ]; then
203-
echo "taskGitRevision image does not match"
204-
exit 1
205-
fi
222+
if [ "$(jq -r '.taskGitRevision' <<< "${params}")" != "main" ]; then
223+
echo "taskGitRevision image does not match"
224+
exit 1
225+
fi
226+
done
206227
finally:
207228
- name: cleanup
208229
taskSpec:

0 commit comments

Comments
 (0)