diff --git a/pipelines/internal/filter-already-released-advisory-rpms/README.md b/pipelines/internal/filter-already-released-advisory-rpms/README.md index 766f87470c..0244192b2b 100644 --- a/pipelines/internal/filter-already-released-advisory-rpms/README.md +++ b/pipelines/internal/filter-already-released-advisory-rpms/README.md @@ -11,5 +11,7 @@ It returns lists of unreleased RPMs and RPMs found in advisories for digest vali | origin | The origin workspace where the release CR comes from | No | - | | advisory_secret_name | The name of the secret that contains the advisory GitLab metadata | No | - | | internalRequestPipelineRunName | Name of the PipelineRun that requested this pipeline | No | - | +| ociStorage | The OCI repository to store results artifact | No | - | | taskGitUrl | The url to the git repo where the release-service-catalog tasks to be used are stored | Yes | https://github.com/konflux-ci/release-service-catalog.git | | taskGitRevision | The revision in the taskGitUrl repo to be used | No | - | +| orasOptions | oras options to pass to oras calls | Yes | "" | diff --git a/pipelines/internal/filter-already-released-advisory-rpms/filter-already-released-advisory-rpms.yaml b/pipelines/internal/filter-already-released-advisory-rpms/filter-already-released-advisory-rpms.yaml index fd82697e7f..bf32f55af9 100644 --- a/pipelines/internal/filter-already-released-advisory-rpms/filter-already-released-advisory-rpms.yaml +++ b/pipelines/internal/filter-already-released-advisory-rpms/filter-already-released-advisory-rpms.yaml @@ -23,6 +23,9 @@ spec: - name: internalRequestPipelineRunName type: string description: Name of the PipelineRun that requested this pipeline + - name: ociStorage + type: string + description: The OCI repository to store results artifact - name: taskGitUrl type: string description: The url to the git repo where the release-service-catalog tasks to be used are stored @@ -30,6 +33,10 @@ spec: - name: taskGitRevision type: string description: The revision in the taskGitUrl repo to be used + - name: orasOptions + type: string + description: oras options to pass to oras calls + default: "" tasks: - name: filter-already-released-advisory-rpms-task taskRef: @@ -50,13 +57,15 @@ spec: value: $(params.advisory_secret_name) - name: internalRequestPipelineRunName value: $(params.internalRequestPipelineRunName) + - name: ociStorage + value: $(params.ociStorage) + - name: orasOptions + value: $(params.orasOptions) results: - name: result value: $(tasks.filter-already-released-advisory-rpms-task.results.result) - - name: unreleased_rpms - value: $(tasks.filter-already-released-advisory-rpms-task.results.unreleased_rpms) - - name: in_advisory_rpms - value: $(tasks.filter-already-released-advisory-rpms-task.results.in_advisory_rpms) + - name: filter_results_artifact + value: $(tasks.filter-already-released-advisory-rpms-task.results.filter_results_artifact) - name: advisory_url value: $(tasks.filter-already-released-advisory-rpms-task.results.advisory_url) - name: advisory_internal_url diff --git a/scripts/run-local-tests.sh b/scripts/run-local-tests.sh index 2adacdef1d..0186a76d64 100755 --- a/scripts/run-local-tests.sh +++ b/scripts/run-local-tests.sh @@ -277,7 +277,7 @@ classify_tasks() { continue fi - # Check if task supports Trusted Artifacts (uses TA step actions) + # Check if task needs OCI registry (uses TA step actions) if grep -q "name: use-trusted-artifact\|name: create-trusted-artifact" "$task_file"; then trusted_artifacts_tasks+=("$item") else diff --git a/tasks/internal/filter-already-released-advisory-rpms-task/README.md b/tasks/internal/filter-already-released-advisory-rpms-task/README.md index a64c087327..7f242e3ddf 100644 --- a/tasks/internal/filter-already-released-advisory-rpms-task/README.md +++ b/tasks/internal/filter-already-released-advisory-rpms-task/README.md @@ -6,11 +6,14 @@ RPMs found in advisories (for digest validation by the calling task). ## Parameters -| Name | Description | Optional | Default value | -|--------------------------------|-----------------------------------------------------------------------|----------|---------------| -| transformedSnapshot | Base64 string of gzipped JSON array of RPM entries with purls | No | - | -| origin | The origin workspace for the release CR | No | - | -| advisory_secret_name | Name of the secret containing advisory GitLab metadata | No | - | -| internalRequestPipelineRunName | Name of the PipelineRun that requested this task | No | - | -| caTrustConfigMapName | The name of the ConfigMap to read CA bundle data from | Yes | trusted-ca | -| caTrustConfigMapKey | The name of the key in the ConfigMap that contains the CA bundle data | Yes | ca-bundle.crt | +| Name | Description | Optional | Default value | +|-------------------------------------------------|--------------------------------------------------------------------------------------------|----------|-----------------------------------------------------| +| transformedSnapshot | Base64 string of gzipped JSON array of RPM entries with purls | No | - | +| origin | The origin workspace for the release CR | No | - | +| advisory_secret_name | Name of the secret containing advisory GitLab metadata | No | - | +| internalRequestPipelineRunName | Name of the PipelineRun that requested this task | No | - | +| ociStorage | The OCI repository to store results artifact | No | - | +| trusted_artifacts_dockerconfig_json_secret_name | The name of the secret that contains the dockerconfig json for trusted artifact operations | Yes | quay-token-konflux-release-trusted-artifacts-secret | +| orasOptions | oras options to pass to oras calls | Yes | "" | +| caTrustConfigMapName | The name of the ConfigMap to read CA bundle data from | Yes | trusted-ca | +| caTrustConfigMapKey | The name of the key in the ConfigMap that contains the CA bundle data | Yes | ca-bundle.crt | diff --git a/tasks/internal/filter-already-released-advisory-rpms-task/filter-already-released-advisory-rpms-task.yaml b/tasks/internal/filter-already-released-advisory-rpms-task/filter-already-released-advisory-rpms-task.yaml index 2d886327dc..a03f8b9af0 100644 --- a/tasks/internal/filter-already-released-advisory-rpms-task/filter-already-released-advisory-rpms-task.yaml +++ b/tasks/internal/filter-already-released-advisory-rpms-task/filter-already-released-advisory-rpms-task.yaml @@ -24,6 +24,17 @@ spec: - name: internalRequestPipelineRunName description: Name of the PipelineRun that requested this task type: string + - name: ociStorage + description: The OCI repository to store results artifact + type: string + - name: trusted_artifacts_dockerconfig_json_secret_name + type: string + description: The name of the secret that contains the dockerconfig json for trusted artifact operations + default: quay-token-konflux-release-trusted-artifacts-secret + - name: orasOptions + type: string + description: oras options to pass to oras calls + default: "" - name: caTrustConfigMapName type: string description: The name of the ConfigMap to read CA bundle data from @@ -39,6 +50,8 @@ spec: description: The name of the InternalRequest PipelineRun - name: internalRequestTaskRunName description: The name of the InternalRequest TaskRun + - name: filter_results_artifact + description: OCI artifact reference containing filter results (unreleased_rpms, in_advisory_rpms JSON files) - name: unreleased_rpms description: RPMs not found in any advisory (gzipped base64 JSON array) - name: in_advisory_rpms @@ -52,6 +65,11 @@ spec: secret: secretName: $(params.advisory_secret_name) defaultMode: 0444 + - name: trusted-artifacts-dockerconfig-secret + secret: + optional: true + secretName: $(params.trusted_artifacts_dockerconfig_json_secret_name) + defaultMode: 0444 - name: trusted-ca configMap: name: $(params.caTrustConfigMapName) @@ -79,13 +97,25 @@ spec: volumeMounts: - name: advisory-secret mountPath: /mnt/advisory_secret + - name: trusted-artifacts-dockerconfig-secret + mountPath: /mnt/trusted_artifacts_dockerconfig env: - name: TRANSFORMED_SNAPSHOT value: "$(params.transformedSnapshot)" + - name: ORAS_OPTIONS + value: "$(params.orasOptions)" script: | #!/usr/bin/env bash set -eo pipefail + # Setup OCI registry credentials for trusted artifacts + # AppSRE clusters do not enable credentials-init in Tekton + TA_DOCKERCONFIG_JSON_PATH="/mnt/trusted_artifacts_dockerconfig/.dockerconfigjson" + if [ -f "${TA_DOCKERCONFIG_JSON_PATH}" ] && [ -s "${TA_DOCKERCONFIG_JSON_PATH}" ]; then + mkdir -p ~/.docker/ + cat "${TA_DOCKERCONFIG_JSON_PATH}" > ~/.docker/config.json + fi + GITLAB_HOST="$(cat /mnt/advisory_secret/gitlab_host)" ACCESS_TOKEN="$(cat /mnt/advisory_secret/gitlab_access_token)" GIT_AUTHOR_NAME="$(cat /mnt/advisory_secret/git_author_name)" @@ -116,6 +146,31 @@ spec: } trap 'exitfunc $? $LINENO "$BASH_COMMAND"' EXIT + push_results_to_oci() { + local results_dir="$1" + local oci_storage + oci_storage="$(params.ociStorage)" + local oci_tag + oci_tag="filter-results-$(date +%s)" + local oci_ref="${oci_storage}:${oci_tag}" + + echo "Pushing filter results to OCI storage: ${oci_ref}" >&2 + local oras_opts="--annotation=quay.expires-after=1d" + # Intentionally unquoted: empty quoted string breaks oras positional args + # shellcheck disable=SC2086 + (cd "${results_dir}" && oras push ${ORAS_OPTIONS:-} ${oras_opts} \ + --registry-config <(select-oci-auth "${oci_storage}") \ + "${oci_ref}" "filter-results.tar.gz:application/gzip") >&2 + + local digest + # Intentionally unquoted: empty quoted string breaks oras positional args + # shellcheck disable=SC2086 + digest=$(oras manifest fetch ${ORAS_OPTIONS:-} "${oci_ref}" \ + --registry-config <(select-oci-auth "${oci_storage}") \ + --descriptor | jq -r '.digest') + echo "oci:${oci_storage}@${digest}" + } + # === Phase 1: Decode transformed snapshot === TRANSFORMED_SNAPSHOT_JSON=$(base64 -d <<< "$TRANSFORMED_SNAPSHOT" | gunzip) echo "Transformed Snapshot JSON: $TRANSFORMED_SNAPSHOT_JSON" @@ -144,44 +199,60 @@ spec: if [[ -z "$EXISTING_ADVISORIES" ]]; then echo "No existing advisories found. All RPMs are unreleased." + + # Create output directory for OCI artifact + RESULTS_DIR=/tmp/filter-results + mkdir -p "$RESULTS_DIR" + # All RPMs are unreleased - UNRELEASED=$(echo "$TRANSFORMED_SNAPSHOT_JSON" | gzip -c | base64 -w 0) - IN_ADVISORY=$(echo "[]" | gzip -c | base64 -w 0) - echo -n "$UNRELEASED" > "$(results.unreleased_rpms.path)" - echo -n "$IN_ADVISORY" > "$(results.in_advisory_rpms.path)" + echo "$TRANSFORMED_SNAPSHOT_JSON" | jq -c '.' > "$RESULTS_DIR/unreleased_rpms.json" + echo "[]" > "$RESULTS_DIR/in_advisory_rpms.json" + + # Create tarball + RESULTS_TAR="$RESULTS_DIR/filter-results.tar.gz" + tar -czf "$RESULTS_TAR" -C "$RESULTS_DIR" unreleased_rpms.json in_advisory_rpms.json + + ARTIFACT_REF=$(push_results_to_oci "$RESULTS_DIR") + echo "Filter results artifact: ${ARTIFACT_REF}" + echo -n "${ARTIFACT_REF}" > "$(results.filter_results_artifact.path)" + jq -c '.' "$RESULTS_DIR/unreleased_rpms.json" | gzip -c | base64 -w 0 \ + > "$(results.unreleased_rpms.path)" + echo -n "[]" | gzip -c | base64 -w 0 > "$(results.in_advisory_rpms.path)" echo -n "" > "$(results.advisory_url.path)" echo -n "" > "$(results.advisory_internal_url.path)" echo -n "Success" > "$(results.result.path)" exit 0 fi - # === Phase 4: Extract all purls from all advisories (batch) === + # === Phase 4: Extract all purls from all advisories === echo "Extracting purls from advisories..." - ALL_ADVISORY_PURLS_FILE=/tmp/all_advisory_purls.json - echo "[]" > "$ALL_ADVISORY_PURLS_FILE" + PURL_MAPS_FILE=/tmp/purl_advisory_maps.jsonl + : > "$PURL_MAPS_FILE" - LATEST_ADVISORY_FILE="" for ADVISORY_SUBDIR in $EXISTING_ADVISORIES; do ADVISORY_FILE="${ADVISORY_BASE_DIR}/${ADVISORY_SUBDIR}/advisory.yaml" if [[ -f "$ADVISORY_FILE" ]]; then - ADVISORY_PURLS=$(yq -o=json \ + yq -o=json \ '.spec.content.artifacts // [] | [.[].purl] | map(select(. != null))' \ - "$ADVISORY_FILE") - MERGED=$(jq --argjson new "$ADVISORY_PURLS" \ - '. + $new | unique' "$ALL_ADVISORY_PURLS_FILE") - echo "$MERGED" > "$ALL_ADVISORY_PURLS_FILE" - - if [[ -z "$LATEST_ADVISORY_FILE" ]]; then - if [[ "$(jq 'length' <<< "$ADVISORY_PURLS")" -gt 0 ]]; then - LATEST_ADVISORY_FILE="$ADVISORY_FILE" - fi - fi + "$ADVISORY_FILE" \ + | jq -c --arg adv "$ADVISORY_FILE" \ + 'reduce .[] as $p ({}; .[$p] = $adv)' \ + >> "$PURL_MAPS_FILE" fi done - echo "Total unique purls from advisories: $(jq 'length' "$ALL_ADVISORY_PURLS_FILE")" + PURL_TO_ADVISORY_MAP_FILE=/tmp/purl_to_advisory_map.json + if [[ -s "$PURL_MAPS_FILE" ]]; then + jq -s 'reduce .[] as $m ({}; . * $m)' \ + "$PURL_MAPS_FILE" > "$PURL_TO_ADVISORY_MAP_FILE" + else + echo "{}" > "$PURL_TO_ADVISORY_MAP_FILE" + fi + rm -f "$PURL_MAPS_FILE" + + echo "Total unique purls from advisories: $(jq 'keys | length' "$PURL_TO_ADVISORY_MAP_FILE")" - # === Phase 5: Categorize RPMs === + # === Phase 5: Categorize RPMs and track source advisories === echo "Categorizing RPMs by advisory presence..." SNAPSHOT_FILE=/tmp/snapshot.json @@ -189,49 +260,64 @@ spec: UNRELEASED_FILE=/tmp/unreleased.json IN_ADVISORY_FILE=/tmp/in_advisory.json - echo "[]" > "$UNRELEASED_FILE" - echo "[]" > "$IN_ADVISORY_FILE" - - while IFS= read -r entry; do - entry_file=/tmp/current_entry.json - echo "$entry" > "$entry_file" + ADVISORY_SET_FILE=/tmp/advisory_set.json - purl=$(jq -r '.purl' "$entry_file") + jq --slurpfile map "$PURL_TO_ADVISORY_MAP_FILE" ' + . as $entries | $map[0] as $m | + reduce $entries[] as $e ( + {unreleased: [], in_advisory: [], advisories: {}}; + if $m[$e.purl] then + .in_advisory += [$e] + | .advisories[$m[$e.purl]] = true + else + .unreleased += [$e] + end + ) | { + unreleased, + in_advisory, + advisories: (.advisories | keys) + } + ' "$SNAPSHOT_FILE" > /tmp/categorized.json - IN_ADVISORY=$(jq --arg purl "$purl" \ - 'map(select(. == $purl)) | length > 0' "$ALL_ADVISORY_PURLS_FILE") - - if [[ "$IN_ADVISORY" == "true" ]]; then - # RPM found in advisory - add to in_advisory list for digest validation - UPDATED=$(jq --slurpfile e "$entry_file" '. + $e' "$IN_ADVISORY_FILE") - echo "$UPDATED" > "$IN_ADVISORY_FILE" - else - # RPM not in any advisory - unreleased - UPDATED=$(jq --slurpfile e "$entry_file" '. + $e' "$UNRELEASED_FILE") - echo "$UPDATED" > "$UNRELEASED_FILE" - fi - - rm -f "$entry_file" - done < <(jq -c '.[]' "$SNAPSHOT_FILE") + jq '.unreleased' /tmp/categorized.json > "$UNRELEASED_FILE" + jq '.in_advisory' /tmp/categorized.json > "$IN_ADVISORY_FILE" + jq '.advisories' /tmp/categorized.json > "$ADVISORY_SET_FILE" + rm -f /tmp/categorized.json # === Phase 6: Output results === UNRELEASED_COUNT=$(jq 'length' "$UNRELEASED_FILE") IN_ADVISORY_COUNT=$(jq 'length' "$IN_ADVISORY_FILE") + ADVISORY_SET_COUNT=$(jq 'length' "$ADVISORY_SET_FILE") + echo "Unreleased RPMs: $UNRELEASED_COUNT" echo "In-advisory RPMs (need digest validation): $IN_ADVISORY_COUNT" + echo "Unique advisories containing in-advisory RPMs: $ADVISORY_SET_COUNT" + + # Create output directory for OCI artifact + RESULTS_DIR=/tmp/filter-results + mkdir -p "$RESULTS_DIR" - # Encode results - UNRELEASED_ENCODED=$(jq -c '.' "$UNRELEASED_FILE" | gzip -c | base64 -w 0) - IN_ADVISORY_ENCODED=$(jq -c '.' "$IN_ADVISORY_FILE" | gzip -c | base64 -w 0) + # Write JSON files (compact format to save space) + jq -c '.' "$UNRELEASED_FILE" > "$RESULTS_DIR/unreleased_rpms.json" + jq -c '.' "$IN_ADVISORY_FILE" > "$RESULTS_DIR/in_advisory_rpms.json" - echo -n "$UNRELEASED_ENCODED" > "$(results.unreleased_rpms.path)" - echo -n "$IN_ADVISORY_ENCODED" > "$(results.in_advisory_rpms.path)" + # Create tarball + RESULTS_TAR="$RESULTS_DIR/filter-results.tar.gz" + tar -czf "$RESULTS_TAR" -C "$RESULTS_DIR" unreleased_rpms.json in_advisory_rpms.json - # Set advisory URL if all RPMs were in advisories - if [[ "$UNRELEASED_COUNT" -eq 0 ]] && [[ -n "$LATEST_ADVISORY_FILE" ]]; then - echo "All RPMs found in advisories." - ADVISORY_TYPE=$(yq -r '.spec.type' "$LATEST_ADVISORY_FILE") - ADVISORY_NAME=$(yq -r '.metadata.name' "$LATEST_ADVISORY_FILE") + ARTIFACT_REF=$(push_results_to_oci "$RESULTS_DIR") + echo "Filter results artifact: ${ARTIFACT_REF}" + echo -n "${ARTIFACT_REF}" > "$(results.filter_results_artifact.path)" + jq -c '.' "$UNRELEASED_FILE" | gzip -c | base64 -w 0 > "$(results.unreleased_rpms.path)" + jq -c '.' "$IN_ADVISORY_FILE" | gzip -c | base64 -w 0 > "$(results.in_advisory_rpms.path)" + + # Set advisory URL only if ALL RPMs are in advisories AND they all came from the SAME SINGLE advisory + if [[ "$UNRELEASED_COUNT" -eq 0 ]] && [[ "$ADVISORY_SET_COUNT" -eq 1 ]]; then + SINGLE_ADVISORY_FILE=$(jq -r '.[0]' "$ADVISORY_SET_FILE") + echo "All RPMs found in the same single advisory: ${SINGLE_ADVISORY_FILE}" + + ADVISORY_TYPE=$(yq -r '.spec.type' "$SINGLE_ADVISORY_FILE") + ADVISORY_NAME=$(yq -r '.metadata.name' "$SINGLE_ADVISORY_FILE") if [[ "${GIT_REPO}" == *"/rhtap-release/"* ]]; then ADVISORY_URL_PREFIX="https://access.stage.redhat.com/errata" @@ -240,11 +326,16 @@ spec: fi LATEST_ADVISORY_URL="${ADVISORY_URL_PREFIX}/${ADVISORY_TYPE}-${ADVISORY_NAME}" - LATEST_ADVISORY_INTERNAL_URL="${GIT_REPO//\.git/}/-/raw/main/${LATEST_ADVISORY_FILE}" + LATEST_ADVISORY_INTERNAL_URL="${GIT_REPO//\.git/}/-/raw/main/${SINGLE_ADVISORY_FILE}" echo -n "$LATEST_ADVISORY_URL" > "$(results.advisory_url.path)" echo -n "$LATEST_ADVISORY_INTERNAL_URL" > "$(results.advisory_internal_url.path)" else + if [[ "$ADVISORY_SET_COUNT" -gt 1 ]]; then + echo "RPMs found in multiple advisories ($ADVISORY_SET_COUNT). Not setting advisory_url." + echo "Advisories:" + jq -r '.[]' "$ADVISORY_SET_FILE" + fi echo -n "" > "$(results.advisory_url.path)" echo -n "" > "$(results.advisory_internal_url.path)" fi diff --git a/tasks/internal/filter-already-released-advisory-rpms-task/tests/mocks.sh b/tasks/internal/filter-already-released-advisory-rpms-task/tests/mocks.sh index 461f3ce9ed..423d320f76 100644 --- a/tasks/internal/filter-already-released-advisory-rpms-task/tests/mocks.sh +++ b/tasks/internal/filter-already-released-advisory-rpms-task/tests/mocks.sh @@ -1,6 +1,24 @@ #!/usr/bin/env bash set -eux +function select-oci-auth() { + echo "/dev/null" +} + +function oras() { + echo "Mock oras called with: $*" >&2 + + if [[ "$1" == "push" ]]; then + return 0 + elif [[ "$1" == "manifest" && "$2" == "fetch" ]]; then + echo '{"digest": "sha256:mockdigest123"}' + return 0 + else + echo "Error: Unexpected oras command: $*" >&2 + exit 1 + fi +} + function git() { echo "Mock git called with: $*" @@ -9,7 +27,6 @@ function git() { elif [[ "$1" == "sparse-checkout" ]]; then : elif [[ "$1" == "checkout" ]]; then - # Create advisory directory structure with advisory.yaml files mkdir -p data/advisories/test-origin/2025/1601 mkdir -p data/advisories/test-origin/2025/1602 touch data/advisories/test-origin/2025/1601/advisory.yaml @@ -39,34 +56,36 @@ function find() { function yq() { echo "Mock yq called with: $*" >&2 - if [[ -z "$3" ]]; then - echo "Error: Empty file path in yq command" >&2 - exit 1 - fi - - advisory_path="$3" - advisory_num=$(echo "$advisory_path" | awk -F'/' '{print $(NF-1)}') + if [[ "$1" == "-o=json" ]]; then + local advisory_path="$3" + local advisory_num + advisory_num=$(echo "$advisory_path" | awk -F'/' '{print $(NF-1)}') - if [[ "$2" == ".spec.type" ]]; then - echo "RHBA" - elif [[ "$2" == ".metadata.name" ]]; then - advisory_year=$(echo "$advisory_path" | awk -F'/' '{print $(NF-2)}') - echo "${advisory_year}:${advisory_num}" - elif [[ "$2" == *".spec.content.artifacts"* ]]; then case "$advisory_num" in 1601) - # These purls match RPMs that should be "in advisory" echo '["pkg:rpm/redhat/released-rpm@1.0-1.fc44?arch=x86_64", "pkg:rpm/redhat/all-released-a@1.0-1.fc44?arch=x86_64", "pkg:rpm/redhat/all-released-b@2.0-1.fc44?arch=x86_64"]' ;; - 1602) - echo '[]' - ;; *) echo '[]' ;; esac + elif [[ "$1" == "-r" ]]; then + local advisory_path="$3" + local advisory_num + advisory_num=$(echo "$advisory_path" | awk -F'/' '{print $(NF-1)}') + + if [[ "$2" == ".spec.type" ]]; then + echo "RHBA" + elif [[ "$2" == ".metadata.name" ]]; then + local advisory_year + advisory_year=$(echo "$advisory_path" | awk -F'/' '{print $(NF-2)}') + echo "${advisory_year}:${advisory_num}" + else + echo "Error: Unexpected yq -r query: $2" >&2 + exit 1 + fi else - echo "Error: Unexpected yq query: $2" >&2 + echo "Error: Unexpected yq command: $*" >&2 exit 1 fi } diff --git a/tasks/internal/filter-already-released-advisory-rpms-task/tests/test-filter-all-released.yaml b/tasks/internal/filter-already-released-advisory-rpms-task/tests/test-filter-all-released.yaml index 39908ee4b2..0c0bec9976 100644 --- a/tasks/internal/filter-already-released-advisory-rpms-task/tests/test-filter-all-released.yaml +++ b/tasks/internal/filter-already-released-advisory-rpms-task/tests/test-filter-all-released.yaml @@ -7,6 +7,21 @@ spec: description: | Test the filter-already-released-advisory-rpms-task when all RPMs are in advisories. Should return empty unreleased list and populated in_advisory list. + params: + - name: ociStorage + description: The OCI repository where artifacts are stored. + type: string + - name: ociArtifactExpiresAfter + type: string + default: "1d" + - name: orasOptions + type: string + default: "--insecure" + - name: trustedArtifactsDebug + type: string + default: "" + - name: dataDir + type: string tasks: - name: run-filter-task taskRef: @@ -22,12 +37,18 @@ spec: value: "filter-advisory-rpms-secret" - name: internalRequestPipelineRunName value: "$(context.pipelineRun.name)" + - name: ociStorage + value: $(params.ociStorage) + - name: orasOptions + value: $(params.orasOptions) - name: validate-result runAfter: - run-filter-task params: - name: result value: "$(tasks.run-filter-task.results.result)" + - name: filter_results_artifact + value: "$(tasks.run-filter-task.results.filter_results_artifact)" - name: unreleased_rpms value: "$(tasks.run-filter-task.results.unreleased_rpms)" - name: in_advisory_rpms @@ -38,6 +59,8 @@ spec: params: - name: result type: string + - name: filter_results_artifact + type: string - name: unreleased_rpms type: string - name: in_advisory_rpms @@ -58,6 +81,13 @@ spec: exit 1 fi + ARTIFACT="$(params.filter_results_artifact)" + if [[ -z "$ARTIFACT" ]]; then + echo "Expected filter_results_artifact to be set" + exit 1 + fi + echo "Filter results artifact: $ARTIFACT" + UNRELEASED=$(base64 -d <<< "$(params.unreleased_rpms)" | gunzip) IN_ADVISORY=$(base64 -d <<< "$(params.in_advisory_rpms)" | gunzip) diff --git a/tasks/internal/filter-already-released-advisory-rpms-task/tests/test-filter-no-advisories.yaml b/tasks/internal/filter-already-released-advisory-rpms-task/tests/test-filter-no-advisories.yaml index 0556fcfaf5..28c14f6ffc 100644 --- a/tasks/internal/filter-already-released-advisory-rpms-task/tests/test-filter-no-advisories.yaml +++ b/tasks/internal/filter-already-released-advisory-rpms-task/tests/test-filter-no-advisories.yaml @@ -7,6 +7,21 @@ spec: description: | Test the filter-already-released-advisory-rpms-task when no advisories exist. All RPMs should be returned as unreleased. + params: + - name: ociStorage + description: The OCI repository where artifacts are stored. + type: string + - name: ociArtifactExpiresAfter + type: string + default: "1d" + - name: orasOptions + type: string + default: "--insecure" + - name: trustedArtifactsDebug + type: string + default: "" + - name: dataDir + type: string tasks: - name: run-filter-task taskRef: @@ -21,12 +36,18 @@ spec: value: "filter-advisory-rpms-secret" - name: internalRequestPipelineRunName value: "$(context.pipelineRun.name)" + - name: ociStorage + value: $(params.ociStorage) + - name: orasOptions + value: $(params.orasOptions) - name: validate-result runAfter: - run-filter-task params: - name: result value: "$(tasks.run-filter-task.results.result)" + - name: filter_results_artifact + value: "$(tasks.run-filter-task.results.filter_results_artifact)" - name: unreleased_rpms value: "$(tasks.run-filter-task.results.unreleased_rpms)" - name: in_advisory_rpms @@ -35,6 +56,8 @@ spec: params: - name: result type: string + - name: filter_results_artifact + type: string - name: unreleased_rpms type: string - name: in_advisory_rpms @@ -53,6 +76,13 @@ spec: exit 1 fi + ARTIFACT="$(params.filter_results_artifact)" + if [[ -z "$ARTIFACT" ]]; then + echo "Expected filter_results_artifact to be set" + exit 1 + fi + echo "Filter results artifact: $ARTIFACT" + UNRELEASED=$(base64 -d <<< "$(params.unreleased_rpms)" | gunzip) IN_ADVISORY=$(base64 -d <<< "$(params.in_advisory_rpms)" | gunzip) diff --git a/tasks/internal/filter-already-released-advisory-rpms-task/tests/test-filter-partial-release.yaml b/tasks/internal/filter-already-released-advisory-rpms-task/tests/test-filter-partial-release.yaml index 1e94f0cafc..33e1613abb 100644 --- a/tasks/internal/filter-already-released-advisory-rpms-task/tests/test-filter-partial-release.yaml +++ b/tasks/internal/filter-already-released-advisory-rpms-task/tests/test-filter-partial-release.yaml @@ -7,6 +7,21 @@ spec: description: | Test the filter-already-released-advisory-rpms-task with partial release. One RPM is in an advisory (released-rpm), one is not (new-rpm). + params: + - name: ociStorage + description: The OCI repository where artifacts are stored. + type: string + - name: ociArtifactExpiresAfter + type: string + default: "1d" + - name: orasOptions + type: string + default: "--insecure" + - name: trustedArtifactsDebug + type: string + default: "" + - name: dataDir + type: string tasks: - name: run-filter-task taskRef: @@ -22,12 +37,18 @@ spec: value: "filter-advisory-rpms-secret" - name: internalRequestPipelineRunName value: "$(context.pipelineRun.name)" + - name: ociStorage + value: $(params.ociStorage) + - name: orasOptions + value: $(params.orasOptions) - name: validate-result runAfter: - run-filter-task params: - name: result value: "$(tasks.run-filter-task.results.result)" + - name: filter_results_artifact + value: "$(tasks.run-filter-task.results.filter_results_artifact)" - name: unreleased_rpms value: "$(tasks.run-filter-task.results.unreleased_rpms)" - name: in_advisory_rpms @@ -36,6 +57,8 @@ spec: params: - name: result type: string + - name: filter_results_artifact + type: string - name: unreleased_rpms type: string - name: in_advisory_rpms @@ -54,6 +77,13 @@ spec: exit 1 fi + ARTIFACT="$(params.filter_results_artifact)" + if [[ -z "$ARTIFACT" ]]; then + echo "Expected filter_results_artifact to be set" + exit 1 + fi + echo "Filter results artifact: $ARTIFACT" + UNRELEASED=$(base64 -d <<< "$(params.unreleased_rpms)" | gunzip) IN_ADVISORY=$(base64 -d <<< "$(params.in_advisory_rpms)" | gunzip) diff --git a/tasks/managed/filter-already-released-advisory-rpms/filter-already-released-advisory-rpms.yaml b/tasks/managed/filter-already-released-advisory-rpms/filter-already-released-advisory-rpms.yaml index b6dd22cec1..5fd8e7fe40 100644 --- a/tasks/managed/filter-already-released-advisory-rpms/filter-already-released-advisory-rpms.yaml +++ b/tasks/managed/filter-already-released-advisory-rpms/filter-already-released-advisory-rpms.yaml @@ -295,26 +295,70 @@ spec: return $rc } - get_repo_latest_version_href() { + # Get the published repository version for a repo (what users can actually access) + # Returns empty string if nothing is published + get_repo_published_version_href() { local repo_name="$1" - local repo_href - repo_href=$(curl_auth -s \ - "${PULP_BASE_URL}/api/pulp/${PULP_DOMAIN}/api/v3/repositories/rpm/rpm/?name=${repo_name}" \ - | jq -r '.results[0].pulp_href') - if [[ -z "$repo_href" || "$repo_href" == "null" ]]; then - return 1 + local dist_response publication_href publication_json + local repo_version_href repo_href repo_json + + # Get distribution for this repo + dist_response=$(curl_auth -s \ + "${PULP_BASE_URL}/api/pulp/${PULP_DOMAIN}/api/v3/distributions/rpm/rpm/?name=${repo_name}") + + # Check if distribution has a repository_version (direct link) + repo_version_href=$(jq -r \ + '.results[0].repository_version // empty' <<< "${dist_response}") + if [[ -n "${repo_version_href}" && "${repo_version_href}" != "null" ]]; then + echo "${repo_version_href}" + return 0 + fi + + # Otherwise check if it has a publication + publication_href=$(jq -r \ + '.results[0].publication // empty' <<< "${dist_response}") + if [[ -n "${publication_href}" && "${publication_href}" != "null" ]]; then + publication_json=$(curl_auth -s "${PULP_BASE_URL}${publication_href}") + repo_version_href=$(jq -r \ + '.repository_version // empty' <<< "${publication_json}") + echo "${repo_version_href}" + return 0 + fi + + # Fallback: check repository's latest version directly + repo_href=$(jq -r \ + '.results[0].repository // empty' <<< "${dist_response}") + if [[ -n "${repo_href}" && "${repo_href}" != "null" ]]; then + repo_json=$(curl_auth -s \ + "${PULP_BASE_URL}${repo_href}") + repo_version_href=$(jq -r \ + '.latest_version_href // empty' <<< "${repo_json}") + if [[ -n "${repo_version_href}" && "${repo_version_href}" != "null" ]]; then + echo "${repo_version_href}" + return 0 + fi fi - curl_auth -s "${PULP_BASE_URL}${repo_href}" | jq -r '.latest_version_href' + + echo "" } - # Check Pulp for RPM digest. Returns: 0=matches, 1=error, 2=not found, 3=mismatch + # Check Pulp for RPM digest in PUBLISHED content only. + # Returns: 0=matches, 1=error, 2=not found/unpublished, 3=mismatch check_pulp_digest() { local repo_name="$1" rpmname="$2" epoch="$3" version="$4" local release="$5" arch="$6" expected_sha="$7" local endpoint="packages" local repo_version_href - repo_version_href="$(get_repo_latest_version_href "${repo_name}")" || return 1 + + # Check PUBLISHED version, not latest version + repo_version_href="$(get_repo_published_version_href "${repo_name}")" || return 1 + + # If no published version, treat as "not found" (allows overwrite) + if [[ -z "${repo_version_href}" || "${repo_version_href}" == "null" ]]; then + echo " -> No published version for ${repo_name}. RPM not accessible to users." >&2 + return 2 + fi local query_url query_url="${PULP_BASE_URL}/api/pulp/${PULP_DOMAIN}/api/v3/content/rpm/${endpoint}/" @@ -354,11 +398,15 @@ spec: RPM_REPOS_JSON="$(jq -c '.mapping["rpm-repositories"] // []' "${DATA_FILE}")" + declare -A ARCH_REPO_INFO_CACHE + while IFS= read -r _repo_line; do + _arch="$(jq -r '.arch' <<< "$_repo_line")" + [[ -z "${ARCH_REPO_INFO_CACHE[$_arch]+x}" ]] && \ + ARCH_REPO_INFO_CACHE["$_arch"]="$(jq -c '{repository_id, repository_name, distro}' <<< "$_repo_line")" + done < <(jq -c '.[]' <<< "$RPM_REPOS_JSON") + get_repo_info_for_arch() { - local arch="$1" - jq -c --arg arch "${arch}" \ - '.[] | select(.arch == $arch) | {repository_id, repository_name, distro}' \ - <<< "${RPM_REPOS_JSON}" | head -1 + echo "${ARCH_REPO_INFO_CACHE[$1]:-}" } should_exclude_file() { @@ -406,9 +454,9 @@ spec: # === Phase 1: Extract RPMs and build purl-enriched snapshot === echo "Extracting RPMs and building purl data..." snapshot_json="$(jq '.' "${SNAPSHOT_FILE}")" - TRANSFORMED_ENTRIES="[]" - COMPONENT_RPMS_MAP_FILE="$(mktemp)" - echo "{}" > "$COMPONENT_RPMS_MAP_FILE" + TRANSFORMED_ENTRIES_FILE="$(mktemp)" + : > "$TRANSFORMED_ENTRIES_FILE" + COMPONENT_RPMS_MAP_DIR="$(mktemp -d)" DEFAULT_ARCH_LIST=() IFS=',' read -ra _raw_default <<< "$(params.DEFAULT_ARCHITECTURES)" @@ -417,18 +465,21 @@ spec: [[ -n "$x" ]] && DEFAULT_ARCH_LIST+=("$x") done - NOARCH_TARGET_REPOS_JSON="[]" + NOARCH_TARGET_REPOS=() for default_arch in "${DEFAULT_ARCH_LIST[@]}"; do repo_info="$(get_repo_info_for_arch "${default_arch}")" - [[ -n "${repo_info}" ]] && NOARCH_TARGET_REPOS_JSON="$(jq --argjson info "${repo_info}" \ - '. + [$info]' <<< "${NOARCH_TARGET_REPOS_JSON}")" + [[ -n "${repo_info}" ]] && NOARCH_TARGET_REPOS+=("${repo_info}") done + json_escape() { + local s="$1" + s="${s//\\/\\\\}" + s="${s//\"/\\\"}" + echo -n "$s" + } + while IFS= read -r component; do - comp_file="$(mktemp)" - echo "$component" > "$comp_file" - image="$(jq -r '.containerImage' "$comp_file")" - name="$(jq -r '.name' "$comp_file")" + IFS='|' read -r image name <<< "$(jq -r '[.containerImage, .name] | join("|")' <<< "$component")" echo "Processing component '${name}' (${image})" >&2 workdir="$(mktemp -d)" @@ -449,14 +500,16 @@ spec: ) if [[ ${#files[@]} -eq 0 ]]; then - rm -rf "${workdir}" "$comp_file" + rm -rf "${workdir}" continue fi mapfile -t ARCHES < <(extract_arches_from_files files) [[ ${#ARCHES[@]} -eq 0 ]] && ARCHES=("${DEFAULT_ARCH_LIST[@]}") - COMPONENT_RPMS="[]" + COMPONENT_RPMS_FILE="$(mktemp)" + : > "$COMPONENT_RPMS_FILE" + for f in "${files[@]}"; do file_abs="${workdir}/files/${f}" local_sha="$(sha256sum "${file_abs}" | awk '{print $1}')" @@ -465,63 +518,74 @@ spec: IFS='|' read -r rpmname epoch version release arch <<< "${q}" [[ -z "${epoch}" || "${epoch}" == "(none)" ]] && epoch="0" + target_repos=() if [[ "${f}" == *.src.rpm ]]; then arch="src" - # Source RPMs use arch="src" in the mapping repo_info="$(get_repo_info_for_arch "src")" if [[ -n "${repo_info}" ]]; then - target_repos_json="[${repo_info}]" + target_repos=("${repo_info}") else echo "Warning: No repository mapping for source arch, skipping ${f}" >&2 continue fi elif [[ "${f}" == *.noarch.rpm ]]; then - [[ "$(jq 'length' <<< "${NOARCH_TARGET_REPOS_JSON}")" -eq 0 ]] && continue - target_repos_json="${NOARCH_TARGET_REPOS_JSON}" + [[ ${#NOARCH_TARGET_REPOS[@]} -eq 0 ]] && continue + target_repos=("${NOARCH_TARGET_REPOS[@]}") else repo_info="$(get_repo_info_for_arch "${arch}")" [[ -z "${repo_info}" ]] && continue - target_repos_json="[${repo_info}]" + target_repos=("${repo_info}") fi - while IFS= read -r repo_obj; do - repo_name="$(jq -r '.repository_name' <<< "${repo_obj}")" - repo_id="$(jq -r '.repository_id' <<< "${repo_obj}")" - distro="$(jq -r '.distro' <<< "${repo_obj}")" + for repo_obj in "${target_repos[@]}"; do + IFS='|' read -r repo_name repo_id distro <<< \ + "$(jq -r '[.repository_name, .repository_id, .distro] | join("|")' <<< "${repo_obj}")" purl="pkg:rpm/redhat/${rpmname}@${version}-${release}?arch=${arch}" [[ -n "${distro}" && "${distro}" != "null" ]] && purl="${purl}&distro=${distro}" [[ -n "${repo_id}" && "${repo_id}" != "null" ]] && purl="${purl}&repository_id=${repo_id}" - entry=$(jq -n \ - --arg name "$name" --arg purl "$purl" --arg sha256 "$local_sha" \ - --arg rpmname "$rpmname" --arg epoch "$epoch" --arg version "$version" \ - --arg release "$release" --arg arch "$arch" \ - --arg repository_name "$repo_name" --arg rpm "$f" \ - --argjson targetRepo "$repo_obj" \ - '{name: $name, purl: $purl, sha256: $sha256, rpmname: $rpmname, - epoch: $epoch, version: $version, release: $release, arch: $arch, - repository_name: $repository_name, rpm: $rpm, targetRepo: $targetRepo}') - TRANSFORMED_ENTRIES=$(jq --argjson e "$entry" '. + [$e]' <<< "$TRANSFORMED_ENTRIES") - - COMPONENT_RPMS=$(jq --arg rpm "$f" --arg sha "$local_sha" \ - --arg rpmname "$rpmname" --arg epoch "$epoch" \ - --arg version "$version" --arg release "$release" \ - --arg arch "$arch" --argjson targetRepo "$repo_obj" \ - '. + [{rpm: $rpm, sha256: $sha, rpmname: $rpmname, epoch: $epoch, - version: $version, release: $release, arch: $arch, - targetRepos: [$targetRepo]}]' <<< "$COMPONENT_RPMS") - done < <(jq -c '.[]' <<< "${target_repos_json}") + _fmt='{"name":"%s","purl":"%s","sha256":"%s",' + _fmt+='"rpmname":"%s","epoch":"%s","version":"%s",' + _fmt+='"release":"%s","arch":"%s",' + _fmt+='"repository_name":"%s","rpm":"%s",' + _fmt+='"targetRepo":%s}\n' + # shellcheck disable=SC2059 + printf "$_fmt" \ + "$(json_escape "$name")" \ + "$(json_escape "$purl")" "$local_sha" \ + "$(json_escape "$rpmname")" "$epoch" \ + "$(json_escape "$version")" \ + "$(json_escape "$release")" "$arch" \ + "$(json_escape "$repo_name")" \ + "$(json_escape "$f")" "$repo_obj" \ + >> "$TRANSFORMED_ENTRIES_FILE" + + _rfmt='{"rpm":"%s","sha256":"%s",' + _rfmt+='"rpmname":"%s","epoch":"%s",' + _rfmt+='"version":"%s","release":"%s",' + _rfmt+='"arch":"%s","targetRepos":[%s]}\n' + # shellcheck disable=SC2059 + printf "$_rfmt" \ + "$(json_escape "$f")" "$local_sha" \ + "$(json_escape "$rpmname")" "$epoch" \ + "$(json_escape "$version")" \ + "$(json_escape "$release")" \ + "$arch" "$repo_obj" \ + >> "$COMPONENT_RPMS_FILE" + done done - jq --arg name "$name" --argjson rpms "$COMPONENT_RPMS" \ - '. + {($name): $rpms}' "$COMPONENT_RPMS_MAP_FILE" > "${COMPONENT_RPMS_MAP_FILE}.tmp" - mv "${COMPONENT_RPMS_MAP_FILE}.tmp" "$COMPONENT_RPMS_MAP_FILE" - rm -rf "${workdir}" "$comp_file" + if [[ -s "$COMPONENT_RPMS_FILE" ]]; then + jq -s '.' "$COMPONENT_RPMS_FILE" > "${COMPONENT_RPMS_MAP_DIR}/${name}.json" + fi + rm -f "$COMPONENT_RPMS_FILE" + rm -rf "${workdir}" done < <(jq -c '.components[]' <<< "${snapshot_json}") - echo "Total transformed entries: $(jq 'length' <<< "$TRANSFORMED_ENTRIES")" + TOTAL_ENTRIES=$(wc -l < "$TRANSFORMED_ENTRIES_FILE") + echo "Total transformed entries: $TOTAL_ENTRIES" - if [[ "$(jq 'length' <<< "$TRANSFORMED_ENTRIES")" -eq 0 ]]; then + if [[ "$TOTAL_ENTRIES" -eq 0 ]]; then echo "No RPMs found in any component, skipping release" echo -n "true" > "$(results.skip_release.path)" echo -n "" > "$(results.latest_advisory_url.path)" @@ -531,7 +595,7 @@ spec: exit 0 fi - transformedSnapshot=$(echo "$TRANSFORMED_ENTRIES" | gzip -c | base64 -w 0) + transformedSnapshot=$(jq -s '.' "$TRANSFORMED_ENTRIES_FILE" | gzip -c | base64 -w 0) # === Phase 2: Create InternalRequest (no Pulp params) === pipelinerun_label="internal-services.appstudio.openshift.io/pipelinerun-uid" @@ -544,6 +608,8 @@ spec: -p origin="$origin" \ -p advisory_secret_name="$advisorySecretName" \ -p internalRequestPipelineRunName="$(params.pipelineRunUid)" \ + -p ociStorage="$(params.ociStorage)" \ + -p orasOptions="$(params.orasOptions)" \ -p taskGitUrl="$(params.taskGitUrl)" \ -p taskGitRevision="$(params.taskGitRevision)" \ -s "$(params.synchronously)" \ @@ -570,26 +636,36 @@ spec: latestAdvisoryUrl="$(jq -jr '.advisory_url // ""' <<< "${results}")" latestAdvisoryInternalUrl="$(jq -jr '.advisory_internal_url // ""' <<< "${results}")" - # === Phase 3: Validate Pulp digests for in-advisory RPMs === - IN_ADVISORY_RAW=$(jq -r '.in_advisory_rpms // ""' <<< "$results") - if [[ -n "$IN_ADVISORY_RAW" ]]; then - IN_ADVISORY_FILE=$(mktemp) - echo "$IN_ADVISORY_RAW" | base64 -d | gunzip > "$IN_ADVISORY_FILE" + # === Phase 3: Pull filter results from OCI artifact === + FILTER_RESULTS_ARTIFACT=$(jq -r '.filter_results_artifact // ""' <<< "$results") + if [[ -z "$FILTER_RESULTS_ARTIFACT" ]]; then + echo "No filter_results_artifact found in results" + exit 1 + fi + + echo "Pulling filter results from: ${FILTER_RESULTS_ARTIFACT}" + FILTER_RESULTS_DIR=$(mktemp -d) + oci_ref="${FILTER_RESULTS_ARTIFACT#oci:}" + oci_storage="${oci_ref%%@*}" + AUTHFILE=$(mktemp) + select-oci-auth "${oci_storage}" > "${AUTHFILE}" + oras pull --registry-config "${AUTHFILE}" "${oci_ref}" -o "$FILTER_RESULTS_DIR" + rm -f "${AUTHFILE}" + + # Extract the tarball + tar -xzf "$FILTER_RESULTS_DIR/filter-results.tar.gz" -C "$FILTER_RESULTS_DIR" + + IN_ADVISORY_FILE="$FILTER_RESULTS_DIR/in_advisory_rpms.json" + UNRELEASED_FILE="$FILTER_RESULTS_DIR/unreleased_rpms.json" + + # === Phase 4: Validate Pulp digests for in-advisory RPMs === + # Advisory is authoritative - if RPM is in advisory, it was released (regardless of Pulp state) + # This check validates that any published RPMs match expected digests (prevents rebuilds) + if [[ -f "$IN_ADVISORY_FILE" ]]; then IN_ADVISORY_COUNT=$(jq 'length' "$IN_ADVISORY_FILE") echo "Validating Pulp digests for $IN_ADVISORY_COUNT in-advisory RPMs..." - while IFS= read -r rpm_entry; do - entry_file=$(mktemp) - echo "$rpm_entry" > "$entry_file" - - rpmname=$(jq -r '.rpmname' "$entry_file") - epoch=$(jq -r '.epoch' "$entry_file") - version=$(jq -r '.version' "$entry_file") - release=$(jq -r '.release' "$entry_file") - arch=$(jq -r '.arch' "$entry_file") - sha256=$(jq -r '.sha256' "$entry_file") - repo_name=$(jq -r '.repository_name' "$entry_file") - + while IFS='|' read -r rpmname epoch version release arch sha256 repo_name; do echo "Checking Pulp digest for ${rpmname}-${version}-${release}.${arch}..." set +e @@ -600,41 +676,37 @@ spec: case $pulp_status in 0) - echo " -> Digest matches in Pulp" + echo " -> Published in Pulp with matching digest" ;; 2) - echo " -> Not in Pulp (yet), advisory is authoritative" + echo " -> Not published in Pulp yet (advisory is authoritative)" ;; 3) echo "Error: Cannot rebuild RPM with same NEVR but different digest" echo "RPM: ${rpmname}-${version}-${release}.${arch}" echo "Snapshot sha256: ${sha256}" echo "Action required: Bump version or release number" - rm -f "$entry_file" "$IN_ADVISORY_FILE" + rm -rf "$FILTER_RESULTS_DIR" exit 1 ;; *) echo "Error: Failed to check Pulp digest (status=$pulp_status)" - rm -f "$entry_file" "$IN_ADVISORY_FILE" + rm -rf "$FILTER_RESULTS_DIR" exit 1 ;; esac - - rm -f "$entry_file" - done < <(jq -c '.[]' "$IN_ADVISORY_FILE") - - rm -f "$IN_ADVISORY_FILE" + done < <(jq -r \ + '.[] | [.rpmname, .epoch, .version, .release, + .arch, .sha256, .repository_name] + | join("|")' "$IN_ADVISORY_FILE") fi - # === Phase 4: Process unreleased RPMs and filter snapshot === - UNRELEASED_RAW=$(jq -r '.unreleased_rpms // ""' <<< "$results") - if [[ -z "$UNRELEASED_RAW" ]]; then - echo "No unreleased_rpms found in results" + # === Phase 5: Process unreleased RPMs and filter snapshot === + if [[ ! -f "$UNRELEASED_FILE" ]]; then + echo "No unreleased_rpms.json found in filter results" exit 1 fi - UNRELEASED_FILE=$(mktemp) - echo "$UNRELEASED_RAW" | base64 -d | gunzip > "$UNRELEASED_FILE" UNRELEASED_COUNT=$(jq 'length' "$UNRELEASED_FILE") echo "Unreleased RPMs: $UNRELEASED_COUNT" @@ -648,12 +720,13 @@ spec: echo -n "$latestAdvisoryInternalUrl" > "$(results.latest_advisory_internal_url.path)" jq '.components = []' "${SNAPSHOT_FILE}" > "${SNAPSHOT_FILE}.tmp" mv "${SNAPSHOT_FILE}.tmp" "${SNAPSHOT_FILE}" - rm -f "$UNRELEASED_FILE" + rm -rf "$FILTER_RESULTS_DIR" exit 0 fi # Build filtered snapshot with rpmsToPublish for unreleased components - FILTERED_COMPONENTS="[]" + FILTERED_COMPS_FILE="$(mktemp)" + : > "$FILTERED_COMPS_FILE" while IFS= read -r comp_name; do [[ -z "$comp_name" ]] && continue @@ -661,22 +734,15 @@ spec: '.components[] | select(.name == $name)' <<< "$snapshot_json") [[ -z "$original_component" ]] && continue - comp_rpms=$(jq --arg name "$comp_name" '.[$name] // []' "$COMPONENT_RPMS_MAP_FILE") - [[ "$(jq 'length' <<< "$comp_rpms")" -eq 0 ]] && continue - - rpms_file="$(mktemp)" - echo "$comp_rpms" > "$rpms_file" - updated_component=$(echo "$original_component" | \ - jq --slurpfile rpms "$rpms_file" '. + {rpmsToPublish: $rpms[0]}') - rm -f "$rpms_file" + comp_rpms_file="${COMPONENT_RPMS_MAP_DIR}/${comp_name}.json" + [[ ! -s "$comp_rpms_file" ]] && continue - comp_file="$(mktemp)" - echo "$updated_component" > "$comp_file" - FILTERED_COMPONENTS=$(jq --slurpfile comp "$comp_file" '. + $comp' <<< "$FILTERED_COMPONENTS") - rm -f "$comp_file" + echo "$original_component" | \ + jq --slurpfile rpms "$comp_rpms_file" '. + {rpmsToPublish: $rpms[0]}' \ + >> "$FILTERED_COMPS_FILE" done < <(jq -r '.[]' <<< "$UNRELEASED_COMPONENTS") - jq --argjson comps "$FILTERED_COMPONENTS" \ + jq --slurpfile comps "$FILTERED_COMPS_FILE" \ '.components = $comps' "${SNAPSHOT_FILE}" > "${SNAPSHOT_FILE}.tmp" mv "${SNAPSHOT_FILE}.tmp" "${SNAPSHOT_FILE}" @@ -685,7 +751,7 @@ spec: echo -n "" > "$(results.latest_advisory_url.path)" echo -n "" > "$(results.latest_advisory_internal_url.path)" - rm -f "$UNRELEASED_FILE" "$COMPONENT_RPMS_MAP_FILE" + rm -rf "$FILTERED_COMPS_FILE" "$COMPONENT_RPMS_MAP_DIR" "$TRANSFORMED_ENTRIES_FILE" "$FILTER_RESULTS_DIR" - name: create-trusted-artifact computeResources: limits: diff --git a/tasks/managed/filter-already-released-advisory-rpms/tests/mocks.sh b/tasks/managed/filter-already-released-advisory-rpms/tests/mocks.sh index eb4be24d11..ad6927a45b 100644 --- a/tasks/managed/filter-already-released-advisory-rpms/tests/mocks.sh +++ b/tasks/managed/filter-already-released-advisory-rpms/tests/mocks.sh @@ -11,10 +11,10 @@ function select-oci-auth() { function oras() { echo "Mock oras called with: $*" echo $* >> $(params.dataDir)/mock_oras.txt - local args="$*" if [[ "$*" == "pull --registry-config"* ]]; then output_file_dir="" + local oci_ref="" echo "none" > "${CONTENT_EXISTS_MODE_FILE}" while [[ $# -gt 0 ]]; do case "$1" in @@ -22,20 +22,45 @@ function oras() { output_file_dir="$2" shift 2 ;; + --registry-config) + shift 2 + ;; + pull) + shift + ;; *) + oci_ref="$1" shift ;; esac done mkdir -p "${output_file_dir}" - touch "${output_file_dir}/hello-2.12.1-6.fc44.aarch64.rpm" - touch "${output_file_dir}/hello-2.12.1-6.fc44.ppc64le.rpm" - touch "${output_file_dir}/hello-2.12.1-6.fc44.s390x.rpm" - touch "${output_file_dir}/hello-2.12.1-6.fc44.src.rpm" - touch "${output_file_dir}/hello-2.12.1-6.fc44.x86_64.rpm" - touch "${output_file_dir}/hello-docs-2.12.1-6.fc44.noarch.rpm" - mkdir -p "${output_file_dir}/logs" + + if [[ "$oci_ref" == *"filter-results"* ]]; then + # Filter results artifact pull - create the tarball + local tmpdir + tmpdir=$(mktemp -d) + cat /tmp/mock_unreleased_rpms.json > "$tmpdir/unreleased_rpms.json" + cat /tmp/mock_in_advisory_rpms.json > "$tmpdir/in_advisory_rpms.json" + tar -czf "${output_file_dir}/filter-results.tar.gz" \ + -C "$tmpdir" unreleased_rpms.json in_advisory_rpms.json + rm -rf "$tmpdir" + else + # RPM artifact pull - create mock RPM files + touch "${output_file_dir}/hello-2.12.1-6.fc44.aarch64.rpm" + touch "${output_file_dir}/hello-2.12.1-6.fc44.ppc64le.rpm" + touch "${output_file_dir}/hello-2.12.1-6.fc44.s390x.rpm" + touch "${output_file_dir}/hello-2.12.1-6.fc44.src.rpm" + touch "${output_file_dir}/hello-2.12.1-6.fc44.x86_64.rpm" + touch "${output_file_dir}/hello-docs-2.12.1-6.fc44.noarch.rpm" + mkdir -p "${output_file_dir}/logs" + fi + return 0 + fi + + if [[ "$*" == "push"* ]] || [[ "$*" == "manifest"* ]]; then + echo "sha256:mockdigest123" return 0 fi } @@ -99,13 +124,15 @@ function internal-request() { } function kubectl() { - UNRELEASED_RPMS=$(echo -n '[{"name":"test-component","purl":"pkg:rpm/redhat/hello@2.12.1-6.fc44?arch=x86_64","sha256":"abc123","rpmname":"hello","epoch":"0","version":"2.12.1","release":"6.fc44","arch":"x86_64","repository_name":"x86_64"}]' | gzip -c | base64 -w 0) - IN_ADVISORY_RPMS=$(echo -n '[]' | gzip -c | base64 -w 0) + # Write mock filter result files for the oras pull mock to serve + cat > /tmp/mock_unreleased_rpms.json << 'MOCKEOF' +[{"name":"test-component","purl":"pkg:rpm/redhat/hello@2.12.1-6.fc44?arch=x86_64","sha256":"abc123","rpmname":"hello","epoch":"0","version":"2.12.1","release":"6.fc44","arch":"x86_64","repository_name":"x86_64"}] +MOCKEOF + echo '[]' > /tmp/mock_in_advisory_rpms.json MOCK_RESULTS='{ "result": "Success", - "unreleased_rpms": "'"$UNRELEASED_RPMS"'", - "in_advisory_rpms": "'"$IN_ADVISORY_RPMS"'", + "filter_results_artifact": "oci:quay.io/mock/filter-results@sha256:mockdigest123", "internalRequestPipelineRunName": "test-pipeline-run", "internalRequestTaskRunName": "test-task-run", "advisory_url": "",