collect-data.yaml

---
apiVersion: tekton.dev/v1
kind: Task
metadata:
  name: collect-data
  annotations:
    tekton.dev/pipelines.minVersion: "0.12.1"
    tekton.dev/tags: release
spec:
  description: |-
    Tekton task to collect the information added to the data field of the release resources.

    The purpose of this task is to collect all the data and supply it to the other task in the pipeline by creating
    a json file called `data.json` in the workspace.

    This task also stores the passed resources as json files in a workspace.

    The parameters to this task are lowercase instead of camelCase because they are passed from the operator, and the
    operator passes them as lowercase.

    A task result is returned for each resource with the relative path to the stored JSON for it in the workspace.

    Finally, the task checks that the keys from the correct resource (a key that should come from the
    ReleasePlanAdmission should not be present in the Release data section).
  params:
    - name: release
      type: string
      description: The namespaced name of the Release
    - name: releasePlan
      type: string
      description: The namespaced name of the ReleasePlan
    - name: releasePlanAdmission
      type: string
      description: The namespaced name of the ReleasePlanAdmission
    - name: releaseServiceConfig
      type: string
      description: The namespaced name of the ReleaseServiceConfig
    - name: snapshot
      type: string
      description: The namespaced name of the Snapshot
    - name: subdirectory
      description: Subdirectory inside the workspace to be used
      type: string
      default: ""
    - name: ociStorage
      description: The OCI repository where the Trusted Artifacts are stored
      type: string
      default: "empty"
    - name: orasOptions
      description: oras options to pass to Trusted Artifacts calls
      type: string
      default: ""
    - name: ociArtifactExpiresAfter
      description: Expiration date for the trusted artifacts created in the
        OCI repository. An empty string means the artifacts do not expire
      type: string
      default: "1d"
    - name: trustedArtifactsDebug
      description: Flag to enable debug logging in trusted artifacts. Set to a non-empty string to enable
      type: string
      default: ""
    - name: dataDir
      description: The location where data will be stored
      type: string
      default: /var/workdir/release
    - name: taskGitUrl
      type: string
      description: The url to the git repo where the release-service-catalog tasks and stepactions to be used are stored
    - name: taskGitRevision
      type: string
      description: The revision in the taskGitUrl repo to be used
    - name: caTrustConfigMapName
      type: string
      description: The name of the ConfigMap to read CA bundle data from
      default: trusted-ca
    - name: caTrustConfigMapKey
      type: string
      description: The name of the key in the ConfigMap that contains the CA bundle data
      default: ca-bundle.crt
    - name: caCertPath
      type: string
      description: Path to CA certificate bundle for TLS verification with self-signed certificates
      default: /mnt/trusted-ca/ca-bundle.crt
  results:
    - name: release
      type: string
      description: The relative path in the workspace to the stored release json
    - name: releasePlan
      type: string
      description: The relative path in the workspace to the stored releasePlan json
    - name: releasePlanAdmission
      type: string
      description: The relative path in the workspace to the stored releasePlanAdmission json
    - name: releaseServiceConfig
      type: string
      description: The relative path in the workspace to the stored releaseServiceConfig json
    - name: snapshotSpec
      type: string
      description: The relative path in the workspace to the stored snapshotSpec json
    - name: data
      type: string
      description: The relative path in the workspace to the stored data json
    - name: resultsDir
      type: string
      description: The relative path in the workspace to the results directory
    - name: singleComponentMode
      type: string
      description: single component mode
    - name: snapshotName
      type: string
      description: name of Snapshot resource
    - name: snapshotNamespace
      type: string
      description: namespace where Snapshot is located
    - name: snapshotBuildId
      type: string
      description: Build Id where Snapshot originated
    - name: releasePipelineMetadata
      type: string
      description: json object containing git resolver metadata about the running release pipeline
    - name: sourceDataArtifact
      type: string
      description: Produced trusted data artifact
    - name: subdirectory
      type: string
      description: Subdirectory inside the workspace to be used
  volumes:
    - name: workdir
      emptyDir: {}
    - name: trusted-ca
      configMap:
        name: $(params.caTrustConfigMapName)
        items:
          - key: $(params.caTrustConfigMapKey)
            path: ca-bundle.crt
        optional: true
  stepTemplate:
    volumeMounts:
      - mountPath: /var/workdir
        name: workdir
      - name: trusted-ca
        mountPath: /mnt/trusted-ca
        readOnly: true
    env:
      - name: IMAGE_EXPIRES_AFTER
        value: $(params.ociArtifactExpiresAfter)
      - name: "ORAS_OPTIONS"
        value: "$(params.orasOptions)"
      - name: "DEBUG"
        value: "$(params.trustedArtifactsDebug)"
    securityContext:
      runAsUser: 1001
  steps:
    - name: collect-data
      image: quay.io/konflux-ci/release-service-utils@sha256:440c27f13adab931460a9371df82d3757b77f7ccc1b7604d737e0c78760a95c9
      computeResources:
        limits:
          memory: 64Mi
        requests:
          memory: 64Mi
          cpu: 50m
      env:
        - name: "RELEASE"
          value: '$(params.release)'
        - name: "RELEASE_PLAN"
          value: '$(params.releasePlan)'
        - name: "RELEASE_PLAN_ADMISSION"
          value: '$(params.releasePlanAdmission)'
        - name: "RELEASE_SERVICE_CONFIG"
          value: '$(params.releaseServiceConfig)'
        - name: "SNAPSHOT"
          value: '$(params.snapshot)'
      script: |
        #!/usr/bin/env bash
        set -eo pipefail

        if [ -f "/mnt/trusted-ca/ca-bundle.crt" ]; then
            export SSL_CERT_FILE="/mnt/trusted-ca/ca-bundle.crt"
        fi

        set -x
        echo -n "$(params.subdirectory)" > "$(results.subdirectory.path)"

        RESULTS_DIR_PATH="results"
        if [ -n "$(params.subdirectory)" ]; then
          mkdir -p "$(params.dataDir)/$(params.subdirectory)"
          RESULTS_DIR_PATH="$(params.subdirectory)/results"
        fi

        mkdir -p "$(params.dataDir)/$RESULTS_DIR_PATH"
        echo -n "$RESULTS_DIR_PATH" > "$(results.resultsDir.path)"

        RELEASE_PATH="$(params.subdirectory)/release.json"
        echo -n "$RELEASE_PATH" > "$(results.release.path)"
        get-resource "release" "${RELEASE}" | tee "$(params.dataDir)/$RELEASE_PATH"

        RELEASEPLAN_PATH="$(params.subdirectory)/release_plan.json"
        echo -n "$RELEASEPLAN_PATH" > "$(results.releasePlan.path)"
        get-resource "releaseplan" "${RELEASE_PLAN}" | tee "$(params.dataDir)/$RELEASEPLAN_PATH"

        RELEASEPLANADMISSION_PATH="$(params.subdirectory)/release_plan_admission.json"
        echo -n "$RELEASEPLANADMISSION_PATH" > "$(results.releasePlanAdmission.path)"
        get-resource "releaseplanadmission" "${RELEASE_PLAN_ADMISSION}" \
          | tee "$(params.dataDir)/$RELEASEPLANADMISSION_PATH"

        RELEASESERVICECONFIG_PATH="$(params.subdirectory)/release_service_config.json"
        echo -n "$RELEASESERVICECONFIG_PATH" > "$(results.releaseServiceConfig.path)"
        get-resource "releaseserviceconfig" "${RELEASE_SERVICE_CONFIG}" \
          | tee "$(params.dataDir)/$RELEASESERVICECONFIG_PATH"

        echo -e "\nFetching Snapshot Spec"
        SNAPSHOTSPEC_PATH="$(params.subdirectory)/snapshot_spec.json"
        echo -n "$SNAPSHOTSPEC_PATH" > "$(results.snapshotSpec.path)"
        # Empty-string componentGroup is treated as unset; fall back to .application, then drop it.
        get-resource "snapshot" "${SNAPSHOT}" "{.spec}" \
          | jq '(if (.componentGroup == null or .componentGroup == "")
            then .componentGroup = .application else . end)
            | del(.application)' \
          | tee "$(params.dataDir)/$SNAPSHOTSPEC_PATH"
        labels=$(get-resource "snapshot" "${SNAPSHOT}" "{.metadata.labels}")
        BUILD_ID=$(jq -r '."appstudio.openshift.io/build-pipelinerun" // ""' <<< "${labels}")
        echo -n "${BUILD_ID}" | tee "$(results.snapshotBuildId.path)"

        echo -e "\nGenerating collectors data"
        collectors_status=$(get-resource "release" "${RELEASE}" "{.status.collectors}")
        echo "***collectors status"
        echo "${collectors_status}"
        echo "***"

        collectors_result=$(jq -c '
          def deepmerge(a; b):
            reduce b[] as $item (a;
            reduce ($item | keys_unsorted[]) as $key (.;
            $item[$key] as $val | ($val | type) as $type | .[$key] = if ($type == "object") then
            deepmerge({}; [if .[$key] == null then {} else .[$key] end, $val])
            elif ($type == "array") then
            (.[$key] + $val | unique)
            else
            $val
            end)
            );

          # Ensure we safely handle missing collectors
          (.? // {}) as $collectors |

          # Flatten and combine the managed and tenant sections
          [($collectors.managed? // {} | to_entries | map(.value)) +
           ($collectors.tenant? // {} | to_entries | map(.value))] |
           flatten |
           deepmerge({}; .)
        ' <<< "${collectors_status}")
        echo "***collectors"
        jq <<< "$collectors_result"
        echo "***"

        echo -e "\nFetching merged data json"
        release_result=$(get-resource "release" "${RELEASE}" "{.spec.data}")

        release_plan_result=$(get-resource "releaseplan" "${RELEASE_PLAN}" "{.spec.data}")

        release_plan_admission_result=$(get-resource "releaseplanadmission" "${RELEASE_PLAN_ADMISSION}" \
            "{.spec.data}")

        # Merge collectors and Release keys. Release has higher priority
        merged_output=$(merge-json "$collectors_result" "$release_result")

        # Merge now with ReleasePlan keys. ReleasePlan has higher priority
        merged_output=$(merge-json "$merged_output" "$release_plan_result")

        # Finally merge with ReleasePlanAdmission keys. ReleasePlanAdmission has higher priority
        merged_output=$(merge-json "$merged_output" "$release_plan_admission_result")

        DATA_PATH="$(params.subdirectory)/data.json"
        echo -n "$DATA_PATH" > "$(results.data.path)"
        echo "$merged_output" | tee "$(params.dataDir)/$DATA_PATH"

        # get pipeline ref info
        pipelineref=$(jq -c '.spec.pipeline.pipelineRef' \
          "$(params.dataDir)/$(params.subdirectory)/release_plan_admission.json")
        resolver=$(jq -r '.resolver // ""' <<< "${pipelineref}")
        if [ "${resolver}" == "git" ] ; then
          url=$(jq -r '.params[] | select(.name=="url") | .value' <<< "${pipelineref}")
          revision=$(jq -r '.params[] | select(.name=="revision") | .value' <<< "${pipelineref}")
          pathinrepo=$(jq -r '.params[] | select(.name=="pathInRepo") | .value' <<< "${pipelineref}")
          orgrepo="${url#*/*/*/}"
          org=$(echo "${orgrepo}" | cut -f1 -d/)
          repo=$(echo "${orgrepo}" | cut -f2 -d/ | cut -d. -f1)

          # taskGitRevision is resolved to a SHA by the release-service operator,
          # fall back to the revision from the pipeline ref if it is not a SHA
          sha="$(params.taskGitRevision)"
          if [[ ! "${sha}" =~ ^[0-9a-f]{40}$ ]]; then
            sha=$(curl-with-retry -s "https://api.github.com/repos/${org}/${repo}/commits/${revision}" \
              | jq -r '.sha // ""' || true)
          fi

        fi

        org="${org:-unknown}"
        repo="${repo:-unknown}"
        revision="${revision:-unknown}"
        pathinrepo="${pathinrepo:-unknown}"
        sha="${sha:-unknown}"

        echo ""
        echo "Release Pipeline Ref Info:"
        echo "--------------------------"

        json=$(jq -n -c \
          --arg org "${org}" \
          --arg repo "${repo}" \
          --arg revision "${revision}" \
          --arg pathinrepo "${pathinrepo}" \
          --arg sha "${sha}" \
          '$ARGS.named')

        echo "${json}" > "$(results.releasePipelineMetadata.path)"
        # pretty print for log message
        jq . <<< "$json"

        SINGLE_COMPONENT_MODE=$(jq -r '.singleComponentMode // "false"' "$(params.dataDir)/$DATA_PATH")
        SNAPSHOT_NAME=$(echo "${SNAPSHOT}" | cut -f2 -d/)
        SNAPSHOT_NAMESPACE=$(echo "${SNAPSHOT}" | cut -f1 -d/)

        echo -n "${SINGLE_COMPONENT_MODE}" | tee "$(results.singleComponentMode.path)"
        echo -n "${SNAPSHOT_NAME}" | tee "$(results.snapshotName.path)"
        echo -n "${SNAPSHOT_NAMESPACE}" | tee "$(results.snapshotNamespace.path)"

    - name: check-data-key-sources
      image: quay.io/konflux-ci/release-service-utils@sha256:440c27f13adab931460a9371df82d3757b77f7ccc1b7604d737e0c78760a95c9
      computeResources:
        limits:
          memory: 32Mi
        requests:
          memory: 32Mi
          cpu: 10m
      script: |
        #!/usr/bin/env bash
        set -ex

        DISALLOWED_KEYS_JSON='{
            "Release": [
                "releaseNotes.product_id",
                "releaseNotes.product_name",
                "releaseNotes.product_version",
                "releaseNotes.product_stream",
                "releaseNotes.cpe",
                "releaseNotes.allow_custom_live_id"
            ],
            "ReleasePlan": [
                "releaseNotes.product_id",
                "releaseNotes.product_name",
                "releaseNotes.product_version",
                "releaseNotes.product_stream",
                "releaseNotes.cpe",
                "releaseNotes.allow_custom_live_id"
            ],
            "ReleasePlanAdmission": [
            ]
        }'

        RC=0

        check_source () { # Expected arguments are [CRD from DISALLOWED_KEYS_JSON, file]
            for KEY in $(jq -r ".$1[]" <<< "$DISALLOWED_KEYS_JSON") ; do
                if [[ $(jq ".spec.data.$KEY" "$2") != "null" ]] ; then
                    echo "Found disallowed key: $KEY in resource $1"
                    RC=1
                fi
            done
        }

        check_source "Release" "$(params.dataDir)/$(params.subdirectory)/release.json"
        check_source "ReleasePlan" "$(params.dataDir)/$(params.subdirectory)/release_plan.json"
        check_source "ReleasePlanAdmission" \
            "$(params.dataDir)/$(params.subdirectory)/release_plan_admission.json"

        exit $RC
    - name: create-trusted-artifact
      computeResources:
        limits:
          memory: 128Mi
        requests:
          memory: 128Mi
          cpu: 250m
      ref:
        resolver: "git"
        params:
          - name: url
            value: "$(params.taskGitUrl)"
          - name: revision
            value: "$(params.taskGitRevision)"
          - name: pathInRepo
            value: stepactions/create-trusted-artifact/create-trusted-artifact.yaml
      params:
        - name: ociStorage
          value: $(params.ociStorage)
        - name: workDir
          value: $(params.dataDir)
        - name: sourceDataArtifact
          value: $(results.sourceDataArtifact.path)
        - name: caCertPath
          value: $(params.caCertPath)