feat: refactor cleanup cronjob to use pagination and reduce memory usage

Replace AWK-based processing with kubectl pagination API to handle large
resource sets more efficiently. Process resources in batches of 100 using
continuation tokens, and stream through jq to minimize memory footprint.
Increase memory limit from 256Mi to 512Mi to accommodate batch processing.
Add deletion and error counters for better observability.

Generated-By: Claude Sonnet 4.5 <noreply@anthropic.com>

Signed-off-by: David Moreno García <damoreno@redhat.com>

Author: davidmogar

components/internal-services/internal-staging/cronjob/cleanup-internal-requests-pipelineruns.yaml

@@ -35,39 +35,63 @@ spec:
                 - |
                   set -o pipefail
                   PATH="/bin:/usr/bin:/usr/local/bin"
-                  PRUNING_CRS_FILE="/var/tmp/crs-to-be-pruned"
-                  KUBECTL_OUTPUT=$(mktemp -p /var/tmp)
                   SECONDS_BACK=$(date -d "${OLDER_THAN}" +%s)
+                  BATCH_SIZE=100
+                  DELETED_COUNT=0
+                  ERROR_COUNT=0
 
-                  if ! kubectl get "${CR_TYPE}" -n "${CR_NAMESPACE}" -l "${LABELS}" \
-                  --template '{{range .items}}{{if .status.completionTime}}{{.metadata.name}}{{"\t"}}{{.metadata.namespace}}{{"\t"}}{{.status.completionTime}}{{"\n"}}{{end}}{{end}}' > $KUBECTL_OUTPUT; then
-                    echo "ERROR: failed to list ${CR_TYPE} resources"
-                    exit 1
-                  fi
+                  echo "INFO: Starting cleanup for ${CR_TYPE} in ${CR_NAMESPACE} older than ${OLDER_THAN}"
+                  echo "INFO: Processing in batches of ${BATCH_SIZE}"
 
-                  awk -v since=${SECONDS_BACK} '{
-                       # parsing the completionTime and converting it to epoch
-                       # so we can compute the precise CRs that should be deleted
-                       gsub("[:\\-TZ]", " ", $3)
-                       t=mktime($3)
-                       completionTime=strftime("%s", t)
-                       #
-                       # completionTime should be smaller than `since` seconds so it can be deleted
-                       if(since > completionTime) {
-                         args="%s:%s\n"
-                         printf(args, $1, $2)
-                       }
-                    }' $KUBECTL_OUTPUT > $PRUNING_CRS_FILE
+                  # Process resources in batches using pagination
+                  CONTINUE=""
+                  while true; do
+                    # Build kubectl command with limit and continue token
+                    CMD="kubectl get ${CR_TYPE} -n ${CR_NAMESPACE} -l ${LABELS} --limit=${BATCH_SIZE}"
+                    if [ -n "$CONTINUE" ]; then
+                      CMD="$CMD --continue=${CONTINUE}"
+                    fi
+                    CMD="$CMD -o json"
+
+                    # Fetch batch
+                    BATCH_OUTPUT=$(mktemp -p /var/tmp)
+                    if ! eval "$CMD" > "$BATCH_OUTPUT" 2>&1; then
+                      echo "ERROR: failed to list ${CR_TYPE} resources"
+                      rm -f "$BATCH_OUTPUT"
+                      exit 1
+                    fi
+
+                    # Process each item in the batch individually to minimize memory
+                    jq -r '.items[] | select(.status.completionTime != null) | "\(.metadata.name):\(.metadata.namespace):\(.status.completionTime)"' "$BATCH_OUTPUT" | \
+                    while IFS=: read -r cr_name cr_namespace completion_time; do
+                      # Convert completionTime to epoch
+                      completion_epoch=$(date -d "${completion_time}" +%s 2>/dev/null || echo "0")
 
-                  while IFS= read -r line; do
-                    cr=${line%:*}
-                    namespace=${line#*:}
-                    if output=$(kubectl delete "${CR_TYPE}" "${cr}" -n "${namespace}" --wait=false --timeout=30s 2>&1); then
-                      echo "INFO: namespace=${namespace} ${output}"
-                    else
-                      echo "ERROR: namespace=${namespace} ${output}"
+                      # Check if old enough to delete
+                      if [ "$SECONDS_BACK" -gt "$completion_epoch" ]; then
+                        if kubectl delete "${CR_TYPE}" "${cr_name}" -n "${cr_namespace}" --wait=false --timeout=30s 2>&1; then
+                          echo "INFO: Deleted ${cr_name} in ${cr_namespace} (completed: ${completion_time})"
+                          DELETED_COUNT=$((DELETED_COUNT + 1))
+                        else
+                          echo "ERROR: Failed to delete ${cr_name} in ${cr_namespace}"
+                          ERROR_COUNT=$((ERROR_COUNT + 1))
+                        fi
+                      fi
+                    done
+
+                    # Check for continuation token
+                    CONTINUE=$(jq -r '.metadata.continue // empty' "$BATCH_OUTPUT")
+                    rm -f "$BATCH_OUTPUT"
+
+                    # Exit if no more pages
+                    if [ -z "$CONTINUE" ]; then
+                      break
                     fi
-                  done < $PRUNING_CRS_FILE
+
+                    echo "INFO: Fetching next batch (continue token present)"
+                  done
+
+                  echo "INFO: Cleanup complete. Deleted: ${DELETED_COUNT}, Errors: ${ERROR_COUNT}"
               imagePullPolicy: IfNotPresent
               image: quay.io/konflux-ci/release-service-utils:9089cafbf36bb889b4b73d8c2965613810f13736
               volumeMounts:
@@ -76,10 +100,10 @@ spec:
               resources:
                 limits:
                   cpu: 200m
-                  memory: 256Mi
+                  memory: 512Mi
                 requests:
                   cpu: 200m
-                  memory: 256Mi
+                  memory: 512Mi
               securityContext:
                 allowPrivilegeEscalation: false
                 capabilities: