Skip to content

Commit 1742ffb

Browse files
davidmogardavidmogar
committed
feat: refactor cleanup cronjob to use pagination and reduce memory usage
Replace AWK-based processing with kubectl pagination API to handle large resource sets more efficiently. Process resources in batches of 100 using continuation tokens, and stream through jq to minimize memory footprint. Increase memory limit from 256Mi to 512Mi to accommodate batch processing. Add deletion and error counters for better observability. Generated-By: Claude Sonnet 4.5 <noreply@anthropic.com> Signed-off-by: David Moreno García <damoreno@redhat.com>
1 parent 1ad840b commit 1742ffb

2 files changed

Lines changed: 108 additions & 60 deletions

File tree

‎components/internal-services/internal-production/cronjob/cleanup-internal-requests-pipelineruns.yaml‎

Lines changed: 54 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -35,39 +35,63 @@ spec:
3535
- |
3636
set -o pipefail
3737
PATH="/bin:/usr/bin:/usr/local/bin"
38-
PRUNING_CRS_FILE="/var/tmp/crs-to-be-pruned"
39-
KUBECTL_OUTPUT=$(mktemp -p /var/tmp)
4038
SECONDS_BACK=$(date -d "${OLDER_THAN}" +%s)
39+
BATCH_SIZE=100
40+
DELETED_COUNT=0
41+
ERROR_COUNT=0
4142
42-
if ! kubectl get "${CR_TYPE}" -n "${CR_NAMESPACE}" -l "${LABELS}" \
43-
--template '{{range .items}}{{if .status.completionTime}}{{.metadata.name}}{{"\t"}}{{.metadata.namespace}}{{"\t"}}{{.status.completionTime}}{{"\n"}}{{end}}{{end}}' > $KUBECTL_OUTPUT; then
44-
echo "ERROR: failed to list ${CR_TYPE} resources"
45-
exit 1
46-
fi
43+
echo "INFO: Starting cleanup for ${CR_TYPE} in ${CR_NAMESPACE} older than ${OLDER_THAN}"
44+
echo "INFO: Processing in batches of ${BATCH_SIZE}"
4745
48-
awk -v since=${SECONDS_BACK} '{
49-
# parsing the completionTime and converting it to epoch
50-
# so we can compute the precise CRs that should be deleted
51-
gsub("[:\\-TZ]", " ", $3)
52-
t=mktime($3)
53-
completionTime=strftime("%s", t)
54-
#
55-
# completionTime should be smaller than `since` seconds so it can be deleted
56-
if(since > completionTime) {
57-
args="%s:%s\n"
58-
printf(args, $1, $2)
59-
}
60-
}' $KUBECTL_OUTPUT > $PRUNING_CRS_FILE
46+
# Process resources in batches using pagination
47+
CONTINUE=""
48+
while true; do
49+
# Build kubectl command with limit and continue token
50+
CMD="kubectl get ${CR_TYPE} -n ${CR_NAMESPACE} -l ${LABELS} --limit=${BATCH_SIZE}"
51+
if [ -n "$CONTINUE" ]; then
52+
CMD="$CMD --continue=${CONTINUE}"
53+
fi
54+
CMD="$CMD -o json"
55+
56+
# Fetch batch
57+
BATCH_OUTPUT=$(mktemp -p /var/tmp)
58+
if ! eval "$CMD" > "$BATCH_OUTPUT" 2>&1; then
59+
echo "ERROR: failed to list ${CR_TYPE} resources"
60+
rm -f "$BATCH_OUTPUT"
61+
exit 1
62+
fi
63+
64+
# Process each item in the batch individually to minimize memory
65+
jq -r '.items[] | select(.status.completionTime != null) | "\(.metadata.name):\(.metadata.namespace):\(.status.completionTime)"' "$BATCH_OUTPUT" | \
66+
while IFS=: read -r cr_name cr_namespace completion_time; do
67+
# Convert completionTime to epoch
68+
completion_epoch=$(date -d "${completion_time}" +%s 2>/dev/null || echo "0")
6169
62-
while IFS= read -r line; do
63-
cr=${line%:*}
64-
namespace=${line#*:}
65-
if output=$(kubectl delete "${CR_TYPE}" "${cr}" -n "${namespace}" --wait=false --timeout=30s 2>&1); then
66-
echo "INFO: namespace=${namespace} ${output}"
67-
else
68-
echo "ERROR: namespace=${namespace} ${output}"
70+
# Check if old enough to delete
71+
if [ "$SECONDS_BACK" -gt "$completion_epoch" ]; then
72+
if kubectl delete "${CR_TYPE}" "${cr_name}" -n "${cr_namespace}" --wait=false --timeout=30s 2>&1; then
73+
echo "INFO: Deleted ${cr_name} in ${cr_namespace} (completed: ${completion_time})"
74+
DELETED_COUNT=$((DELETED_COUNT + 1))
75+
else
76+
echo "ERROR: Failed to delete ${cr_name} in ${cr_namespace}"
77+
ERROR_COUNT=$((ERROR_COUNT + 1))
78+
fi
79+
fi
80+
done
81+
82+
# Check for continuation token
83+
CONTINUE=$(jq -r '.metadata.continue // empty' "$BATCH_OUTPUT")
84+
rm -f "$BATCH_OUTPUT"
85+
86+
# Exit if no more pages
87+
if [ -z "$CONTINUE" ]; then
88+
break
6989
fi
70-
done < $PRUNING_CRS_FILE
90+
91+
echo "INFO: Fetching next batch (continue token present)"
92+
done
93+
94+
echo "INFO: Cleanup complete. Deleted: ${DELETED_COUNT}, Errors: ${ERROR_COUNT}"
7195
imagePullPolicy: IfNotPresent
7296
image: quay.io/konflux-ci/release-service-utils:9089cafbf36bb889b4b73d8c2965613810f13736
7397
volumeMounts:
@@ -76,10 +100,10 @@ spec:
76100
resources:
77101
limits:
78102
cpu: 200m
79-
memory: 256Mi
103+
memory: 512Mi
80104
requests:
81105
cpu: 200m
82-
memory: 256Mi
106+
memory: 512Mi
83107
securityContext:
84108
allowPrivilegeEscalation: false
85109
capabilities:

‎components/internal-services/internal-staging/cronjob/cleanup-internal-requests-pipelineruns.yaml‎

Lines changed: 54 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -35,39 +35,63 @@ spec:
3535
- |
3636
set -o pipefail
3737
PATH="/bin:/usr/bin:/usr/local/bin"
38-
PRUNING_CRS_FILE="/var/tmp/crs-to-be-pruned"
39-
KUBECTL_OUTPUT=$(mktemp -p /var/tmp)
4038
SECONDS_BACK=$(date -d "${OLDER_THAN}" +%s)
39+
BATCH_SIZE=100
40+
DELETED_COUNT=0
41+
ERROR_COUNT=0
4142
42-
if ! kubectl get "${CR_TYPE}" -n "${CR_NAMESPACE}" -l "${LABELS}" \
43-
--template '{{range .items}}{{if .status.completionTime}}{{.metadata.name}}{{"\t"}}{{.metadata.namespace}}{{"\t"}}{{.status.completionTime}}{{"\n"}}{{end}}{{end}}' > $KUBECTL_OUTPUT; then
44-
echo "ERROR: failed to list ${CR_TYPE} resources"
45-
exit 1
46-
fi
43+
echo "INFO: Starting cleanup for ${CR_TYPE} in ${CR_NAMESPACE} older than ${OLDER_THAN}"
44+
echo "INFO: Processing in batches of ${BATCH_SIZE}"
4745
48-
awk -v since=${SECONDS_BACK} '{
49-
# parsing the completionTime and converting it to epoch
50-
# so we can compute the precise CRs that should be deleted
51-
gsub("[:\\-TZ]", " ", $3)
52-
t=mktime($3)
53-
completionTime=strftime("%s", t)
54-
#
55-
# completionTime should be smaller than `since` seconds so it can be deleted
56-
if(since > completionTime) {
57-
args="%s:%s\n"
58-
printf(args, $1, $2)
59-
}
60-
}' $KUBECTL_OUTPUT > $PRUNING_CRS_FILE
46+
# Process resources in batches using pagination
47+
CONTINUE=""
48+
while true; do
49+
# Build kubectl command with limit and continue token
50+
CMD="kubectl get ${CR_TYPE} -n ${CR_NAMESPACE} -l ${LABELS} --limit=${BATCH_SIZE}"
51+
if [ -n "$CONTINUE" ]; then
52+
CMD="$CMD --continue=${CONTINUE}"
53+
fi
54+
CMD="$CMD -o json"
55+
56+
# Fetch batch
57+
BATCH_OUTPUT=$(mktemp -p /var/tmp)
58+
if ! eval "$CMD" > "$BATCH_OUTPUT" 2>&1; then
59+
echo "ERROR: failed to list ${CR_TYPE} resources"
60+
rm -f "$BATCH_OUTPUT"
61+
exit 1
62+
fi
63+
64+
# Process each item in the batch individually to minimize memory
65+
jq -r '.items[] | select(.status.completionTime != null) | "\(.metadata.name):\(.metadata.namespace):\(.status.completionTime)"' "$BATCH_OUTPUT" | \
66+
while IFS=: read -r cr_name cr_namespace completion_time; do
67+
# Convert completionTime to epoch
68+
completion_epoch=$(date -d "${completion_time}" +%s 2>/dev/null || echo "0")
6169
62-
while IFS= read -r line; do
63-
cr=${line%:*}
64-
namespace=${line#*:}
65-
if output=$(kubectl delete "${CR_TYPE}" "${cr}" -n "${namespace}" --wait=false --timeout=30s 2>&1); then
66-
echo "INFO: namespace=${namespace} ${output}"
67-
else
68-
echo "ERROR: namespace=${namespace} ${output}"
70+
# Check if old enough to delete
71+
if [ "$SECONDS_BACK" -gt "$completion_epoch" ]; then
72+
if kubectl delete "${CR_TYPE}" "${cr_name}" -n "${cr_namespace}" --wait=false --timeout=30s 2>&1; then
73+
echo "INFO: Deleted ${cr_name} in ${cr_namespace} (completed: ${completion_time})"
74+
DELETED_COUNT=$((DELETED_COUNT + 1))
75+
else
76+
echo "ERROR: Failed to delete ${cr_name} in ${cr_namespace}"
77+
ERROR_COUNT=$((ERROR_COUNT + 1))
78+
fi
79+
fi
80+
done
81+
82+
# Check for continuation token
83+
CONTINUE=$(jq -r '.metadata.continue // empty' "$BATCH_OUTPUT")
84+
rm -f "$BATCH_OUTPUT"
85+
86+
# Exit if no more pages
87+
if [ -z "$CONTINUE" ]; then
88+
break
6989
fi
70-
done < $PRUNING_CRS_FILE
90+
91+
echo "INFO: Fetching next batch (continue token present)"
92+
done
93+
94+
echo "INFO: Cleanup complete. Deleted: ${DELETED_COUNT}, Errors: ${ERROR_COUNT}"
7195
imagePullPolicy: IfNotPresent
7296
image: quay.io/konflux-ci/release-service-utils:9089cafbf36bb889b4b73d8c2965613810f13736
7397
volumeMounts:
@@ -76,10 +100,10 @@ spec:
76100
resources:
77101
limits:
78102
cpu: 200m
79-
memory: 256Mi
103+
memory: 512Mi
80104
requests:
81105
cpu: 200m
82-
memory: 256Mi
106+
memory: 512Mi
83107
securityContext:
84108
allowPrivilegeEscalation: false
85109
capabilities:

0 commit comments

Comments
 (0)