fix(KONFLUX-13012): clean up orphaned IRs before creating new ones

When the managed task is retried (e.g. due to timeout), the previous
InternalRequest and its PipelineRun on the internal cluster continue
running. The retry creates a new InternalRequest, leading to concurrent
operations that cause duplicate Pulp pushes and CGW "Record already
present" errors.

Before creating a new InternalRequest, the script now deletes any
existing InternalRequests with the same PipelineRun UID label. Combined
with the finalizer change in internal-services(PR 709), this ensures
the associated PipelineRun is cancelled before the new one starts.

Assisted-by: Cursor AI
Signed-off-by: Scott Wickersham <swickers@redhat.com>

Author: swickersh

utils/internal-request

@@ -264,6 +264,37 @@ if [[ -n "${SERVICEACCOUNT}" ]]; then
         '.spec.serviceAccount = $serviceAccount' <<< $PAYLOAD)
 fi
 
+# Clean up any existing InternalRequests from prior attempts within the same
+# PipelineRun. This handles cases where a managed task retry or timeout left
+# an orphaned InternalRequest whose PipelineRun may still be running on the
+# internal cluster. The internal-services controller will cancel the associated
+# PipelineRun when it detects the InternalRequest deletion.
+PIPELINERUN_UID_LABEL="internal-services.appstudio.openshift.io/pipelinerun-uid"
+PIPELINERUN_UID_VALUE=""
+for label in "${LABELS[@]}"; do
+    KEY=$(echo "$label" | cut -d'=' -f1)
+    VALUE=$(echo "$label" | cut -d'=' -f2-)
+    if [ "$KEY" = "$PIPELINERUN_UID_LABEL" ]; then
+        PIPELINERUN_UID_VALUE="$VALUE"
+        break
+    fi
+done
+
+if [ -n "$PIPELINERUN_UID_VALUE" ]; then
+    EXISTING_IRS=$(kubectl get internalrequest -l "${PIPELINERUN_UID_LABEL}=${PIPELINERUN_UID_VALUE}" \
+        --no-headers -o custom-columns=":metadata.name" 2>/dev/null || true)
+
+    if [ -n "$EXISTING_IRS" ]; then
+        echo "Found existing InternalRequests from prior attempts. Cleaning up..."
+        for IR_NAME in $EXISTING_IRS; do
+            echo "Deleting InternalRequest ${IR_NAME}..."
+            kubectl delete internalrequest "$IR_NAME" 2>/dev/null || true
+        done
+        echo "Cleanup complete. Waiting for associated PipelineRuns to be cancelled..."
+        sleep 5
+    fi
+fi
+
 # Create InternalRequest using kubectl
 RESOURCE=$(echo "$PAYLOAD" | kubectl create -f - -o json)
 INTERNAL_REQUEST_NAME=$(echo "$RESOURCE" | jq -r '.metadata.name')

utils/tests/test_internal_request.py

@@ -0,0 +1,130 @@
+"""Tests for the internal-request utility script."""
+
+from __future__ import annotations
+
+import os
+import stat
+import subprocess
+from pathlib import Path
+
+
+SCRIPT_PATH = Path(__file__).resolve().parents[1] / "internal-request"
+
+
+def _write_executable(path: Path, content: str) -> None:
+    path.write_text(content, encoding="utf-8")
+    path.chmod(path.stat().st_mode | stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH)
+
+
+def _run_internal_request(
+    tmp_path: Path, labels: list[str] | None = None, existing_irs: str = ""
+):
+    bin_dir = tmp_path / "bin"
+    bin_dir.mkdir()
+    kubectl_log = tmp_path / "kubectl.log"
+    sleep_log = tmp_path / "sleep.log"
+
+    _write_executable(
+        bin_dir / "kubectl",
+        """#!/usr/bin/env bash
+set -euo pipefail
+echo "$*" >> "$KUBECTL_LOG"
+if [[ "${1:-}" == "get" && "${2:-}" == "internalrequest" ]]; then
+    printf "%s\\n" "${MOCK_EXISTING_IRS:-}"
+    exit 0
+fi
+if [[ "${1:-}" == "delete" && "${2:-}" == "internalrequest" ]]; then
+    exit 0
+fi
+if [[ "${1:-}" == "create" ]]; then
+    cat >/dev/null
+    echo '{"metadata":{"name":"new-ir"}}'
+    exit 0
+fi
+exit 1
+""",
+    )
+
+    _write_executable(
+        bin_dir / "sleep",
+        """#!/usr/bin/env bash
+set -euo pipefail
+echo "$*" >> "$SLEEP_LOG"
+""",
+    )
+
+    cmd = [
+        "bash",
+        str(SCRIPT_PATH),
+        "--pipeline",
+        "test-pipeline",
+        "-p",
+        "taskGitUrl=https://github.com/konflux-ci/release-service-catalog",
+        "-p",
+        "taskGitRevision=main",
+        "-s",
+        "false",
+    ]
+    for label in labels or []:
+        cmd.extend(["-l", label])
+
+    env = os.environ.copy()
+    env["PATH"] = f"{bin_dir}:{env['PATH']}"
+    env["KUBECTL_LOG"] = str(kubectl_log)
+    env["SLEEP_LOG"] = str(sleep_log)
+    env["MOCK_EXISTING_IRS"] = existing_irs
+
+    result = subprocess.run(cmd, capture_output=True, text=True, check=False, env=env)
+
+    kubectl_calls = kubectl_log.read_text(encoding="utf-8").splitlines()
+    sleep_calls = (
+        sleep_log.read_text(encoding="utf-8").splitlines() if sleep_log.exists() else []
+    )
+    return result, kubectl_calls, sleep_calls
+
+
+def test_internal_request_cleans_up_existing_requests(tmp_path):
+    result, kubectl_calls, sleep_calls = _run_internal_request(
+        tmp_path=tmp_path,
+        labels=["internal-services.appstudio.openshift.io/pipelinerun-uid=uid-123"],
+        existing_irs="old-ir-1\nold-ir-2",
+    )
+
+    assert result.returncode == 0, result.stderr
+    selector = (
+        "get internalrequest -l "
+        "internal-services.appstudio.openshift.io/pipelinerun-uid=uid-123"
+    )
+    assert any(call.startswith(selector) for call in kubectl_calls)
+    assert "delete internalrequest old-ir-1" in kubectl_calls
+    assert "delete internalrequest old-ir-2" in kubectl_calls
+    assert "5" in sleep_calls
+    assert any(call.startswith("create -f - -o json") for call in kubectl_calls)
+
+
+def test_internal_request_skips_cleanup_when_no_existing_requests(tmp_path):
+    result, kubectl_calls, sleep_calls = _run_internal_request(
+        tmp_path=tmp_path,
+        labels=["internal-services.appstudio.openshift.io/pipelinerun-uid=uid-123"],
+        existing_irs="",
+    )
+
+    assert result.returncode == 0, result.stderr
+    assert any(call.startswith("get internalrequest -l") for call in kubectl_calls)
+    assert not any(call.startswith("delete internalrequest") for call in kubectl_calls)
+    assert sleep_calls == []
+    assert any(call.startswith("create -f - -o json") for call in kubectl_calls)
+
+
+def test_internal_request_skips_cleanup_without_pipelinerun_uid_label(tmp_path):
+    result, kubectl_calls, sleep_calls = _run_internal_request(
+        tmp_path=tmp_path,
+        labels=["some-other-label=foo"],
+        existing_irs="old-ir-1",
+    )
+
+    assert result.returncode == 0, result.stderr
+    assert not any(call.startswith("get internalrequest -l") for call in kubectl_calls)
+    assert not any(call.startswith("delete internalrequest") for call in kubectl_calls)
+    assert sleep_calls == []
+    assert any(call.startswith("create -f - -o json") for call in kubectl_calls)