fix(KONFLUX-13012): clean up orphaned IRs before creating new ones

utils/internal-request

@@ -48,7 +48,7 @@
 # Note: This script is intended to be used with a specific Kubernetes API
 # that includes the 'InternalRequest' resource type.
 
-set -e
+set -eo pipefail
 
 # Set defaults
 SYNC=true
@@ -258,12 +258,64 @@ if [[ -n ${LABELS[@]} ]]; then
         --argjson labels "$LABEL_JSON" \
         '.metadata.labels += $labels' <<< $PAYLOAD)
 fi
+# Always stamp the pipeline name onto the IR so the cleanup selector can scope
+# deletions to this specific pipeline, preventing accidental removal of IRs
+# created by other tasks running in parallel within the same PipelineRun.
+PIPELINE_NAME_LABEL="internal-services.appstudio.openshift.io/pipeline-name"
+PAYLOAD=$(jq \
+    --arg key "$PIPELINE_NAME_LABEL" \
+    --arg value "$PIPELINE" \
+    '.metadata.labels += {($key): $value}' <<< $PAYLOAD)
 if [[ -n "${SERVICEACCOUNT}" ]]; then
     PAYLOAD=$(jq \
         --arg serviceAccount "$SERVICEACCOUNT" \
         '.spec.serviceAccount = $serviceAccount' <<< $PAYLOAD)
 fi
 
+# Clean up any existing InternalRequests from prior attempts within the same
+# PipelineRun. This handles cases where a managed task retry or timeout left
+# an orphaned InternalRequest whose PipelineRun may still be running on the
+# internal cluster. The internal-services controller will cancel the associated
+# PipelineRun when it detects the InternalRequest deletion.
+#
+# The selector uses both pipelinerun-uid (supplied by the caller) and the
+# pipeline-name that was auto-stamped on the IR above. Combining them scopes
+# the cleanup to only IRs from this specific pipeline invocation, so parallel
+# tasks in the same PipelineRun that call different pipelines are never
+# accidentally deleted.
+PIPELINERUN_UID_LABEL="internal-services.appstudio.openshift.io/pipelinerun-uid"
+PIPELINERUN_UID_VALUE=""
+for label in "${LABELS[@]}"; do
+    KEY=$(echo "$label" | cut -d'=' -f1)
+    VALUE=$(echo "$label" | cut -d'=' -f2-)
+    if [ "$KEY" = "$PIPELINERUN_UID_LABEL" ]; then
+        PIPELINERUN_UID_VALUE="$VALUE"
+        break
+    fi
+done
+
+if [ -n "$PIPELINERUN_UID_VALUE" ]; then
+    LABEL_SELECTOR="${PIPELINERUN_UID_LABEL}=${PIPELINERUN_UID_VALUE},${PIPELINE_NAME_LABEL}=${PIPELINE}"
+    EXISTING_IRS=$(kubectl get internalrequest \
+        -l "$LABEL_SELECTOR" \
+        -o json | jq -r '.items[].metadata.name')
+
+    if [ -n "$EXISTING_IRS" ]; then
+        echo "Found existing InternalRequests from prior attempts. Cleaning up..."
+        while IFS= read -r IR_NAME; do
+            echo "Deleting InternalRequest ${IR_NAME}..."
+            kubectl delete internalrequest "$IR_NAME" --wait=true --timeout=60s
+        done <<< "$EXISTING_IRS"
+        # Sleep to give the internal-services controller time to react to the IR
+        # deletion and cancel the associated PipelineRun before we create a new one.
+        # This is a best-effort heuristic; --wait=true above ensures the IR is gone
+        # from the API server, but cancellation propagation to the internal cluster
+        # is asynchronous and its duration is not observable from here.
+        echo "Cleanup complete. Waiting for PipelineRun cancellation to propagate..."
+        sleep 5
+    fi
+fi
+
 # Create InternalRequest using kubectl
 RESOURCE=$(echo "$PAYLOAD" | kubectl create -f - -o json)
 INTERNAL_REQUEST_NAME=$(echo "$RESOURCE" | jq -r '.metadata.name')

utils/tests/test_internal_request.py

@@ -0,0 +1,184 @@
+"""Tests for the internal-request utility script."""
+
+from __future__ import annotations
+
+import os
+import stat
+import subprocess
+from pathlib import Path
+
+SCRIPT_PATH = Path(__file__).resolve().parents[1] / "internal-request"
+
+
+def _write_executable(path: Path, content: str) -> None:
+    path.write_text(content, encoding="utf-8")
+    path.chmod(path.stat().st_mode | stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH)
+
+
+def _run_internal_request(
+    tmp_path: Path,
+    labels: list[str] | None = None,
+    existing_irs: list[str] | None = None,
+    delete_fails: bool = False,
+):
+    bin_dir = tmp_path / "bin"
+    bin_dir.mkdir()
+    kubectl_log = tmp_path / "kubectl.log"
+    sleep_log = tmp_path / "sleep.log"
+
+    # Build a JSON array of IR objects for the mock get response.
+    ir_names = existing_irs or []
+    items_json = ", ".join(f'{{"metadata": {{"name": "{name}"}}}}' for name in ir_names)
+    mock_get_json = f'{{"items": [{items_json}]}}'
+
+    delete_exit = "1" if delete_fails else "0"
+
+    _write_executable(
+        bin_dir / "kubectl",
+        f"""#!/usr/bin/env bash
+set -euo pipefail
+echo "$*" >> "$KUBECTL_LOG"
+if [[ "${{1:-}}" == "get" && "${{2:-}}" == "internalrequest" ]]; then
+    echo '{mock_get_json}'
+    exit 0
+fi
+if [[ "${{1:-}}" == "delete" && "${{2:-}}" == "internalrequest" ]]; then
+    exit {delete_exit}
+fi
+if [[ "${{1:-}}" == "create" ]]; then
+    cat >/dev/null
+    echo '{{"metadata":{{"name":"new-ir"}}}}'
+    exit 0
+fi
+exit 1
+""",
+    )
+
+    _write_executable(
+        bin_dir / "sleep",
+        """#!/usr/bin/env bash
+set -euo pipefail
+echo "$*" >> "$SLEEP_LOG"
+""",
+    )
+
+    cmd = [
+        "bash",
+        str(SCRIPT_PATH),
+        "--pipeline",
+        "test-pipeline",
+        "-p",
+        "taskGitUrl=https://github.com/konflux-ci/release-service-catalog",
+        "-p",
+        "taskGitRevision=main",
+        "-s",
+        "false",
+    ]
+    for label in labels or []:
+        cmd.extend(["-l", label])
+
+    env = os.environ.copy()
+    env["PATH"] = f"{bin_dir}:{env['PATH']}"
+    env["KUBECTL_LOG"] = str(kubectl_log)
+    env["SLEEP_LOG"] = str(sleep_log)
+
+    result = subprocess.run(cmd, capture_output=True, text=True, check=False, env=env)
+
+    kubectl_calls = kubectl_log.read_text(encoding="utf-8").splitlines()
+    sleep_calls = (
+        sleep_log.read_text(encoding="utf-8").splitlines() if sleep_log.exists() else []
+    )
+    return result, kubectl_calls, sleep_calls
+
+
+PIPELINERUN_UID_LABEL = "internal-services.appstudio.openshift.io/pipelinerun-uid"
+PIPELINE_NAME_LABEL = "internal-services.appstudio.openshift.io/pipeline-name"
+# The test helper always passes --pipeline test-pipeline
+TEST_PIPELINE = "test-pipeline"
+
+
+def test_internal_request_cleans_up_existing_requests(tmp_path):
+    """Delete existing IRs and sleep before creating a new one."""
+    result, kubectl_calls, sleep_calls = _run_internal_request(
+        tmp_path=tmp_path,
+        labels=[f"{PIPELINERUN_UID_LABEL}=uid-123"],
+        existing_irs=["old-ir-1", "old-ir-2"],
+    )
+
+    assert result.returncode == 0, result.stderr
+    selector = (
+        f"get internalrequest -l "
+        f"{PIPELINERUN_UID_LABEL}=uid-123,{PIPELINE_NAME_LABEL}={TEST_PIPELINE}"
+    )
+    assert any(call.startswith(selector) for call in kubectl_calls)
+    assert any(c.startswith("delete internalrequest old-ir-1") for c in kubectl_calls)
+    assert any(c.startswith("delete internalrequest old-ir-2") for c in kubectl_calls)
+    assert "5" in sleep_calls
+    assert any(call.startswith("create -f - -o json") for call in kubectl_calls)
+
+
+def test_internal_request_skips_cleanup_when_no_existing_requests(tmp_path):
+    """Skip delete and sleep when no existing IRs match the selector."""
+    result, kubectl_calls, sleep_calls = _run_internal_request(
+        tmp_path=tmp_path,
+        labels=[f"{PIPELINERUN_UID_LABEL}=uid-123"],
+        existing_irs=[],
+    )
+
+    assert result.returncode == 0, result.stderr
+    assert any(call.startswith("get internalrequest -l") for call in kubectl_calls)
+    assert not any(call.startswith("delete internalrequest") for call in kubectl_calls)
+    assert sleep_calls == []
+    assert any(call.startswith("create -f - -o json") for call in kubectl_calls)
+
+
+def test_internal_request_skips_cleanup_without_pipelinerun_uid_label(tmp_path):
+    """Skip cleanup when the pipelinerun-uid label is absent from the IR labels."""
+    result, kubectl_calls, sleep_calls = _run_internal_request(
+        tmp_path=tmp_path,
+        labels=["some-other-label=foo"],
+        existing_irs=["old-ir-1"],
+    )
+
+    assert result.returncode == 0, result.stderr
+    assert not any(call.startswith("get internalrequest -l") for call in kubectl_calls)
+    assert not any(call.startswith("delete internalrequest") for call in kubectl_calls)
+    assert sleep_calls == []
+    assert any(call.startswith("create -f - -o json") for call in kubectl_calls)
+
+
+def test_internal_request_does_not_delete_parallel_task_irs(tmp_path):
+    """Include pipeline-name in the selector to avoid matching IRs from other parallel tasks.
+
+    IRs that call a different --pipeline must never be deleted.
+    """
+    result, kubectl_calls, sleep_calls = _run_internal_request(
+        tmp_path=tmp_path,
+        labels=[f"{PIPELINERUN_UID_LABEL}=uid-123"],
+        existing_irs=["old-ir-1"],
+    )
+
+    assert result.returncode == 0, result.stderr
+    selector = (
+        f"get internalrequest -l "
+        f"{PIPELINERUN_UID_LABEL}=uid-123,{PIPELINE_NAME_LABEL}={TEST_PIPELINE}"
+    )
+    assert any(call.startswith(selector) for call in kubectl_calls)
+    assert not any(
+        call == f"get internalrequest -l {PIPELINERUN_UID_LABEL}=uid-123 -o json"
+        for call in kubectl_calls
+    ), "Selector must not use pipelinerun-uid alone"
+
+
+def test_internal_request_fails_when_delete_fails(tmp_path):
+    """Exit non-zero and skip IR creation when deletion of an existing IR fails."""
+    result, kubectl_calls, sleep_calls = _run_internal_request(
+        tmp_path=tmp_path,
+        labels=[f"{PIPELINERUN_UID_LABEL}=uid-123"],
+        existing_irs=["old-ir-1"],
+        delete_fails=True,
+    )
+
+    assert result.returncode != 0, "Expected non-zero exit when delete fails"
+    assert any(c.startswith("delete internalrequest old-ir-1") for c in kubectl_calls)
+    assert not any(call.startswith("create -f - -o json") for call in kubectl_calls)