test_tekton_tasks.sh

#!/bin/bash

# This script will run task tests for all task directories
# provided either via TEST_ITEMS env var, or as arguments
# when running the script.
#
# Requirements:
# - Connection to a running k8s cluster (e.g. kind)
# - Tekton installed on the cluster
# - tkn installed
#

# yield empty strings for unmatched patterns
shopt -s nullglob

WORKSPACE_TEMPLATE=${BASH_SOURCE%/*/*}/resources/workspace-template.yaml

# For tasks that use a step command/args referencing a .py file, merge test mocks
# into that step: prefer tests/mocks.yaml (render_python_task_mocks_from_yaml.py)
# else tests/mocks.sh (legacy: body after shebang). Task-specific hooks still run after.
apply_python_command_mocks_merge() {
  local task_copy="$1"
  local tests_dir="$2"
  local mocks_sh="${tests_dir}/mocks.sh"
  local mocks_yaml="${tests_dir}/mocks.yaml"
  local step_count i merged_tmp w joined _tt_scripts_dir
  local -a command_from_task args_from_task entrypoint_argv

  [[ -f "$mocks_yaml" ]] || [[ -f "$mocks_sh" ]] || return 0

  _tt_scripts_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"

  step_count=$(yq '.spec.steps | length' "$task_copy")

  for ((i = 0; i < step_count; i++)); do
    command_from_task=()
    args_from_task=()
    mapfile -t command_from_task < <(
      yq -r ".spec.steps[$i].command // [] | .[]?" "$task_copy" 2>/dev/null
    )
    [[ ${#command_from_task[@]} -gt 0 ]] || continue

    mapfile -t args_from_task < <(
      yq -r ".spec.steps[$i].args // [] | .[]?" "$task_copy" 2>/dev/null
    )

    entrypoint_argv=("${command_from_task[@]}" "${args_from_task[@]}")
    joined=""
    for w in "${entrypoint_argv[@]}"; do
      joined+=" $w"
    done
    [[ "$joined" == *".py"* ]] || continue

    if [[ -f "$mocks_yaml" ]]; then
      echo "  Merging tests/mocks.yaml into step $i (Python entrypoint) for task tests"
    else
      echo "  Merging tests/mocks.sh into step $i (Python entrypoint) for task tests"
    fi

    merged_tmp=$(mktemp)
    {
      echo '#!/usr/bin/env bash'
      echo "TASK_ENTRYPOINT=("
      for w in "${entrypoint_argv[@]}"; do
        # Do not use printf %q for Tekton placeholders: single-quoted %q output
        # prevents Tekton from rewriting $(params.*) inside spec.steps[].script.
        if [[ "$w" == *'$('* ]]; then
          printf '  "%s"\n' "$w"
        else
          printf '  %q\n' "$w"
        fi
      done
      echo ")"
      if [[ -f "$mocks_yaml" ]]; then
        python3 "${_tt_scripts_dir}/render_python_task_mocks_from_yaml.py" "$tests_dir"
      else
        tail -n +2 "$mocks_sh"
      fi
    } >"$merged_tmp"

    yq -i ".spec.steps[$i].script = load_str(\"$merged_tmp\")" "$task_copy"
    rm -f "$merged_tmp"

    yq -i "del(.spec.steps[$i].command)" "$task_copy"
    yq -i "del(.spec.steps[$i].args)" "$task_copy"
  done
}

# When mocks.yaml sets hostname on an http_json service, map that hostname to the
# mock bind address via PipelineRun podTemplate.hostAliases (Task CRDs cannot set this).
apply_mock_host_aliases_to_pipelinerun_spec() {
  local tests_dir="$1"
  local test_path="$2"
  local task_name="$3"
  local pipelinerun_json="$4"
  local mocks_yaml="${tests_dir}/mocks.yaml"
  local mock_host mock_bind pipeline_task

  [[ -f "$mocks_yaml" ]] || { echo "$pipelinerun_json"; return 0; }

  mock_host=$(yq -r '.services[0].hostname // ""' "$mocks_yaml")
  mock_bind=$(yq -r '.services[0].bind // "127.0.0.1"' "$mocks_yaml")
  if [[ -z "$mock_host" || "$mock_host" == "null" || "$mock_host" == "$mock_bind" ]]; then
    echo "$pipelinerun_json"
    return 0
  fi

  pipeline_task=$(
    yq -r ".spec.tasks[] | select(.taskRef.name == \"${task_name}\") | .name" \
      "$test_path"
  )
  if [[ -z "$pipeline_task" || "$pipeline_task" == "null" ]]; then
    echo "  Warning: mocks.yaml hostname set but no taskRef.name=${task_name} in ${test_path}" >&2
    echo "$pipelinerun_json"
    return 0
  fi

  echo "  Applying hostAliases ${mock_host} -> ${mock_bind} for pipeline task ${pipeline_task}" >&2
  jq \
    --arg task "$pipeline_task" \
    --arg ip "$mock_bind" \
    --arg host "$mock_host" \
    '.spec.taskRunSpecs = [{
      pipelineTaskName: $task,
      podTemplate: {
        hostAliases: [{
          ip: $ip,
          hostnames: [$host]
        }]
      }
    }]' <<<"$pipelinerun_json"
}

needs_mock_host_aliases() {
  local tests_dir="$1"
  local mocks_yaml="${tests_dir}/mocks.yaml"
  local mock_host mock_bind

  [[ -f "$mocks_yaml" ]] || return 1
  mock_host=$(yq -r '.services[0].hostname // ""' "$mocks_yaml")
  mock_bind=$(yq -r '.services[0].bind // "127.0.0.1"' "$mocks_yaml")
  [[ -n "$mock_host" && "$mock_host" != "null" && "$mock_host" != "$mock_bind" ]]
}

show_help() {
  echo "Usage: $0 [--remove-compute-resources] [--no-cleanup] [item1] [item2] [...]"
  echo
  echo Flags:
  echo "  --help: Show this help message"
  echo "  --no-cleanup: Keeps test resources after each test"
  echo "  --remove-compute-resources: Remove compute resources from tasks"
  echo
  echo "Items can be task directories or paths to task test yaml files"
  echo "(useful when working on a single test). They can be supplied"
  echo "either as arguments or via the TEST_ITEMS environment variable."
  echo
  echo "Examples:"
  echo "  $0 --remove-compute-resources tasks/apply-mapping"
  echo "  $0 tasks/apply-mapping/tests/test-apply-mapping.yaml"
  exit 1
}

REMOVE_COMPUTE_RESOURCES=false
NO_CLEANUP=false
CLI_TEST_ITEMS=""

while [[ $# -gt 0 ]]; do
  case "$1" in
    --remove-compute-resources)
      REMOVE_COMPUTE_RESOURCES=true
      shift
      ;;
    --no-cleanup)
      NO_CLEANUP=true
      shift
      ;;
    --help)
      show_help
      ;;
    --*)
      show_help
      ;;
    *)
      CLI_TEST_ITEMS+="$1 "
      shift
      ;;
  esac
done

if [[ -n "$CLI_TEST_ITEMS" ]]; then
  TEST_ITEMS="${CLI_TEST_ITEMS% }"  # Remove trailing space
else
  TEST_ITEMS="${TEST_ITEMS:-}"      # Use env var or empty string
fi

if [ -z "${TEST_ITEMS}" ]
then
  show_help
fi

if [ -z "${USE_TRUSTED_ARTIFACTS}" ]
then
  echo "Defaulting to PVC based workspaces..."
  # empty is needed since trusted-artifacts needs a non-empty storage
  # parameter in order to reach the skipping logic
  export TRUSTED_ARTIFACT_OCI_STORAGE="empty"
else

  echo "Using Trusted Artifacts for workspaces..."
  export TRUSTED_ARTIFACT_OCI_STORAGE=registry-service.kind-registry/trusted-artifacts
  export TRUSTED_ARTIFACT_OCI_DOCKER_CONFIG_JSON_PATH=${DOCKER_CONFIG_JSON}

  echo "Using docker config stored in ${DOCKER_CONFIG_JSON}"
  kubectl create secret generic docker-config \
    --from-file=.dockerconfigjson="${TRUSTED_ARTIFACT_OCI_DOCKER_CONFIG_JSON_PATH}" \
    --type=kubernetes.io/dockerconfigjson --dry-run=client -o yaml | kubectl apply -f -
  kubectl patch serviceaccount default -p \
    '{"imagePullSecrets": [{"name": "docker-config"}], "secrets": [{"name": "docker-config"}]}'

fi

# Check that all directories exist. If not, fail
for ITEM in $TEST_ITEMS
do
  if [[ "$ITEM" == *tests/test-*.yaml && -f "$ITEM" ]]; then
    true
  elif [[ -d "$ITEM" ]]; then
    true
  else
    echo "Error: Invalid file or directory: $ITEM"
    exit 1
  fi
done

# install step actions
echo "Installing StepActions"
SCRIPT_DIR=$(cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd)
STEPACTION_ROOT=${SCRIPT_DIR}/../../stepactions
stepActionFiles=$(find $STEPACTION_ROOT -maxdepth 2 -name "*.yaml")

for stepAction in ${stepActionFiles};
do
  name=$(yq ".metadata.name" "${stepAction}")
  echo "  Installing StepAction $name"
  kubectl apply -f $stepAction
done

# Clean up leftover resources from previous test runs to prevent cluster exhaustion
echo "Cleaning up old test resources..."
kubectl delete pipelineruns -l tekton.dev/pipeline --field-selector=status.conditions[0].status!=Unknown || true
kubectl get taskruns -o json | jq -r '.items[] | select(.status.completionTime != null) | select((now - (.status.completionTime | fromdateiso8601)) > 3600) | .metadata.name' | xargs -r kubectl delete taskrun --ignore-not-found=true || true
# Clean up all completed and failed pods to free disk space (this also cleans up their emptyDir volumes)
kubectl delete pods --field-selector=status.phase==Succeeded || true
kubectl delete pods --field-selector=status.phase==Failed || true
# Also clean up pods in other terminal states that might be stuck
kubectl delete pods --field-selector=status.phase==Unknown || true
# Delete very old pods regardless of status (older than 1 hour)
kubectl get pods -o json | jq -r '.items[] | select(.metadata.creationTimestamp != null) | select((now - (.metadata.creationTimestamp | fromdateiso8601)) > 3600) | .metadata.name' | xargs -r kubectl delete pod --ignore-not-found=true || true

# Clean up unused container images to free disk space
echo "Cleaning up unused container images..."
# Get the kind cluster node name
KIND_NODE=$(kubectl get nodes -o jsonpath='{.items[0].metadata.name}')
if [ ! -z "$KIND_NODE" ]; then
  # For kind clusters, exec into the node to prune images
  docker exec "$KIND_NODE" crictl rmi --prune || true

  # Check disk space after initial cleanup
  echo "Disk space after initial cleanup:"
  docker exec "$KIND_NODE" df -h / | grep -E '(Filesystem|/$)'
fi

echo "Initial cleanup complete"

# Create global trusted-ca ConfigMap for all tests
# This prevents race conditions where tests delete/recreate the same ConfigMap
echo "Creating global trusted-ca ConfigMap..."
kubectl delete configmap trusted-ca --ignore-not-found || true
kubectl create configmap trusted-ca --from-literal=ca-bundle.crt=testcert || true
echo "Global ConfigMap created"

# Initialize test counter for periodic cleanup
TEST_COUNTER=0

# Create global trusted-ca ConfigMap for all tests
# This prevents race conditions where tests delete/recreate the same ConfigMap
echo "Creating global trusted-ca ConfigMap..."
kubectl delete configmap trusted-ca --ignore-not-found || true
kubectl create configmap trusted-ca --from-literal=ca-bundle.crt=testcert || true
echo "Global ConfigMap created"

for ITEM in $TEST_ITEMS
do
  echo Task item: $ITEM
  TASK_NAME=$(echo $ITEM | cut -d '/' -f 3)
  TEST_COUNTER=$((TEST_COUNTER + 1))
  TASK_DIR=$(echo $ITEM | cut -d '/' -f -3)
  echo "  Task name: $TASK_NAME"

  TASK_PATH=${TASK_DIR}/${TASK_NAME}.yaml
  if [ ! -f $TASK_PATH ]
  then
    echo Error: Task file does not exist: $TASK_PATH
    exit 1
  fi

  TESTS_DIR=${TASK_DIR}/tests
  if [ ! -d $TESTS_DIR ]
  then
    echo Error: tests dir does not exist: $TESTS_DIR
    exit 1
  fi

  if [[ "$ITEM" == *tests/test-*.yaml ]]; then
    TEST_PATHS=($ITEM)
  else
    TEST_PATHS=($TESTS_DIR/test*.yaml)
  fi
  if [ ${#TEST_PATHS[@]} -eq 0 ]
  then
    echo "  Warning: No tests. Skipping..."
    continue
  fi

  # Use a copy of the task file to prevent modifying to original task file
  TASK_COPY=$(mktemp)
  cp "$TASK_PATH" "$TASK_COPY"

  apply_python_command_mocks_merge "$TASK_COPY" "$TESTS_DIR"

  if [ -f ${TESTS_DIR}/pre-apply-task-hook.sh ]
  then
    echo Found pre-apply-task-hook.sh file in dir: $TESTS_DIR. Executing...
    ${TESTS_DIR}/pre-apply-task-hook.sh "$TASK_COPY"
  fi

  # Update stepaction resolvers
  # - we want to remove the git resolver params so we can use the StepAction on cluster
  echo "Updating StepAction resolvers"
  for stepAction in ${stepActionFiles};
  do
    name=$(yq ".metadata.name" "${stepAction}")
    echo "  Update resolver for $name"
    yq -i "(.spec.steps[] | select(.name == \"$name\") | .ref) = {\"name\": \"$name\"}" $TASK_COPY
  done

  if [[ "$REMOVE_COMPUTE_RESOURCES" == "true" ]]; then
    echo "Removing compute resources from task $TASK_NAME"
    yq -i 'del(.spec.steps[].computeResources)' "$TASK_COPY"
  fi

  echo "  Installing task"
  kubectl apply -f "$TASK_COPY"

  if [ -z "${USE_TRUSTED_ARTIFACTS}" ]; then
    workSpaceParams="volumeClaimTemplateFile=$WORKSPACE_TEMPLATE"
    dataDir=/workspace/data
  else
    workSpaceParams="emptyDir="
    # to avoid tar extraction errors, we need to specify a subdirectory
    # inside the volume.
    dataDir=/var/workdir/release
  fi

  rm -f "$TASK_COPY"

  for TEST_PATH in ${TEST_PATHS[@]}
  do
    echo "  Installing test pipeline: $TEST_PATH"
    kubectl apply -f $TEST_PATH
    TEST_NAME=$(yq '.metadata.name' $TEST_PATH)

    # If a test is creating a trusted artifact, provide the necessary parameters to it.
    # This way, we can support testing TA-based tasks which do not produce any artifacts
    # directly. If a task requires a TA strategy, the test will also need to utilize the
    # strategy to run successfully.
    # If a test doesn't use the parameters, it is considered a PVC-only test and should
    # continue to work.
    ociStorageParamCheck=$(yq '(.spec.params[] | select(.name == "ociStorage"))' "$TEST_PATH")
    ociStorageParam=""
    if [ ! -z "${ociStorageParamCheck}" ]; then
      ociStorageParam="-p ociStorage=${TRUSTED_ARTIFACT_OCI_STORAGE}"
    fi
    dataDirParamCheck=$(yq '(.spec.params[] | select(.name == "dataDir"))' "$TEST_PATH")
    dataDirParam=""
    if [ ! -z "${dataDirParamCheck}" ]; then
      dataDirParam="-p dataDir=${dataDir}"
    fi

    # Sometimes the pipeline is not available immediately
    while ! kubectl get pipeline $TEST_NAME > /dev/null 2>&1
    do
      echo "  Pipeline $TEST_NAME not ready. Waiting 5s..."
      sleep 5
    done

    if needs_mock_host_aliases "$TESTS_DIR"; then
      PIPELINERUNJSON=$(
        tkn p start --dry-run --use-param-defaults "$TEST_NAME" \
          ${ociStorageParam} ${dataDirParam} \
          -w "name=tests-workspace,${workSpaceParams}" -o json
      )
      PIPELINERUNJSON=$(
        apply_mock_host_aliases_to_pipelinerun_spec \
          "$TESTS_DIR" "$TEST_PATH" "$TASK_NAME" "$PIPELINERUNJSON"
      )
      CREATED_PRLINE=$(echo "$PIPELINERUNJSON" | kubectl create -f - -o json)
      PIPELINERUN=$(jq -r '.metadata.name' <<< "${CREATED_PRLINE}")
    else
      PIPELINERUNJSON=$(
        tkn p start --use-param-defaults "$TEST_NAME" \
          ${ociStorageParam} ${dataDirParam} \
          -w "name=tests-workspace,${workSpaceParams}" -o json
      )
      PIPELINERUN=$(jq -r '.metadata.name' <<< "${PIPELINERUNJSON}")
    fi

    echo "  Started pipelinerun $PIPELINERUN"
    sleep 1  # allow a second for the pr object to appear (including a status condition)
    while [ "$(kubectl get pr $PIPELINERUN -o=jsonpath='{.status.conditions[0].status}')" == "Unknown" ]
    do
      echo "  PipelineRun $PIPELINERUN in progress (status Unknown). Waiting for update..."
      sleep 5
    done
    tkn pr logs $PIPELINERUN

    PR_STATUS=$(kubectl get pr $PIPELINERUN -o=jsonpath='{.status.conditions[0].status}')

    ASSERT_TASK_FAILURE=$(yq '.metadata.annotations.test/assert-task-failure' < $TEST_PATH)
    if [ "$ASSERT_TASK_FAILURE" != "null" ]
    then
      if [ "$PR_STATUS" == "True" ]
      then
        echo "  Pipeline $TEST_NAME succeeded but was expected to fail"
        exit 1
      else
        echo "  Pipeline $TEST_NAME failed (expected). Checking that it failed in task ${ASSERT_TASK_FAILURE}..."

        # Check that the pipelinerun failed on the tested task and not somewhere else
        TASKRUN=$(kubectl get pr $PIPELINERUN -o json|jq -r "(.status.childReferences // [])[] | select(.pipelineTaskName == \"${ASSERT_TASK_FAILURE}\") | .name")
        if [ -z "$TASKRUN" ]
        then
          echo "    Unable to find task $ASSERT_TASK_FAILURE in childReferences of pipelinerun $PIPELINERUN. Pipelinerun failed earlier?"
          kubectl get pr $PIPELINERUN -o json
          exit 1
        else
          echo "    Found taskrun $TASKRUN"
        fi
        if [ $(kubectl get tr $TASKRUN -o=jsonpath='{.status.conditions[0].status}') != "False" ]
        then
          echo "    Taskrun did not fail - pipelinerun failed later on?"
          kubectl get tr $TASKRUN -o json
          exit 1
        else
          echo "    Taskrun failed as expected"
        fi

      fi
    else
      if [ "$PR_STATUS" == "True" ]
      then
        echo "  Pipelinerun $TEST_NAME succeeded"
      else
        echo "  Pipelinerun $TEST_NAME failed"

        # Debug: Get init container logs from failed TaskRuns
        echo "  === DEBUG: Fetching init container logs from failed TaskRuns ==="
        for TR in $(kubectl get pr $PIPELINERUN -o json | jq -r '.status.childReferences[]?.name // empty'); do
          TR_STATUS=$(kubectl get tr $TR -o jsonpath='{.status.conditions[0].status}' 2>/dev/null || echo "Unknown")
          if [ "$TR_STATUS" == "False" ]; then
            echo "  --- TaskRun $TR failed ---"
            POD=$(kubectl get tr $TR -o jsonpath='{.status.podName}' 2>/dev/null)
            if [ ! -z "$POD" ]; then
              echo "  Pod: $POD"
              echo "  --- Pod Events ---"
              kubectl get events --field-selector involvedObject.name=$POD 2>/dev/null || true
              echo "  --- Pod Describe (init containers) ---"
              kubectl get pod $POD -o jsonpath='{range .status.initContainerStatuses[*]}Init Container: {.name} - State: {.state}{"\n"}{end}' 2>/dev/null || true
              echo "  --- place-scripts init container logs ---"
              kubectl logs $POD -c place-scripts 2>/dev/null || echo "  (no logs available)"
              echo "  --- place-scripts command and args ---"
              kubectl get pod $POD -o json | jq '.spec.initContainers[] | select(.name == "place-scripts") | {command, args}' 2>/dev/null | head -c 5000 || true
              echo "  --- place-scripts args count and sizes ---"
              kubectl get pod $POD -o json | jq '.spec.initContainers[] | select(.name == "place-scripts") | {arg_count: (.args | length), total_size: (.args | join("") | length), individual_sizes: (.args | to_entries | map({index: .key, size: (.value | length)}))}' 2>/dev/null || true
            fi
          fi
        done
        echo "  === END DEBUG ==="

        exit 1
      fi
    fi

    if [[ "$NO_CLEANUP" != "true" ]]; then
      # Cleanup test resources to prevent cluster exhaustion when running many tests
      echo "  Cleaning up test resources..."
      kubectl delete pipelinerun $PIPELINERUN --ignore-not-found=true
      kubectl delete pipeline $TEST_NAME --ignore-not-found=true

      # Clean up old completed PipelineRuns (keep only last 5 to avoid filling the cluster)
      OLD_PRS=$(kubectl get pipelineruns -o json | jq -r '.items[] | select(.status.conditions[0].status != "Unknown") | .metadata.name' | head -n -5)
      if [ ! -z "$OLD_PRS" ]; then
        echo "$OLD_PRS" | xargs -r kubectl delete pipelinerun --ignore-not-found=true
      fi

      # Clean up completed TaskRuns to free disk space
      OLD_TRS=$(kubectl get taskruns -o json | jq -r '.items[] | select(.status.conditions[0].status != "Unknown") | .metadata.name' | head -n -10)
      if [ ! -z "$OLD_TRS" ]; then
        echo "  Cleaning up completed TaskRuns..."
        echo "$OLD_TRS" | xargs -r kubectl delete taskrun --ignore-not-found=true
      fi

      # Clean up old Pods in terminal states to free disk space (and their emptyDir volumes)
      # Keep last 10 of each status (Succeeded, Failed, Unknown)
      OLD_PODS=$(kubectl get pods -o json | jq -r '
        .items
        | group_by(.status.phase)
        | map(select(.[0].status.phase == "Succeeded" or .[0].status.phase == "Failed" or .[0].status.phase == "Unknown"))
        | map(.[:-10])
        | flatten
        | .[].metadata.name
      ')
      if [ ! -z "$OLD_PODS" ]; then
        echo "  Cleaning up old Pods in terminal states (and emptyDir volumes)..."
        echo "$OLD_PODS" | xargs -r kubectl delete pod --ignore-not-found=true
      fi
    fi
    echo
  done

  if [[ "$NO_CLEANUP" != "true" ]]; then
    # Cleanup task after all its tests complete
    echo "Cleaning up task $TASK_NAME"
    kubectl delete task $TASK_NAME --ignore-not-found=true
  fi

  KIND_NODE=$(kubectl get nodes -o jsonpath='{.items[0].metadata.name}')
  if [ ! -z "$KIND_NODE" ]; then
    if [ $((TEST_COUNTER % 20)) -eq 0 ]; then
      echo "Periodic cleanup at test #$TEST_COUNTER:"
      echo "  Disk space before cleanup:"
      docker exec "$KIND_NODE" df -h / | grep -E '/$' | awk '{print "    Used: " $3 " / " $2 " (" $5 " full), Available: " $4}'
      echo "  Pruning unused container images..."
      docker exec "$KIND_NODE" crictl rmi --prune || true
      echo "  Disk space after cleanup:"
      docker exec "$KIND_NODE" df -h / | grep -E '/$' | awk '{print "    Used: " $3 " / " $2 " (" $5 " full), Available: " $4}'
    else
      echo "Disk space at test #$TEST_COUNTER:"
      docker exec "$KIND_NODE" df -h / | grep -E '/$' | awk '{print "  Used: " $3 " / " $2 " (" $5 " full), Available: " $4}'
    fi
  fi

done