fix(ci): improve setup_aws.sh retry resilience and add download jitter (mongodb#732)

## Problem

CI setup tasks are failing intermittently during retries due to:
1. **Thundering herd**: When CI retries trigger, multiple tasks start
simultaneously and overwhelm external download endpoints
2. **Weak retry params**: `setup_aws.sh` was missed in PR mongodb#720 and had
weaker retry parameters (3s delay, 180s timeout vs 10s/600s)
3. **No error visibility**: `--fail` flag hides HTTP error responses,
making debugging difficult
4. **External API dependency**: Fetching `stable.txt` from `dl.k8s.io`
adds an extra failure point
5. **Single endpoint**: No fallback if `dl.k8s.io` is temporarily
unavailable (503 errors)

## Solution

### 1. Shared `curl_with_retry()` function
Created a shared wrapper in `scripts/funcs/install` with:
- **Random jitter (0-30s)** before downloads to spread out concurrent
requests
- **Consistent retry params**: 5 retries, 10s delay, 600s timeout
- **`--fail-with-body`** instead of `--fail` to show HTTP error
responses in logs
- **`SKIP_DOWNLOAD_JITTER=1`** env var to disable jitter for local
development

### 2. Pin kubectl version
- Added `KUBECTL_VERSION=v1.35.0` to `root-context` (like
`HELM_VERSION`)
- Eliminates external API call to `stable.txt` endpoint
- Deterministic builds with same kubectl version every time

### 3. CDN fallback for kubectl
- If `dl.k8s.io` fails, automatically tries `cdn.dl.k8s.io` directly
- Provides resilience against temporary endpoint outages (503 errors)

### 4. Remove redundant shellcheck setup
- Removed `setup_shellcheck.sh` - pre-commit already handles shellcheck
via `.pre-commit-config.yaml`

Author: nammn

.evergreen-functions.yml

@@ -128,15 +128,6 @@ functions:
       working_dir: src/github.com/mongodb/mongodb-kubernetes
       binary: scripts/evergreen/setup_kubectl.sh
 
-  setup_shellcheck:
-    command: subprocess.exec
-    type: setup
-    params:
-      working_dir: src/github.com/mongodb/mongodb-kubernetes
-      add_to_path:
-        - ${workdir}/bin
-      binary: scripts/evergreen/setup_shellcheck.sh
-
   setup_aws: &setup_aws
     command: subprocess.exec
     type: setup

.evergreen.yml

@@ -327,7 +327,6 @@ tasks:
       - func: clone
       - func: python_venv
       - func: download_kube_tools
-      - func: setup_shellcheck
       - command: github.generate_token
         params:
           expansion_name: GH_TOKEN
@@ -637,7 +636,6 @@ task_groups:
       - func: clone
       - func: python_venv
       - func: download_kube_tools
-      - func: setup_shellcheck
     tasks:
       - lint_repo
       - unit_tests_golang

scripts/dev/contexts/root-context

@@ -154,5 +154,6 @@ export RELEASE_INITIAL_VERSION="1.3.0"
 export CLUSTER_TYPE=kind
 export OPERATOR_CLUSTER_SCOPED=false
 
-# for downloading helm binaries
+# for downloading helm and kubectl binaries
 export HELM_VERSION="v3.19.4"
+export KUBECTL_VERSION="v1.35.0"

scripts/dev/setup_evg_host.sh

@@ -45,21 +45,20 @@ download_kind() {
 }
 
 download_kubectl() {
-  kubectl_version=$(curl --retry 5 -Ls https://dl.k8s.io/release/stable.txt)
-  echo "Downloading kubectl ${kubectl_version}..."
-  kubectl_version=$(echo "${kubectl_version}" | tail -n1 | tr -d '\n')
-
-  curl --retry 5 -LOs "https://dl.k8s.io/release/${kubectl_version}/bin/linux/${ARCH}/kubectl"
-  chmod +x kubectl
+  # Use pinned version (KUBECTL_VERSION from root-context)
+  # Falls back to fetching stable.txt if KUBECTL_VERSION is not set
+  local version="${KUBECTL_VERSION:-}"
+  if [[ -z "${version}" ]]; then
+    version=$(curl_with_retry -Ls https://dl.k8s.io/release/stable.txt | tail -n1 | tr -d '\n')
+  fi
+
+  download_kubectl_binary "${version}" "${ARCH}"
   sudo mv kubectl /usr/local/bin/kubectl
 }
 
 download_helm() {
-  echo "Downloading helm..."
-  curl -s -o helm.tar.gz -L "https://get.helm.sh/helm-${HELM_VERSION}-linux-${ARCH}tar.gz"
-  tar -xf helm.tar.gz 2>/dev/null
-  sudo mv linux-"${ARCH}"helm /usr/local/bin/helm
-  rm helm.tar.gz
+  download_helm_binary "${HELM_VERSION}" "${ARCH}"
+  sudo mv linux-"${ARCH}"/helm /usr/local/bin/helm
   rm -rf linux-"${ARCH}/"
 }
 

scripts/evergreen/lint_code.sh

@@ -2,14 +2,15 @@
 set -Eeou pipefail
 
 source scripts/dev/set_env_context.sh
+source scripts/funcs/install
 
 # Set required version
 required_version="v2.0.2"
 
 # Install or update golangci-lint if not installed or version is incorrect
 if ! [[ -x "$(command -v golangci-lint)" ]]; then
     echo "Installing/updating golangci-lint to version ${required_version}..."
-    curl --retry 5 --retry-delay 3 --retry-all-errors --fail --show-error --max-time 180 -sSL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b "$(go env GOPATH)"/bin "${required_version}"
+    curl_with_retry -sSL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b "$(go env GOPATH)"/bin "${required_version}"
 else
     echo "golangci-lint is already installed"
 fi

scripts/evergreen/setup_aws.sh

@@ -19,7 +19,7 @@ install_aws_cli_binary() {
     cd "${temp_dir}"
 
     echo "Downloading AWS CLI v2 for ${aws_arch}..."
-    curl --retry 5 --retry-delay 3 --retry-all-errors --fail --show-error --max-time 180 -s "https://awscli.amazonaws.com/awscli-exe-linux-${aws_arch}.zip" -o "awscliv2.zip"
+    curl_with_retry -s "https://awscli.amazonaws.com/awscli-exe-linux-${aws_arch}.zip" -o "awscliv2.zip"
 
     unzip -q awscliv2.zip
     sudo ./aws/install --update

scripts/evergreen/setup_gcloud_cli.sh

@@ -2,8 +2,9 @@
 set -Eeou pipefail
 
 source scripts/dev/set_env_context.sh
+source scripts/funcs/install
 
-curl --retry 5 --retry-delay 3 --retry-all-errors --fail --show-error --max-time 180 --silent -LO "https://dl.google.com/dl/cloudsdk/channels/rapid/downloads/google-cloud-cli-linux-x86_64.tar.gz"
+curl_with_retry --silent -LO "https://dl.google.com/dl/cloudsdk/channels/rapid/downloads/google-cloud-cli-linux-x86_64.tar.gz"
 tar xf google-cloud-cli-linux-x86_64.tar.gz -C "${workdir}"
 "${workdir}"/google-cloud-sdk/install.sh --quiet
 source "${workdir}/google-cloud-sdk/path.bash.inc"

scripts/evergreen/setup_jq.sh

@@ -12,4 +12,5 @@ source scripts/funcs/install
 jq_arch=$(detect_architecture "jq")
 echo "Detected architecture: ${jq_arch}"
 
-download_and_install_binary "${PROJECT_DIR:-${workdir}}/bin" jq "https://github.com/stedolan/jq/releases/download/jq-1.8.1/jq-linux-${jq_arch}"
+# Use jqlang/jq (canonical repo) directly to avoid redirect from stedolan/jq
+download_and_install_binary "${PROJECT_DIR:-${workdir}}/bin" jq "https://github.com/jqlang/jq/releases/download/jq-1.8.1/jq-linux-${jq_arch}"

scripts/evergreen/setup_kind.sh

@@ -15,7 +15,7 @@ latest_version="v0.29.0"
 if [[ "${arch_suffix}" == "amd64" || "${arch_suffix}" == "arm64" ]]; then
   mkdir -p "${PROJECT_DIR}/bin/"
   echo "Saving kind to ${PROJECT_DIR}/bin"
-  curl --retry 5 --retry-delay 5 --retry-all-errors --fail --show-error --max-time 600 -L "https://github.com/kubernetes-sigs/kind/releases/download/${latest_version}/kind-${os}-${arch_suffix}" -o kind
+  curl_with_retry -L "https://github.com/kubernetes-sigs/kind/releases/download/${latest_version}/kind-${os}-${arch_suffix}" -o kind
 
   chmod +x kind
   sudo mv kind "${PROJECT_DIR}/bin"

scripts/evergreen/setup_kubectl.sh

@@ -12,20 +12,16 @@ bindir="${PROJECT_DIR}/bin"
 tmpdir="${PROJECT_DIR}/tmp"
 mkdir -p "${bindir}" "${tmpdir}"
 
-kubectl_version=$(curl --retry 5 --retry-delay 5 --retry-all-errors --fail --max-time 120 -Ls https://dl.k8s.io/release/stable.txt)
-echo "Downloading kubectl ${kubectl_version} for ${ARCH}"
-kubectl_version=$(echo "${kubectl_version}" | tail -n1 | tr -d '\n')
-
-curl --retry 5 --retry-delay 5 --retry-all-errors --fail --show-error --max-time 600 -LOs "https://dl.k8s.io/release/${kubectl_version}/bin/linux/${ARCH}/kubectl"
-chmod +x kubectl
+# Use pinned version from root-context (no external API call needed)
+download_kubectl_binary "${KUBECTL_VERSION}" "${ARCH}"
 echo "kubectl version --client"
 ./kubectl version --client
 mv kubectl "${bindir}"
 
-echo "Downloading helm ${HELM_VERSION} for ${ARCH}"
-helm_archive="${tmpdir}/helm.tgz"
-curl --retry 5 --retry-delay 5 --retry-all-errors --fail --show-error --max-time 600 -s "https://get.helm.sh/helm-${HELM_VERSION}-linux-${ARCH}.tar.gz" --output "${helm_archive}"
+pushd "${tmpdir}" > /dev/null
+download_helm_binary "${HELM_VERSION}" "${ARCH}"
+mv "linux-${ARCH}/helm" "${bindir}"
+rm -rf "linux-${ARCH}/"
+popd > /dev/null
 
-tar xfz "${helm_archive}" -C "${tmpdir}" &> /dev/null
-mv "${tmpdir}/linux-${ARCH}/helm" "${bindir}"
 "${bindir}"/helm version