From 10f097ed18b0c40125a1c9cee42bd18691b39d27 Mon Sep 17 00:00:00 2001 From: "aks-node-assistant[bot]" <190555641+aks-node-assistant[bot]@users.noreply.github.com> Date: Thu, 12 Mar 2026 17:07:29 -0700 Subject: [PATCH 01/11] feat: bump windows image version for 2026-03B (#8074) Co-authored-by: Jane Jung Co-authored-by: janenotjung-hue <107402425+janenotjung-hue@users.noreply.github.com> Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> Co-authored-by: aks-node-assistant[bot] <190555641+aks-node-assistant[bot]@users.noreply.github.com> From 412ee54dc39e0176ed18f0aea3172d0effff461d Mon Sep 17 00:00:00 2001 From: Ramkumar Chinchani Date: Sun, 15 Mar 2026 22:08:49 -0700 Subject: [PATCH 02/11] feat(rcv1p): unify cert bootstrap flow and add Windows CA refresh task https://eng.ms/docs/products/onecert-certificates-key-vault-and-dsms/onecert-customer-guide/autorotationandecr/overviewrcv https://eng.ms/docs/products/onecert-certificates-key-vault-and-dsms/onecert-customer-guide/autorotationandecr/rcv1ptsg cse_cmd.sh.gtpl: derive cert endpoint mode from target cloud and always run custom-cloud init script. cse_cmd.sh: same mode logic as template; remove LOCATION export. init-aks-custom-cloud.sh: merged legacy + operation-requests logic into one script with distro-aware cert install paths. parts/linux/cloud-init/artifacts/init-aks-custom-cloud-mariner.sh: removed (merged into unified script). parts/linux/cloud-init/artifacts/init-aks-custom-cloud-operation-requests.sh: removed (merged into unified script). parts/linux/cloud-init/artifacts/init-aks-custom-cloud-operation-requests-mariner.sh: removed (merged into unified script). const.go: keep only unified custom-cloud init script constant. variables.go: simplify script selection to always use unified init script. kubernetesfunc.ps1: add location-aware CA retrieval (legacy/rcv1p) and scheduled refresh task registration helper. kuberneteswindowssetup.ps1: pass location to CA retrieval and register refresh task for custom cloud. --- aks-node-controller/parser/helper.go | 7 +- .../parser/templates/cse_cmd.sh.gtpl | 1 + .../init-aks-custom-cloud-mariner.sh | 186 --------- ...custom-cloud-operation-requests-mariner.sh | 236 ------------ ...nit-aks-custom-cloud-operation-requests.sh | 346 ----------------- .../artifacts/init-aks-custom-cloud.sh | 358 ++++++++++++++++-- parts/windows/kuberneteswindowssetup.ps1 | 4 +- pkg/agent/const.go | 9 +- pkg/agent/variables.go | 19 +- staging/cse/windows/kubernetesfunc.ps1 | 132 +++++-- 10 files changed, 454 insertions(+), 844 deletions(-) delete mode 100644 parts/linux/cloud-init/artifacts/init-aks-custom-cloud-mariner.sh delete mode 100644 parts/linux/cloud-init/artifacts/init-aks-custom-cloud-operation-requests-mariner.sh delete mode 100644 parts/linux/cloud-init/artifacts/init-aks-custom-cloud-operation-requests.sh diff --git a/aks-node-controller/parser/helper.go b/aks-node-controller/parser/helper.go index f5644dcda02..ab9e6210ccf 100644 --- a/aks-node-controller/parser/helper.go +++ b/aks-node-controller/parser/helper.go @@ -64,6 +64,7 @@ func getFuncMap() template.FuncMap { return template.FuncMap{ "getInitAKSCustomCloudFilepath": getInitAKSCustomCloudFilepath, "getIsAksCustomCloud": getIsAksCustomCloud, + "getCloudLocation": getCloudLocation, } } @@ -538,11 +539,15 @@ func getIsAksCustomCloud(customCloudConfig *aksnodeconfigv1.CustomCloudConfig) b return strings.EqualFold(customCloudConfig.GetCustomCloudEnvName(), helpers.AksCustomCloudName) } +func getCloudLocation(v *aksnodeconfigv1.Configuration) string { + return strings.ToLower(strings.Join(strings.Fields(v.GetClusterConfig().GetLocation()), "")) +} + /* GetCloudTargetEnv determines and returns whether the region is a sovereign cloud which have their own data compliance regulations (China/Germany/USGov) or standard. */ // Azure public cloud. func getCloudTargetEnv(v *aksnodeconfigv1.Configuration) string { - loc := strings.ToLower(strings.Join(strings.Fields(v.GetClusterConfig().GetLocation()), "")) + loc := getCloudLocation(v) switch { case strings.HasPrefix(loc, "china"): return "AzureChinaCloud" diff --git a/aks-node-controller/parser/templates/cse_cmd.sh.gtpl b/aks-node-controller/parser/templates/cse_cmd.sh.gtpl index b1359b071d9..d685a3444da 100644 --- a/aks-node-controller/parser/templates/cse_cmd.sh.gtpl +++ b/aks-node-controller/parser/templates/cse_cmd.sh.gtpl @@ -3,4 +3,5 @@ echo $(date),$(hostname) > ${PROVISION_OUTPUT}; REPO_DEPOT_ENDPOINT="{{.CustomCloudConfig.RepoDepotEndpoint}}" {{getInitAKSCustomCloudFilepath}} >> /var/log/azure/cluster-provision.log 2>&1; {{end}} +LOCATION="{{getCloudLocation .}}" /usr/bin/nohup /bin/bash -c "/bin/bash /opt/azure/containers/provision_start.sh" diff --git a/parts/linux/cloud-init/artifacts/init-aks-custom-cloud-mariner.sh b/parts/linux/cloud-init/artifacts/init-aks-custom-cloud-mariner.sh deleted file mode 100644 index 587da9ba270..00000000000 --- a/parts/linux/cloud-init/artifacts/init-aks-custom-cloud-mariner.sh +++ /dev/null @@ -1,186 +0,0 @@ -#!/bin/bash -set -x -mkdir -p /root/AzureCACertificates - -IS_MARINER=0 -IS_AZURELINUX=0 -# shellcheck disable=SC3010 -if [[ -f /etc/os-release ]]; then - . /etc/os-release - # shellcheck disable=SC3010 - if [[ $NAME == *"Mariner"* ]]; then - IS_MARINER=1 - elif [[ $NAME == *"Microsoft Azure Linux"* ]]; then - IS_AZURELINUX=1 - else - echo "Unknown Linux distribution" - exit 1 - fi -else - echo "Unsupported operating system" - exit 1 -fi - -echo "distribution is $distribution" -echo "Running on $NAME" - -# http://168.63.129.16 is a constant for the host's wireserver endpoint -certs=$(curl "http://168.63.129.16/machine?comp=acmspackage&type=cacertificates&ext=json") -IFS_backup=$IFS -IFS=$'\r\n' -certNames=($(echo $certs | grep -oP '(?<=Name\": \")[^\"]*')) -certBodies=($(echo $certs | grep -oP '(?<=CertBody\": \")[^\"]*')) -for i in ${!certBodies[@]}; do - echo ${certBodies[$i]} | sed 's/\\r\\n/\n/g' | sed 's/\\//g' > "/root/AzureCACertificates/$(echo ${certNames[$i]} | sed 's/.cer/.crt/g')" -done -IFS=$IFS_backup - -cp /root/AzureCACertificates/*.crt /etc/pki/ca-trust/source/anchors/ -/usr/bin/update-ca-trust - -# This section creates a cron job to poll for refreshed CA certs daily -# It can be removed if not needed or desired -action=${1:-init} -if [ "$action" = "ca-refresh" ]; then - exit -fi - -scriptPath=$0 -# Determine an absolute, canonical path to this script for use in cron. -if command -v readlink >/dev/null 2>&1; then - # Use readlink -f when available to resolve the canonical path; fall back to $0 on error. - scriptPath="$(readlink -f "$0" 2>/dev/null || printf '%s' "$0")" -fi - -if ! crontab -l 2>/dev/null | grep -q "\"$scriptPath\" ca-refresh"; then - # Quote the script path in the cron entry to avoid issues with spaces or special characters. - if ! (crontab -l 2>/dev/null ; printf '%s\n' "0 19 * * * \"$scriptPath\" ca-refresh") | crontab -; then - echo "Failed to install ca-refresh cron job via crontab" >&2 - fi -fi - -cloud-init status --wait - -function init_mariner_repo_depot { - local repodepot_endpoint=$1 - echo "Adding [extended] repo" - cp /etc/yum.repos.d/mariner-extras.repo /etc/yum.repos.d/mariner-extended.repo - sed -i -e "s|extras|extended|" /etc/yum.repos.d/mariner-extended.repo - sed -i -e "s|Extras|Extended|" /etc/yum.repos.d/mariner-extended.repo - - echo "Adding [nvidia] repo" - cp /etc/yum.repos.d/mariner-extras.repo /etc/yum.repos.d/mariner-nvidia.repo - sed -i -e "s|extras|nvidia|" /etc/yum.repos.d/mariner-nvidia.repo - sed -i -e "s|Extras|Nvidia|" /etc/yum.repos.d/mariner-nvidia.repo - - echo "Adding [cloud-native] repo" - cp /etc/yum.repos.d/mariner-extras.repo /etc/yum.repos.d/mariner-cloud-native.repo - sed -i -e "s|extras|cloud-native|" /etc/yum.repos.d/mariner-cloud-native.repo - sed -i -e "s|Extras|Cloud-Native|" /etc/yum.repos.d/mariner-cloud-native.repo - - echo "Pointing Mariner repos at RepoDepot..." - for f in /etc/yum.repos.d/*.repo - do - sed -i -e "s|https://packages.microsoft.com|${repodepot_endpoint}/mariner/packages.microsoft.com|" $f - echo "$f modified." - done - echo "Mariner repo setup complete." -} - -function init_azurelinux_repo_depot { - local repodepot_endpoint=$1 - repos=("amd" "base" "cloud-native" "extended" "ms-non-oss" "ms-oss" "nvidia") - - # tbd maybe we do this a bit nicer - rm -f /etc/yum.repos.d/azurelinux* - - for repo in "${repos[@]}"; do - output_file="/etc/yum.repos.d/azurelinux-${repo}.repo" - repo_content=( - "[azurelinux-official-$repo]" - "name=Azure Linux Official $repo \$releasever \$basearch" - "baseurl=$repodepot_endpoint/azurelinux/\$releasever/prod/$repo/\$basearch" - "gpgkey=file:///etc/pki/rpm-gpg/MICROSOFT-RPM-GPG-KEY" - "gpgcheck=1" - "repo_gpgcheck=1" - "enabled=1" - "skip_if_unavailable=True" - "sslverify=1" - ) - - rm -f "$output_file" - - for line in "${repo_content[@]}"; do - echo "$line" >> "$output_file" - done - - echo "File '$output_file' has been created." - done - echo "Azure Linux repo setup complete." -} - -dnf_makecache() { - local retries=10 - local dnf_makecache_output=/tmp/dnf-makecache.out - local i - for i in $(seq 1 $retries); do - ! (dnf makecache -y 2>&1 | tee $dnf_makecache_output | grep -E "^([WE]:.*)|([eE]rr.*)$") && \ - cat $dnf_makecache_output && break || \ - cat $dnf_makecache_output - if [ $i -eq $retries ]; then - return 1 - else sleep 5 - fi - done - echo "Executed dnf makecache -y $i times" -} - -marinerRepoDepotEndpoint="$(echo "${REPO_DEPOT_ENDPOINT}" | sed 's/\/ubuntu//')" -if [ -z "$marinerRepoDepotEndpoint" ]; then - >&2 echo "repo depot endpoint empty while running custom-cloud init script" -else - # logic taken from https://repodepot.azure.com/scripts/cloud-init/setup_repodepot.sh - if [ "$IS_MARINER" -eq 1 ]; then - echo "Initializing Mariner repo depot settings..." - init_mariner_repo_depot ${marinerRepoDepotEndpoint} - dnf_makecache || exit 1 - elif [ "$IS_AZURELINUX" -eq 1 ]; then - echo "Initializing Azure Linux repo depot settings..." - init_azurelinux_repo_depot ${marinerRepoDepotEndpoint} - dnf_makecache || exit 1 - else - echo "No customizations for distribution: $NAME" - fi -fi - -# Set the chrony config to use the PHC /dev/ptp0 clock -cat > /etc/chrony.conf < "/root/AzureCACertificates/$cert_filename" - echo "Successfully saved certificate: $cert_filename" - else - echo "Warning: Failed to retrieve certificate content for $cert_filename" - fi - done -} - -# Process root certificates -process_cert_operations "operationrequestsroot" - -# Process intermediate certificates -process_cert_operations "operationrequestsintermediate" - -# Copy all certificate files to the Mariner/AzureLinux system certificate directory -cp /root/AzureCACertificates/*.crt /etc/pki/ca-trust/source/anchors/ - -# Update the system certificate store using Mariner/AzureLinux command -/usr/bin/update-ca-trust - -# This section creates a cron job to poll for refreshed CA certs daily -# It can be removed if not needed or desired -action=${1:-init} -if [ "$action" = "ca-refresh" ]; then - exit -fi - -scriptPath=$0 -# Determine an absolute, canonical path to this script for use in cron. -if command -v readlink >/dev/null 2>&1; then - # Use readlink -f when available to resolve the canonical path; fall back to $0 on error. - scriptPath="$(readlink -f "$0" 2>/dev/null || printf '%s' "$0")" -fi - -if ! crontab -l 2>/dev/null | grep -q "\"$scriptPath\" ca-refresh"; then - # Quote the script path in the cron entry to avoid issues with spaces or special characters. - if ! (crontab -l 2>/dev/null ; printf '%s\n' "0 19 * * * \"$scriptPath\" ca-refresh") | crontab -; then - echo "Failed to install ca-refresh cron job via crontab" >&2 - fi -fi - -function init_mariner_repo_depot { - local repodepot_endpoint=$1 - echo "Adding [extended] repo" - cp /etc/yum.repos.d/mariner-extras.repo /etc/yum.repos.d/mariner-extended.repo - sed -i -e "s|extras|extended|" /etc/yum.repos.d/mariner-extended.repo - sed -i -e "s|Extras|Extended|" /etc/yum.repos.d/mariner-extended.repo - - echo "Adding [nvidia] repo" - cp /etc/yum.repos.d/mariner-extras.repo /etc/yum.repos.d/mariner-nvidia.repo - sed -i -e "s|extras|nvidia|" /etc/yum.repos.d/mariner-nvidia.repo - sed -i -e "s|Extras|Nvidia|" /etc/yum.repos.d/mariner-nvidia.repo - - echo "Adding [cloud-native] repo" - cp /etc/yum.repos.d/mariner-extras.repo /etc/yum.repos.d/mariner-cloud-native.repo - sed -i -e "s|extras|cloud-native|" /etc/yum.repos.d/mariner-cloud-native.repo - sed -i -e "s|Extras|Cloud-Native|" /etc/yum.repos.d/mariner-cloud-native.repo - - echo "Pointing Mariner repos at RepoDepot..." - for f in /etc/yum.repos.d/*.repo - do - sed -i -e "s|https://packages.microsoft.com|${repodepot_endpoint}/mariner/packages.microsoft.com|" $f - echo "$f modified." - done - echo "Mariner repo setup complete." -} - -function init_azurelinux_repo_depot { - local repodepot_endpoint=$1 - repos=("amd" "base" "cloud-native" "extended" "ms-non-oss" "ms-oss" "nvidia") - - # tbd maybe we do this a bit nicer - rm -f /etc/yum.repos.d/azurelinux* - - for repo in "${repos[@]}"; do - output_file="/etc/yum.repos.d/azurelinux-${repo}.repo" - repo_content=( - "[azurelinux-official-$repo]" - "name=Azure Linux Official $repo \$releasever \$basearch" - "baseurl=$repodepot_endpoint/azurelinux/\$releasever/prod/$repo/\$basearch" - "gpgkey=file:///etc/pki/rpm-gpg/MICROSOFT-RPM-GPG-KEY" - "gpgcheck=1" - "repo_gpgcheck=1" - "enabled=1" - "skip_if_unavailable=True" - "sslverify=1" - ) - - rm -f "$output_file" - - for line in "${repo_content[@]}"; do - echo "$line" >> "$output_file" - done - - echo "File '$output_file' has been created." - done -} - -cloud-init status --wait - -dnf_makecache() { - local retries=10 - local dnf_makecache_output=/tmp/dnf-makecache.out - local i - for i in $(seq 1 $retries); do - ! (dnf makecache -y 2>&1 | tee $dnf_makecache_output | grep -E "^([WE]:.*)|([eE]rr.*)$") && \ - cat $dnf_makecache_output && break || \ - cat $dnf_makecache_output - if [ $i -eq $retries ]; then - return 1 - else sleep 5 - fi - done - echo "Executed dnf makecache -y $i times" -} - -marinerRepoDepotEndpoint="$(echo "${REPO_DEPOT_ENDPOINT}" | sed 's/\/ubuntu//')" -if [ -z "$marinerRepoDepotEndpoint" ]; then - >&2 echo "repo depot endpoint empty while running custom-cloud init script" -else - # logic taken from https://repodepot.azure.com/scripts/cloud-init/setup_repodepot.sh - if [ "$IS_MARINER" -eq 1 ]; then - echo "Initializing Mariner repo depot settings..." - init_mariner_repo_depot ${marinerRepoDepotEndpoint} - dnf_makecache || exit 1 - elif [ "$IS_AZURELINUX" -eq 1 ]; then - echo "Initializing Azure Linux repo depot settings..." - init_azurelinux_repo_depot ${marinerRepoDepotEndpoint} - dnf_makecache || exit 1 - else - echo "No customizations for distribution: $NAME" - fi -fi - -#EOF diff --git a/parts/linux/cloud-init/artifacts/init-aks-custom-cloud-operation-requests.sh b/parts/linux/cloud-init/artifacts/init-aks-custom-cloud-operation-requests.sh deleted file mode 100644 index 99ae86d0242..00000000000 --- a/parts/linux/cloud-init/artifacts/init-aks-custom-cloud-operation-requests.sh +++ /dev/null @@ -1,346 +0,0 @@ -#!/bin/bash -set -x -mkdir -p /root/AzureCACertificates - -IS_FLATCAR=0 -IS_UBUNTU=0 -IS_ACL=0 -# shellcheck disable=SC3010 -if [[ -f /etc/os-release ]]; then - . /etc/os-release - # shellcheck disable=SC3010 - if [[ $NAME == *"Ubuntu"* ]]; then - IS_UBUNTU=1 - elif [[ $ID == *"flatcar"* ]]; then - IS_FLATCAR=1 - elif [[ $ID == "azurecontainerlinux" ]] || { [[ $ID == "azurelinux" ]] && [[ ${VARIANT_ID:-} == "azurecontainerlinux" ]]; }; then - IS_ACL=1 - else - echo "Unknown Linux distribution" - exit 1 - fi -else - echo "Unsupported operating system" - exit 1 -fi - -echo "distribution is $distribution" -echo "Running on $NAME" - -# http://168.63.129.16 is a constant for the host's wireserver endpoint -WIRESERVER_ENDPOINT="http://168.63.129.16" - -# Function to make HTTP request with retry logic for rate limiting -make_request_with_retry() { - local url="$1" - local max_retries=10 - local retry_delay=3 - local attempt=1 - - local response - while [ $attempt -le $max_retries ]; do - response=$(curl -f --no-progress-meter "$url") - local request_status=$? - - if echo "$response" | grep -q "RequestRateLimitExceeded"; then - sleep $retry_delay - retry_delay=$((retry_delay * 2)) - attempt=$((attempt + 1)) - elif [ $request_status -ne 0 ]; then - sleep $retry_delay - attempt=$((attempt + 1)) - else - echo "$response" - return 0 - fi - done - - echo "exhausted all retries, last response: $response" - return 1 -} - -# Function to process certificate operations from a given endpoint -process_cert_operations() { - local endpoint_type="$1" - local operation_response - - echo "Retrieving certificate operations for type: $endpoint_type" - operation_response=$(make_request_with_retry "${WIRESERVER_ENDPOINT}/machine?comp=acmspackage&type=$endpoint_type&ext=json") - local request_status=$? - if [ -z "$operation_response" ] || [ $request_status -ne 0 ]; then - echo "Warning: No response received or request failed for: ${WIRESERVER_ENDPOINT}/machine?comp=acmspackage&type=$endpoint_type&ext=json" - return - fi - - # Extract ResourceFileName values from the JSON response - local cert_filenames - mapfile -t cert_filenames < <(echo "$operation_response" | grep -oP '(?<="ResouceFileName": ")[^"]*') - - if [ ${#cert_filenames[@]} -eq 0 ]; then - echo "No certificate filenames found in response for $endpoint_type" - return - fi - - # Process each certificate file - for cert_filename in "${cert_filenames[@]}"; do - echo "Processing certificate file: $cert_filename" - - # Extract filename and extension - local filename="${cert_filename%.*}" - local extension="${cert_filename##*.}" - - echo "Downloading certificate: filename=$filename, extension=$extension" - - # Retrieve the actual certificate content with retry logic - local cert_content - cert_content=$(make_request_with_retry "${WIRESERVER_ENDPOINT}/machine?comp=acmspackage&type=$filename&ext=$extension") - local request_status=$? - if [ -z "$cert_content" ] || [ $request_status -ne 0 ]; then - echo "Warning: No response received or request failed for: ${WIRESERVER_ENDPOINT}/machine?comp=acmspackage&type=$filename&ext=$extension" - continue - fi - - if [ -n "$cert_content" ]; then - # Save the certificate to the appropriate location - echo "$cert_content" > "/root/AzureCACertificates/$cert_filename" - echo "Successfully saved certificate: $cert_filename" - else - echo "Warning: Failed to retrieve certificate content for $cert_filename" - fi - done -} - -# Process root certificates -process_cert_operations "operationrequestsroot" - -# Process intermediate certificates -process_cert_operations "operationrequestsintermediate" - -if [ "$IS_ACL" -eq 1 ]; then - cp /root/AzureCACertificates/*.crt /etc/pki/ca-trust/source/anchors/ - update-ca-trust -elif [ "${IS_FLATCAR}" -eq 0 ]; then - # Copy all certificate files to the system certificate directory - cp /root/AzureCACertificates/*.crt /usr/local/share/ca-certificates/ - - # Update the system certificate store - update-ca-certificates - - # This copies the updated bundle to the location used by OpenSSL which is commonly used - cp /etc/ssl/certs/ca-certificates.crt /usr/lib/ssl/cert.pem -else - for cert in /root/AzureCACertificates/*.crt; do - destcert="${cert##*/}" - destcert="${destcert%.*}.pem" - cp "$cert" /etc/ssl/certs/"$destcert" - done - update-ca-certificates -fi - - - -# This section creates a cron job to poll for refreshed CA certs daily -# It can be removed if not needed or desired -action=${1:-init} -if [ "$action" = "ca-refresh" ]; then - exit -fi - -function init_ubuntu_main_repo_depot { - local repodepot_endpoint="$1" - # Initialize directory for keys - mkdir -p /etc/apt/keyrings - - # This copies the updated bundle to the location used by OpenSSL which is commonly used - echo "Copying updated bundle to OpenSSL .pem file..." - cp /etc/ssl/certs/ca-certificates.crt /usr/lib/ssl/cert.pem - echo "Updated bundle copied." - - # Back up sources.list and sources.list.d contents - mkdir -p /etc/apt/backup/ - if [ -f "/etc/apt/sources.list" ]; then - mv /etc/apt/sources.list /etc/apt/backup/ - fi - for sources_file in /etc/apt/sources.list.d/*; do - if [ -f "$sources_file" ]; then - mv "$sources_file" /etc/apt/backup/ - fi - done - - # Set location of sources file - . /etc/os-release - aptSourceFile="/etc/apt/sources.list.d/ubuntu.sources" - - # Create main sources file - cat < /etc/apt/sources.list.d/ubuntu.sources - -Types: deb -URIs: ${repodepot_endpoint}/ubuntu -Suites: ${VERSION_CODENAME} ${VERSION_CODENAME}-updates ${VERSION_CODENAME}-backports ${VERSION_CODENAME}-security -Components: main universe restricted multiverse -Signed-By: /usr/share/keyrings/ubuntu-archive-keyring.gpg -EOF - - # Update the apt sources file using the RepoDepot Ubuntu URL for this cloud. Update it by replacing - # all urls with the RepoDepot Ubuntu url - ubuntuUrl=${repodepot_endpoint}/ubuntu - echo "Converting URLs in $aptSourceFile to RepoDepot URLs..." - sed -i "s,https\?://.[^ ]*,$ubuntuUrl,g" $aptSourceFile - echo "apt source URLs converted, see new file below:" - echo "" - echo "-----" - cat $aptSourceFile - echo "-----" - echo "" -} - -function check_url { - local url=$1 - echo "Checking url: $url" - - # Use curl to check the URL and capture both stdout and stderr - curl_exit_code=$(curl -s --head --request GET $url) - # Check the exit status of curl - # shellcheck disable=SC3010 - if [[ $? -ne 0 ]] || echo "$curl_exit_code" | grep -E "404 Not Found" > /dev/null; then - echo "ERROR: $url is not available. Please manually check if the url is valid before re-running script" - exit 1 - fi -} - -function write_to_sources_file { - local sources_list_d_file=$1 - local source_uri=$2 - shift 2 - local key_paths=("$@") - - sources_file_path="/etc/apt/sources.list.d/${sources_list_d_file}.sources" - ubuntuDist=$(lsb_release -c | awk '{print $2}') - - tee -a $sources_file_path < /dev/null - echo "$key_name key added to keyring." -} - -function derive_key_paths { - local key_names=("$@") - local key_paths=() - - for key_name in "${key_names[@]}"; do - key_paths+=("/etc/apt/keyrings/${key_name}.gpg") - done - - echo "${key_paths[*]}" -} - -function add_ms_keys { - # Add the Microsoft package server keys to keyring. - echo "Adding Microsoft keys to keyring..." - - add_key_ubuntu microsoft.asc - add_key_ubuntu msopentech.asc -} - -function aptget_update { - echo "apt-get updating..." - echo "note: depending on how many sources have been added this may take a couple minutes..." - if apt-get update | grep -q "404 Not Found"; then - echo "ERROR: apt-get update failed to find all sources. Please validate the sources or remove bad sources from your sources and try again." - exit 1 - else - echo "apt-get update complete!" - fi -} - -function init_ubuntu_pmc_repo_depot { - local repodepot_endpoint="$1" - # Add Microsoft packages source to the azure specific sources.list. - echo "Adding the packages.microsoft.com Ubuntu-$ubuntuRel repo..." - - microsoftPackageSource="$repodepot_endpoint/microsoft/ubuntu/$ubuntuRel/prod" - check_url $microsoftPackageSource - write_to_sources_file microsoft-prod $microsoftPackageSource $(derive_key_paths microsoft.asc msopentech.asc) - write_to_sources_file microsoft-prod-testing $microsoftPackageSource $(derive_key_paths microsoft.asc msopentech.asc) - echo "Ubuntu ($ubuntuRel) repo added." - echo "Adding packages.microsoft.com keys" - add_ms_keys $repodepot_endpoint -} - -if [ "$IS_UBUNTU" -eq 1 ]; then - scriptPath=$0 - # Determine an absolute, canonical path to this script for use in cron. - if command -v readlink >/dev/null 2>&1; then - # Use readlink -f when available to resolve the canonical path; fall back to $0 on error. - scriptPath="$(readlink -f "$0" 2>/dev/null || printf '%s' "$0")" - fi - - if ! crontab -l 2>/dev/null | grep -q "\"$scriptPath\" ca-refresh"; then - # Quote the script path in the cron entry to avoid issues with spaces or special characters. - if ! (crontab -l 2>/dev/null ; printf '%s\n' "0 19 * * * \"$scriptPath\" ca-refresh") | crontab -; then - echo "Failed to install ca-refresh cron job via crontab" >&2 - fi - fi - - cloud-init status --wait - rootRepoDepotEndpoint="$(echo "${REPO_DEPOT_ENDPOINT}" | sed 's/\/ubuntu//')" - # logic taken from https://repodepot.azure.com/scripts/cloud-init/setup_repodepot.sh - ubuntuRel=$(lsb_release --release | awk '{print $2}') - ubuntuDist=$(lsb_release -c | awk '{print $2}') - # initialize archive.ubuntu.com repo - init_ubuntu_main_repo_depot ${rootRepoDepotEndpoint} - init_ubuntu_pmc_repo_depot ${rootRepoDepotEndpoint} - # update apt list - echo "Running apt-get update" - aptget_update -elif [ "$IS_FLATCAR" -eq 1 ] || [ "$IS_ACL" -eq 1 ]; then - script_path="$(readlink -f "$0")" - svc="/etc/systemd/system/azure-ca-refresh.service" - tmr="/etc/systemd/system/azure-ca-refresh.timer" - - cat >"$svc" <"$tmr" < "/root/AzureCACertificates/$(echo ${certNames[$i]} | sed "s/.cer/.${ext}/g")" -done -IFS=$IFS_backup +WIRESERVER_ENDPOINT="http://168.63.129.16" + +function make_request_with_retry { + local url="$1" + local max_retries=10 + local retry_delay=3 + local attempt=1 + + local response + while [ $attempt -le $max_retries ]; do + response=$(curl -f --no-progress-meter "$url") + local request_status=$? + + if echo "$response" | grep -q "RequestRateLimitExceeded"; then + sleep $retry_delay + retry_delay=$((retry_delay * 2)) + attempt=$((attempt + 1)) + elif [ $request_status -ne 0 ]; then + sleep $retry_delay + attempt=$((attempt + 1)) + else + echo "$response" + return 0 + fi + done -if [ "$IS_ACL" -eq 1 ]; then - cp /root/AzureCACertificates/*.crt /etc/pki/ca-trust/source/anchors/ - update-ca-trust -elif [ "$IS_FLATCAR" -eq 1 ]; then - cp /root/AzureCACertificates/*.pem /etc/ssl/certs/ - update-ca-certificates -else - cp /root/AzureCACertificates/*.crt /usr/local/share/ca-certificates/ - update-ca-certificates + echo "exhausted all retries, last response: $response" + return 1 +} - # This copies the updated bundle to the location used by OpenSSL which is commonly used - cp /etc/ssl/certs/ca-certificates.crt /usr/lib/ssl/cert.pem +function is_opted_in_for_root_certs { + local opt_in_response + + opt_in_response=$(make_request_with_retry "${WIRESERVER_ENDPOINT}/acms/isOptedInForRootCerts") + local request_status=$? + if [ $request_status -ne 0 ] || [ -z "$opt_in_response" ]; then + echo "Warning: failed to determine IsOptedInForRootCerts state" + return 1 + fi + + if echo "$opt_in_response" | grep -q "IsOptedInForRootCerts=true"; then + echo "IsOptedInForRootCerts=true" + return 0 + fi + + echo "Skipping custom cloud root cert installation because IsOptedInForRootCerts is not true" + return 1 +} + +function get_trust_store_dir { + if [ "$IS_ACL" -eq 1 ] || [ "$IS_MARINER" -eq 1 ] || [ "$IS_AZURELINUX" -eq 1 ]; then + echo "/etc/pki/ca-trust/source/anchors" + elif [ "$IS_FLATCAR" -eq 1 ]; then + echo "/etc/ssl/certs" + else + echo "/usr/local/share/ca-certificates" + fi +} + +function debug_print_trust_store { + local stage="$1" + local trust_store_dir + + trust_store_dir=$(get_trust_store_dir) + echo "Trust store contents ${stage} cert copy: ${trust_store_dir}" + ls -al "$trust_store_dir" || true +} + +function retrieve_legacy_certs { + local certs + local cert_names + local cert_bodies + local i + + certs=$(make_request_with_retry "${WIRESERVER_ENDPOINT}/machine?comp=acmspackage&type=cacertificates&ext=json") + if [ -z "$certs" ]; then + echo "Warning: failed to retrieve legacy custom cloud certificates" + return 1 + fi + + IFS_backup=$IFS + IFS=$'\r\n' + cert_names=($(echo $certs | grep -oP '(?<=Name\": \")[^\"]*')) + cert_bodies=($(echo $certs | grep -oP '(?<=CertBody\": \")[^\"]*')) + for i in ${!cert_bodies[@]}; do + echo ${cert_bodies[$i]} | sed 's/\\r\\n/\n/g' | sed 's/\\//g' > "/root/AzureCACertificates/$(echo ${cert_names[$i]} | sed 's/.cer/.crt/g')" + done + IFS=$IFS_backup +} + +function process_cert_operations { + local endpoint_type="$1" + local operation_response + + echo "Retrieving certificate operations for type: $endpoint_type" + operation_response=$(make_request_with_retry "${WIRESERVER_ENDPOINT}/machine?comp=acmspackage&type=$endpoint_type&ext=json") + local request_status=$? + if [ -z "$operation_response" ] || [ $request_status -ne 0 ]; then + echo "Warning: No response received or request failed for: ${WIRESERVER_ENDPOINT}/machine?comp=acmspackage&type=$endpoint_type&ext=json" + return 1 + fi + + local cert_filenames + mapfile -t cert_filenames < <(echo "$operation_response" | grep -oP '(?<="ResouceFileName": ")[^"]*') + + if [ ${#cert_filenames[@]} -eq 0 ]; then + echo "No certificate filenames found in response for $endpoint_type" + return 1 + fi + + for cert_filename in "${cert_filenames[@]}"; do + echo "Processing certificate file: $cert_filename" + + local filename="${cert_filename%.*}" + local extension="${cert_filename##*.}" + local cert_content + + cert_content=$(make_request_with_retry "${WIRESERVER_ENDPOINT}/machine?comp=acmspackage&type=$filename&ext=$extension") + local request_status=$? + if [ -z "$cert_content" ] || [ $request_status -ne 0 ]; then + echo "Warning: No response received or request failed for: ${WIRESERVER_ENDPOINT}/machine?comp=acmspackage&type=$filename&ext=$extension" + continue + fi + + echo "$cert_content" > "/root/AzureCACertificates/$cert_filename" + echo "Successfully saved certificate: $cert_filename" + done +} + +function retrieve_rcv1p_certs { + process_cert_operations "operationrequestsroot" || return 1 + process_cert_operations "operationrequestsintermediate" || return 1 +} + +function install_certs_to_trust_store { + mkdir -p /root/AzureCACertificates + + debug_print_trust_store "before" + + if [ "$IS_ACL" -eq 1 ] || [ "$IS_MARINER" -eq 1 ] || [ "$IS_AZURELINUX" -eq 1 ]; then + cp /root/AzureCACertificates/*.crt /etc/pki/ca-trust/source/anchors/ + update-ca-trust + elif [ "$IS_FLATCAR" -eq 1 ]; then + for cert in /root/AzureCACertificates/*.crt; do + destcert="${cert##*/}" + destcert="${destcert%.*}.pem" + cp "$cert" /etc/ssl/certs/"$destcert" + done + update-ca-certificates + else + cp /root/AzureCACertificates/*.crt /usr/local/share/ca-certificates/ + update-ca-certificates + + # This copies the updated bundle to the location used by OpenSSL which is commonly used + cp /etc/ssl/certs/ca-certificates.crt /usr/lib/ssl/cert.pem + fi + + debug_print_trust_store "after" +} + +# Certificate refresh behavior summary: +# - legacy mode directly attempts certificate download from wireserver and only in ussec and usnat regions. +# - rcv1p mode first checks IsOptedInForRootCerts, then downloads only when opted in. +# - Wireserver failures are treated as non-fatal, and cert trust-store updates are skipped gracefully. + +location_normalized="${LOCATION,,}" +location_normalized="${location_normalized//[[:space:]]/}" +if [ -z "$location_normalized" ]; then + echo "Warning: LOCATION is empty; defaulting custom cloud certificate endpoint mode to rcv1p" +fi + +cert_endpoint_mode="rcv1p" +case "$location_normalized" in + ussec*|usnat*) cert_endpoint_mode="legacy" ;; +esac +echo "Using custom cloud certificate endpoint mode: ${cert_endpoint_mode}" +rm -f /root/AzureCACertificates/* +if [ "$cert_endpoint_mode" = "legacy" ]; then + if retrieve_legacy_certs; then + install_certs_to_trust_store + else + echo "Warning: failed to retrieve legacy certificates from wireserver; continuing without trust store updates" + fi +elif [ "$cert_endpoint_mode" = "rcv1p" ]; then + if is_opted_in_for_root_certs; then + if retrieve_rcv1p_certs; then + install_certs_to_trust_store + else + echo "Warning: failed to retrieve rcv1p certificates from wireserver; continuing without trust store updates" + fi + fi fi # This section creates a cron job to poll for refreshed CA certs daily @@ -201,7 +371,80 @@ function init_ubuntu_pmc_repo_depot { add_ms_keys $repodepot_endpoint } -if [ "$IS_UBUNTU" -eq 1 ]; then +function init_mariner_repo_depot { + local repodepot_endpoint=$1 + echo "Adding [extended] repo" + cp /etc/yum.repos.d/mariner-extras.repo /etc/yum.repos.d/mariner-extended.repo + sed -i -e "s|extras|extended|" /etc/yum.repos.d/mariner-extended.repo + sed -i -e "s|Extras|Extended|" /etc/yum.repos.d/mariner-extended.repo + + echo "Adding [nvidia] repo" + cp /etc/yum.repos.d/mariner-extras.repo /etc/yum.repos.d/mariner-nvidia.repo + sed -i -e "s|extras|nvidia|" /etc/yum.repos.d/mariner-nvidia.repo + sed -i -e "s|Extras|Nvidia|" /etc/yum.repos.d/mariner-nvidia.repo + + echo "Adding [cloud-native] repo" + cp /etc/yum.repos.d/mariner-extras.repo /etc/yum.repos.d/mariner-cloud-native.repo + sed -i -e "s|extras|cloud-native|" /etc/yum.repos.d/mariner-cloud-native.repo + sed -i -e "s|Extras|Cloud-Native|" /etc/yum.repos.d/mariner-cloud-native.repo + + echo "Pointing Mariner repos at RepoDepot..." + for f in /etc/yum.repos.d/*.repo; do + sed -i -e "s|https://packages.microsoft.com|${repodepot_endpoint}/mariner/packages.microsoft.com|" $f + echo "$f modified." + done + echo "Mariner repo setup complete." +} + +function init_azurelinux_repo_depot { + local repodepot_endpoint=$1 + local repos=("amd" "base" "cloud-native" "extended" "ms-non-oss" "ms-oss" "nvidia") + + rm -f /etc/yum.repos.d/azurelinux* + + for repo in "${repos[@]}"; do + output_file="/etc/yum.repos.d/azurelinux-${repo}.repo" + repo_content=( + "[azurelinux-official-$repo]" + "name=Azure Linux Official $repo \$releasever \$basearch" + "baseurl=$repodepot_endpoint/azurelinux/\$releasever/prod/$repo/\$basearch" + "gpgkey=file:///etc/pki/rpm-gpg/MICROSOFT-RPM-GPG-KEY" + "gpgcheck=1" + "repo_gpgcheck=1" + "enabled=1" + "skip_if_unavailable=True" + "sslverify=1" + ) + + rm -f "$output_file" + + for line in "${repo_content[@]}"; do + echo "$line" >> "$output_file" + done + + echo "File '$output_file' has been created." + done + echo "Azure Linux repo setup complete." +} + +function dnf_makecache { + local retries=10 + local dnf_makecache_output=/tmp/dnf-makecache.out + local i + for i in $(seq 1 $retries); do + ! (dnf makecache -y 2>&1 | tee $dnf_makecache_output | grep -E "^([WE]:.*)|([eE]rr.*)$") && \ + cat $dnf_makecache_output && break || \ + cat $dnf_makecache_output + if [ $i -eq $retries ]; then + return 1 + else + sleep 5 + fi + done + echo "Executed dnf makecache -y $i times" +} + +if [ "$IS_UBUNTU" -eq 1 ] || [ "$IS_MARINER" -eq 1 ] || [ "$IS_AZURELINUX" -eq 1 ]; then scriptPath=$0 # Determine an absolute, canonical path to this script for use in cron. if command -v readlink >/dev/null 2>&1; then @@ -260,11 +503,72 @@ EOF systemctl enable --now azure-ca-refresh.timer fi +if [ "$IS_UBUNTU" -eq 1 ]; then + rootRepoDepotEndpoint="$(echo "${REPO_DEPOT_ENDPOINT}" | sed 's/\/ubuntu//')" + if [ -n "$rootRepoDepotEndpoint" ]; then + cloud-init status --wait + ubuntuRel=$(lsb_release --release | awk '{print $2}') + ubuntuDist=$(lsb_release -c | awk '{print $2}') + init_ubuntu_main_repo_depot ${rootRepoDepotEndpoint} + init_ubuntu_pmc_repo_depot ${rootRepoDepotEndpoint} + echo "Running apt-get update" + aptget_update + else + echo "REPO_DEPOT_ENDPOINT empty, skipping Ubuntu RepoDepot initialization" + fi +elif [ "$IS_MARINER" -eq 1 ] || [ "$IS_AZURELINUX" -eq 1 ]; then + cloud-init status --wait + + marinerRepoDepotEndpoint="$(echo "${REPO_DEPOT_ENDPOINT}" | sed 's/\/ubuntu//')" + if [ -z "$marinerRepoDepotEndpoint" ]; then + >&2 echo "repo depot endpoint empty while running custom-cloud init script" + else + if [ "$IS_MARINER" -eq 1 ]; then + echo "Initializing Mariner repo depot settings..." + init_mariner_repo_depot ${marinerRepoDepotEndpoint} + dnf_makecache || exit 1 + else + echo "Initializing Azure Linux repo depot settings..." + init_azurelinux_repo_depot ${marinerRepoDepotEndpoint} + dnf_makecache || exit 1 + fi + fi +fi + # Disable systemd-timesyncd and install chrony and uses local time source # ACL has PTP clock config compiled into chronyd with no config file or sourcedir directives, # so it uses only the local PTP clock and has no DHCP-injectable NTP sources. if [ "$IS_ACL" -eq 1 ]; then echo "Skipping chrony configuration for ACL (PTP clock baked into chronyd, no external NTP sources)" +elif [ "$IS_MARINER" -eq 1 ] || [ "$IS_AZURELINUX" -eq 1 ]; then +cat > /etc/chrony.conf < $certFilePath + } + + return $true } - Write-Log "Convert CA certificates rawdata" - $caCerts=($rawData.Content) | ConvertFrom-Json - if ([string]::IsNullOrEmpty($caCerts)) { - Set-ExitCode -ExitCode $global:WINDOWS_CSE_ERROR_EMPTY_CA_CERTIFICATES -ErrorMessage "CA certificates rawdata is empty" + $optInUri = 'http://168.63.129.16/acms/isOptedInForRootCerts' + $optInResponse = Retry-Command -Command 'Invoke-WebRequest' -Args @{Uri=$optInUri; UseBasicParsing=$true} -Retries 5 -RetryDelaySeconds 10 + if (($optInResponse.Content -notmatch 'IsOptedInForRootCerts=true')) { + Write-Log "Skipping custom cloud root cert installation because IsOptedInForRootCerts is not true" + return $false } - $certificates = $caCerts.Certificates - for ($index = 0; $index -lt $certificates.Length ; $index++) { - $name=$certificates[$index].Name - $certFilePath = Join-Path $caFolder $name - Write-Log "Write certificate $name to $certFilePath" - $certificates[$index].CertBody > $certFilePath + $operationRequestTypes = @("operationrequestsroot", "operationrequestsintermediate") + $downloadedAny = $false + + foreach ($requestType in $operationRequestTypes) { + $operationRequestUri = "http://168.63.129.16/machine?comp=acmspackage&type=$requestType&ext=json" + $operationResponse = Retry-Command -Command 'Invoke-WebRequest' -Args @{Uri=$operationRequestUri; UseBasicParsing=$true} -Retries 5 -RetryDelaySeconds 10 + $operationJson = ($operationResponse.Content) | ConvertFrom-Json + + if ($null -eq $operationJson -or $null -eq $operationJson.OperationRequests) { + Write-Log "Warning: no operation requests found for $requestType" + continue + } + + foreach ($operation in $operationJson.OperationRequests) { + $resourceFileName = $operation.ResouceFileName + if ([string]::IsNullOrEmpty($resourceFileName)) { + continue + } + + $resourceType = [IO.Path]::GetFileNameWithoutExtension($resourceFileName) + $resourceExt = [IO.Path]::GetExtension($resourceFileName).TrimStart('.') + $resourceUri = "http://168.63.129.16/machine?comp=acmspackage&type=$resourceType&ext=$resourceExt" + + $certContentResponse = Retry-Command -Command 'Invoke-WebRequest' -Args @{Uri=$resourceUri; UseBasicParsing=$true} -Retries 5 -RetryDelaySeconds 10 + if ([string]::IsNullOrEmpty($certContentResponse.Content)) { + Write-Log "Warning: empty certificate content for $resourceFileName" + continue + } + + $certFilePath = Join-Path $caFolder $resourceFileName + Write-Log "Write certificate $resourceFileName to $certFilePath" + $certContentResponse.Content > $certFilePath + $downloadedAny = $true + } + } + + if (-not $downloadedAny) { + Write-Log "Warning: no CA certificates were downloaded in rcv1p mode" } + + return $downloadedAny } catch { - # Catch all exceptions in this function. NOTE: exit cannot be caught. - Set-ExitCode -ExitCode $global:WINDOWS_CSE_ERROR_GET_CA_CERTIFICATES -ErrorMessage $_ + Write-Log "Warning: failed to retrieve CA certificates. Error: $_" + return $false } } From bad56ebb211c826fb9b78a50eb36475a13b96488 Mon Sep 17 00:00:00 2001 From: Ramkumar Chinchani Date: Wed, 18 Mar 2026 14:08:50 -0700 Subject: [PATCH 03/11] feat: enhance CA certificates refresh task with endpoint mode based on location --- .../artifacts/init-aks-custom-cloud.sh | 33 ++++++++++++------- parts/windows/kuberneteswindowssetup.ps1 | 4 ++- staging/cse/windows/kubernetesfunc.ps1 | 15 +++++---- 3 files changed, 34 insertions(+), 18 deletions(-) diff --git a/parts/linux/cloud-init/artifacts/init-aks-custom-cloud.sh b/parts/linux/cloud-init/artifacts/init-aks-custom-cloud.sh index fab9e105975..9f3b4fe479e 100644 --- a/parts/linux/cloud-init/artifacts/init-aks-custom-cloud.sh +++ b/parts/linux/cloud-init/artifacts/init-aks-custom-cloud.sh @@ -198,16 +198,28 @@ function install_certs_to_trust_store { # - rcv1p mode first checks IsOptedInForRootCerts, then downloads only when opted in. # - Wireserver failures are treated as non-fatal, and cert trust-store updates are skipped gracefully. -location_normalized="${LOCATION,,}" -location_normalized="${location_normalized//[[:space:]]/}" -if [ -z "$location_normalized" ]; then - echo "Warning: LOCATION is empty; defaulting custom cloud certificate endpoint mode to rcv1p" +# Action values: +# - init: normal provisioning path +# - ca-refresh: scheduled refresh path +action=${1:-init} +requested_cert_endpoint_mode="${2:-}" + +cert_endpoint_mode="" +if [ "$action" = "ca-refresh" ] && [ -n "$requested_cert_endpoint_mode" ]; then + cert_endpoint_mode="${requested_cert_endpoint_mode,,}" +else + location_normalized="${LOCATION,,}" + location_normalized="${location_normalized//[[:space:]]/}" + if [ -z "$location_normalized" ]; then + echo "Warning: LOCATION is empty; defaulting custom cloud certificate endpoint mode to rcv1p" + fi + + cert_endpoint_mode="rcv1p" + case "$location_normalized" in + ussec*|usnat*) cert_endpoint_mode="legacy" ;; + esac fi -cert_endpoint_mode="rcv1p" -case "$location_normalized" in - ussec*|usnat*) cert_endpoint_mode="legacy" ;; -esac echo "Using custom cloud certificate endpoint mode: ${cert_endpoint_mode}" rm -f /root/AzureCACertificates/* if [ "$cert_endpoint_mode" = "legacy" ]; then @@ -228,7 +240,6 @@ fi # This section creates a cron job to poll for refreshed CA certs daily # It can be removed if not needed or desired -action=${1:-init} if [ "$action" = "ca-refresh" ]; then exit fi @@ -454,7 +465,7 @@ if [ "$IS_UBUNTU" -eq 1 ] || [ "$IS_MARINER" -eq 1 ] || [ "$IS_AZURELINUX" -eq 1 if ! crontab -l 2>/dev/null | grep -q "\"$scriptPath\" ca-refresh"; then # Quote the script path in the cron entry to avoid issues with spaces or special characters. - if ! (crontab -l 2>/dev/null ; printf '%s\n' "0 19 * * * \"$scriptPath\" ca-refresh") | crontab -; then + if ! (crontab -l 2>/dev/null ; printf '%s\n' "0 19 * * * \"$scriptPath\" ca-refresh \"$cert_endpoint_mode\"") | crontab -; then echo "Failed to install ca-refresh cron job via crontab" >&2 fi fi @@ -483,7 +494,7 @@ Wants=network-online.target [Service] Type=oneshot -ExecStart=$script_path ca-refresh +ExecStart=$script_path ca-refresh $cert_endpoint_mode EOF cat >"$tmr" < Date: Wed, 18 Mar 2026 17:14:10 -0700 Subject: [PATCH 04/11] feat: add tests for certificate endpoint mode handling in AKS custom cloud spec --- .../artifacts/init_aks_custom_cloud_spec.sh | 39 +++++ staging/cse/windows/kubernetesfunc.tests.ps1 | 147 ++++++++++++++++++ 2 files changed, 186 insertions(+) create mode 100644 spec/parts/linux/cloud-init/artifacts/init_aks_custom_cloud_spec.sh create mode 100644 staging/cse/windows/kubernetesfunc.tests.ps1 diff --git a/spec/parts/linux/cloud-init/artifacts/init_aks_custom_cloud_spec.sh b/spec/parts/linux/cloud-init/artifacts/init_aks_custom_cloud_spec.sh new file mode 100644 index 00000000000..f00709306c2 --- /dev/null +++ b/spec/parts/linux/cloud-init/artifacts/init_aks_custom_cloud_spec.sh @@ -0,0 +1,39 @@ +#!/bin/bash + +Describe 'init-aks-custom-cloud.sh refresh mode wiring' + script_path='./parts/linux/cloud-init/artifacts/init-aks-custom-cloud.sh' + + It 'parses action and optional requested cert endpoint mode arguments' + When run grep -Eq '^action=\$\{1:-init\}$' "$script_path" + The status should eq 0 + + When run grep -Eq '^requested_cert_endpoint_mode="\$\{2:-\}"$' "$script_path" + The status should eq 0 + End + + It 'uses requested mode during ca-refresh when provided' + When run grep -Eq '^if \[ "\$action" = "ca-refresh" \] && \[ -n "\$requested_cert_endpoint_mode" \]; then$' "$script_path" + The status should eq 0 + + When run grep -Eq '^\s*cert_endpoint_mode="\$\{requested_cert_endpoint_mode,,\}"$' "$script_path" + The status should eq 0 + End + + It 'exits early in ca-refresh mode after certificate refresh logic' + When run grep -Eq '^if \[ "\$action" = "ca-refresh" \]; then$' "$script_path" + The status should eq 0 + + When run grep -Eq '^\s*exit$' "$script_path" + The status should eq 0 + End + + It 'passes cert endpoint mode into cron refresh command' + When run grep -Eq 'ca-refresh "\$cert_endpoint_mode"' "$script_path" + The status should eq 0 + End + + It 'passes cert endpoint mode into systemd refresh command' + When run grep -Eq '^ExecStart=\$script_path ca-refresh \$cert_endpoint_mode$' "$script_path" + The status should eq 0 + End +End diff --git a/staging/cse/windows/kubernetesfunc.tests.ps1 b/staging/cse/windows/kubernetesfunc.tests.ps1 new file mode 100644 index 00000000000..ba14ebb48ef --- /dev/null +++ b/staging/cse/windows/kubernetesfunc.tests.ps1 @@ -0,0 +1,147 @@ +if (-not (Get-PSDrive -Name C -ErrorAction SilentlyContinue)) { + New-PSDrive -Name C -PSProvider FileSystem -Root ([System.IO.Path]::GetTempPath()) | Out-Null +} + +function Write-Log { + param($Message) + Write-Host "$Message" +} + +function Logs-To-Event { + param($TaskName, $TaskMessage) + Write-Host "$TaskName $TaskMessage" +} + +function Set-ExitCode { + param($ExitCode, $ErrorMessage) + throw "Unexpected Set-ExitCode: $ExitCode $ErrorMessage" +} + +function Create-Directory { + param($FullPath, $DirectoryUsage) + if (-not (Test-Path $FullPath)) { + New-Item -Path $FullPath -ItemType Directory -Force | Out-Null + } +} + +function Get-ScheduledTask { + param($TaskName, $ErrorAction) +} + +function New-ScheduledTaskAction { + param($Execute, $Argument) +} + +function New-ScheduledTaskPrincipal { + param($UserId, $LogonType, $RunLevel) +} + +function New-JobTrigger { + param([switch]$Daily, $At, $DaysInterval) +} + +function New-ScheduledTask { + param($Action, $Principal, $Trigger, $Description) +} + +function Register-ScheduledTask { + param($TaskName, $InputObject) +} + +. $PSScriptRoot\..\..\..\parts\windows\windowscsehelper.ps1 +. $PSCommandPath.Replace('.tests.ps1', '.ps1') + +Describe 'Get-CustomCloudCertEndpointModeFromLocation' { + It 'returns legacy for ussec regions' { + Get-CustomCloudCertEndpointModeFromLocation -Location 'ussecwest' | Should Be 'legacy' + } + + It 'returns legacy for usnat regions' { + Get-CustomCloudCertEndpointModeFromLocation -Location 'usnatcentral' | Should Be 'legacy' + } + + It 'returns rcv1p for public regions' { + Get-CustomCloudCertEndpointModeFromLocation -Location 'southcentralus' | Should Be 'rcv1p' + } + + It 'handles mixed-case input' { + Get-CustomCloudCertEndpointModeFromLocation -Location 'UsSeCeast' | Should Be 'legacy' + } +} + +Describe 'Register-CACertificatesRefreshTask' { + BeforeEach { + $script:lastScheduledTaskArgument = $null + + Mock Logs-To-Event + Mock Write-Log + Mock New-ScheduledTaskPrincipal -MockWith { return @{ Kind = 'principal' } } + Mock New-JobTrigger -MockWith { return @{ Kind = 'trigger' } } + Mock New-ScheduledTask -MockWith { return @{ Kind = 'definition' } } + Mock Register-ScheduledTask + Mock New-ScheduledTaskAction -MockWith { + param($Execute, $Argument) + $script:lastScheduledTaskArgument = $Argument + return @{ Execute = $Execute; Argument = $Argument } + } + } + + It 'skips registration when the task already exists' { + Mock Get-ScheduledTask -MockWith { return @{ TaskName = 'aks-ca-certs-refresh-task' } } + + Register-CACertificatesRefreshTask -Location 'southcentralus' -CertEndpointMode 'rcv1p' + + Assert-MockCalled -CommandName Register-ScheduledTask -Exactly -Times 0 + Assert-MockCalled -CommandName New-ScheduledTaskAction -Exactly -Times 0 + } + + It 'creates a scheduled task that passes the explicit cert endpoint mode' { + Mock Get-ScheduledTask -MockWith { return $null } + + Register-CACertificatesRefreshTask -Location 'southcentralus' -CertEndpointMode 'rcv1p' + + Assert-MockCalled -CommandName Register-ScheduledTask -Exactly -Times 1 + $script:lastScheduledTaskArgument | Should Match ([regex]::Escape("Get-CACertificates -Location 'southcentralus' -CertEndpointMode 'rcv1p'")) + } +} + +Describe 'Get-CACertificates' { + BeforeEach { + Mock Write-Log + Mock Create-Directory -MockWith { + param($FullPath, $DirectoryUsage) + if (-not (Test-Path $FullPath)) { + New-Item -Path $FullPath -ItemType Directory -Force | Out-Null + } + } + + if (Test-Path 'C:\ca') { + Remove-Item -Path 'C:\ca' -Recurse -Force + } + } + + It 'uses the legacy endpoint when CertEndpointMode is legacy regardless of location' { + Mock Retry-Command -MockWith { + param($Command, $Args, $Retries, $RetryDelaySeconds) + return [PSCustomObject]@{ + Content = '{"Certificates":[{"Name":"legacy.crt","CertBody":"legacy-body"}]}' + } + } + + $result = Get-CACertificates -Location 'southcentralus' -CertEndpointMode 'legacy' + + $result | Should Be $true + Assert-MockCalled -CommandName Retry-Command -Exactly -Times 1 -ParameterFilter { $Args.Uri -eq 'http://168.63.129.16/machine?comp=acmspackage&type=cacertificates&ext=json' } + Assert-MockCalled -CommandName Retry-Command -Exactly -Times 0 -ParameterFilter { $Args.Uri -eq 'http://168.63.129.16/acms/isOptedInForRootCerts' } + } + + It 'returns false when certificate retrieval throws' { + Mock Retry-Command -MockWith { + throw 'simulated retrieval failure' + } + + $result = Get-CACertificates -Location 'ussecwest' -CertEndpointMode 'rcv1p' + + $result | Should Be $false + } +} From 38b506272db4560ae9c456b69c04396097b458b5 Mon Sep 17 00:00:00 2001 From: Ramkumar Chinchani Date: Thu, 19 Mar 2026 12:44:29 -0700 Subject: [PATCH 05/11] feat: simplify certificate endpoint mode handling and refresh task registration --- .../artifacts/init-aks-custom-cloud.sh | 41 ++++++++----------- parts/windows/kuberneteswindowssetup.ps1 | 4 +- .../artifacts/init_aks_custom_cloud_spec.sh | 21 ++++++---- staging/cse/windows/kubernetesfunc.ps1 | 15 +++---- staging/cse/windows/kubernetesfunc.tests.ps1 | 14 +++---- 5 files changed, 44 insertions(+), 51 deletions(-) diff --git a/parts/linux/cloud-init/artifacts/init-aks-custom-cloud.sh b/parts/linux/cloud-init/artifacts/init-aks-custom-cloud.sh index 9f3b4fe479e..c7176be2393 100644 --- a/parts/linux/cloud-init/artifacts/init-aks-custom-cloud.sh +++ b/parts/linux/cloud-init/artifacts/init-aks-custom-cloud.sh @@ -198,28 +198,19 @@ function install_certs_to_trust_store { # - rcv1p mode first checks IsOptedInForRootCerts, then downloads only when opted in. # - Wireserver failures are treated as non-fatal, and cert trust-store updates are skipped gracefully. -# Action values: -# - init: normal provisioning path -# - ca-refresh: scheduled refresh path -action=${1:-init} -requested_cert_endpoint_mode="${2:-}" - -cert_endpoint_mode="" -if [ "$action" = "ca-refresh" ] && [ -n "$requested_cert_endpoint_mode" ]; then - cert_endpoint_mode="${requested_cert_endpoint_mode,,}" -else - location_normalized="${LOCATION,,}" - location_normalized="${location_normalized//[[:space:]]/}" - if [ -z "$location_normalized" ]; then - echo "Warning: LOCATION is empty; defaulting custom cloud certificate endpoint mode to rcv1p" - fi +refresh_location="${2:-${LOCATION}}" - cert_endpoint_mode="rcv1p" - case "$location_normalized" in - ussec*|usnat*) cert_endpoint_mode="legacy" ;; - esac +location_normalized="${refresh_location,,}" +location_normalized="${location_normalized//[[:space:]]/}" +if [ -z "$location_normalized" ]; then + echo "Warning: LOCATION is empty; defaulting custom cloud certificate endpoint mode to rcv1p" fi +cert_endpoint_mode="rcv1p" +case "$location_normalized" in + ussec*|usnat*) cert_endpoint_mode="legacy" ;; +esac + echo "Using custom cloud certificate endpoint mode: ${cert_endpoint_mode}" rm -f /root/AzureCACertificates/* if [ "$cert_endpoint_mode" = "legacy" ]; then @@ -238,8 +229,12 @@ elif [ "$cert_endpoint_mode" = "rcv1p" ]; then fi fi -# This section creates a cron job to poll for refreshed CA certs daily -# It can be removed if not needed or desired +# In ca-refresh mode (invoked by the scheduled cron/systemd task with the location as arg), +# only the cert refresh above is needed; exit before running the full init path. +# Action values: +# - init (default): full provisioning path +# - ca-refresh : periodic refresh path; location is passed as arg to avoid env dependency +action=${1:-init} if [ "$action" = "ca-refresh" ]; then exit fi @@ -465,7 +460,7 @@ if [ "$IS_UBUNTU" -eq 1 ] || [ "$IS_MARINER" -eq 1 ] || [ "$IS_AZURELINUX" -eq 1 if ! crontab -l 2>/dev/null | grep -q "\"$scriptPath\" ca-refresh"; then # Quote the script path in the cron entry to avoid issues with spaces or special characters. - if ! (crontab -l 2>/dev/null ; printf '%s\n' "0 19 * * * \"$scriptPath\" ca-refresh \"$cert_endpoint_mode\"") | crontab -; then + if ! (crontab -l 2>/dev/null ; printf '%s\n' "0 19 * * * \"$scriptPath\" ca-refresh \"$LOCATION\"") | crontab -; then echo "Failed to install ca-refresh cron job via crontab" >&2 fi fi @@ -494,7 +489,7 @@ Wants=network-online.target [Service] Type=oneshot -ExecStart=$script_path ca-refresh $cert_endpoint_mode +ExecStart=$script_path ca-refresh $LOCATION EOF cat >"$tmr" < Date: Thu, 19 Mar 2026 13:04:03 -0700 Subject: [PATCH 06/11] feat: implement conditional CA certificates refresh task registration for legacy and opted-in rcv1p modes --- .../artifacts/init-aks-custom-cloud.sh | 29 +++++++++------ parts/windows/kuberneteswindowssetup.ps1 | 4 ++- .../artifacts/init_aks_custom_cloud_spec.sh | 11 ++++++ staging/cse/windows/kubernetesfunc.ps1 | 21 +++++++++++ staging/cse/windows/kubernetesfunc.tests.ps1 | 36 +++++++++++++++++++ 5 files changed, 89 insertions(+), 12 deletions(-) diff --git a/parts/linux/cloud-init/artifacts/init-aks-custom-cloud.sh b/parts/linux/cloud-init/artifacts/init-aks-custom-cloud.sh index c7176be2393..eeb01c392fe 100644 --- a/parts/linux/cloud-init/artifacts/init-aks-custom-cloud.sh +++ b/parts/linux/cloud-init/artifacts/init-aks-custom-cloud.sh @@ -212,8 +212,10 @@ case "$location_normalized" in esac echo "Using custom cloud certificate endpoint mode: ${cert_endpoint_mode}" +install_ca_refresh_schedule=0 rm -f /root/AzureCACertificates/* if [ "$cert_endpoint_mode" = "legacy" ]; then + install_ca_refresh_schedule=1 if retrieve_legacy_certs; then install_certs_to_trust_store else @@ -221,6 +223,7 @@ if [ "$cert_endpoint_mode" = "legacy" ]; then fi elif [ "$cert_endpoint_mode" = "rcv1p" ]; then if is_opted_in_for_root_certs; then + install_ca_refresh_schedule=1 if retrieve_rcv1p_certs; then install_certs_to_trust_store else @@ -458,10 +461,12 @@ if [ "$IS_UBUNTU" -eq 1 ] || [ "$IS_MARINER" -eq 1 ] || [ "$IS_AZURELINUX" -eq 1 scriptPath="$(readlink -f "$0" 2>/dev/null || printf '%s' "$0")" fi - if ! crontab -l 2>/dev/null | grep -q "\"$scriptPath\" ca-refresh"; then - # Quote the script path in the cron entry to avoid issues with spaces or special characters. - if ! (crontab -l 2>/dev/null ; printf '%s\n' "0 19 * * * \"$scriptPath\" ca-refresh \"$LOCATION\"") | crontab -; then - echo "Failed to install ca-refresh cron job via crontab" >&2 + if [ "$install_ca_refresh_schedule" -eq 1 ]; then + if ! crontab -l 2>/dev/null | grep -q "\"$scriptPath\" ca-refresh"; then + # Quote the script path in the cron entry to avoid issues with spaces or special characters. + if ! (crontab -l 2>/dev/null ; printf '%s\n' "0 19 * * * \"$scriptPath\" ca-refresh \"$LOCATION\"") | crontab -; then + echo "Failed to install ca-refresh cron job via crontab" >&2 + fi fi fi @@ -477,11 +482,12 @@ if [ "$IS_UBUNTU" -eq 1 ] || [ "$IS_MARINER" -eq 1 ] || [ "$IS_AZURELINUX" -eq 1 echo "Running apt-get update" aptget_update elif [ "$IS_FLATCAR" -eq 1 ] || [ "$IS_ACL" -eq 1 ]; then - script_path="$(readlink -f "$0")" - svc="/etc/systemd/system/azure-ca-refresh.service" - tmr="/etc/systemd/system/azure-ca-refresh.timer" + if [ "$install_ca_refresh_schedule" -eq 1 ]; then + script_path="$(readlink -f "$0")" + svc="/etc/systemd/system/azure-ca-refresh.service" + tmr="/etc/systemd/system/azure-ca-refresh.timer" - cat >"$svc" <"$svc" <"$tmr" <"$tmr" < Date: Thu, 19 Mar 2026 14:54:49 -0700 Subject: [PATCH 07/11] feat: enhance CA certificates refresh task registration for legacy CSE packages --- parts/windows/kuberneteswindowssetup.ps1 | 9 ++++++++- .../cloud-init/artifacts/init_aks_custom_cloud_spec.sh | 10 +++++----- staging/cse/windows/kubernetesfunc.tests.ps1 | 3 --- 3 files changed, 13 insertions(+), 9 deletions(-) diff --git a/parts/windows/kuberneteswindowssetup.ps1 b/parts/windows/kuberneteswindowssetup.ps1 index 8d9297b090a..3bf59f71b0b 100644 --- a/parts/windows/kuberneteswindowssetup.ps1 +++ b/parts/windows/kuberneteswindowssetup.ps1 @@ -475,7 +475,14 @@ function BasePrep { Adjust-DynamicPortRange Register-LogsCleanupScriptTask Register-NodeResetScriptTask - if (Should-InstallCACertificatesRefreshTask -Location $Location) { + # Guard against older CSE packages that do not yet export Should-InstallCACertificatesRefreshTask. + # If the function is absent (old package), fall back to the previous unconditional behaviour so + # that legacy/ussec/usnat clusters continue to register the refresh task. + if (Get-Command -Name Should-InstallCACertificatesRefreshTask -ErrorAction Ignore) { + if (Should-InstallCACertificatesRefreshTask -Location $Location) { + Register-CACertificatesRefreshTask -Location $Location + } + } elseif (Get-Command -Name Register-CACertificatesRefreshTask -ErrorAction Ignore) { Register-CACertificatesRefreshTask -Location $Location } diff --git a/spec/parts/linux/cloud-init/artifacts/init_aks_custom_cloud_spec.sh b/spec/parts/linux/cloud-init/artifacts/init_aks_custom_cloud_spec.sh index f85f580a8cc..8b54975d51b 100644 --- a/spec/parts/linux/cloud-init/artifacts/init_aks_custom_cloud_spec.sh +++ b/spec/parts/linux/cloud-init/artifacts/init_aks_custom_cloud_spec.sh @@ -18,7 +18,7 @@ Describe 'init-aks-custom-cloud.sh refresh mode wiring' When run grep -Eq '^location_normalized="\$\{refresh_location,,\}"$' "$script_path" The status should eq 0 - When run grep -Eq 'ussec\*\|usnat\*\) cert_endpoint_mode="legacy"' "$script_path" + When run grep -Eq 'ussec\*|usnat\*\) cert_endpoint_mode="legacy"' "$script_path" The status should eq 0 End @@ -26,10 +26,10 @@ Describe 'init-aks-custom-cloud.sh refresh mode wiring' When run grep -Eq '^install_ca_refresh_schedule=0$' "$script_path" The status should eq 0 - When run grep -Eq '^\s*install_ca_refresh_schedule=1$' "$script_path" + When run grep -Eq '^[[:space:]]*install_ca_refresh_schedule=1$' "$script_path" The status should eq 0 - When run grep -Eq '^\s*if \[ "\$install_ca_refresh_schedule" -eq 1 \]; then$' "$script_path" + When run grep -Eq '^[[:space:]]*if \[ "\$install_ca_refresh_schedule" -eq 1 \]; then$' "$script_path" The status should eq 0 End @@ -37,12 +37,12 @@ Describe 'init-aks-custom-cloud.sh refresh mode wiring' When run grep -Eq '^if \[ "\$action" = "ca-refresh" \]; then$' "$script_path" The status should eq 0 - When run grep -Eq '^\s*exit$' "$script_path" + When run grep -Eq '^[[:space:]]*exit$' "$script_path" The status should eq 0 End It 'passes LOCATION directly into cron refresh command' - When run grep -Eq 'ca-refresh \\\\"\$LOCATION\\\\"' "$script_path" + When run grep -Eq 'ca-refresh \\"\$LOCATION\\"' "$script_path" The status should eq 0 End diff --git a/staging/cse/windows/kubernetesfunc.tests.ps1 b/staging/cse/windows/kubernetesfunc.tests.ps1 index 948cd229dc0..8b062a273d0 100644 --- a/staging/cse/windows/kubernetesfunc.tests.ps1 +++ b/staging/cse/windows/kubernetesfunc.tests.ps1 @@ -74,7 +74,6 @@ Describe 'Register-CACertificatesRefreshTask' { $script:lastScheduledTaskArgument = $null Mock Logs-To-Event - Mock Write-Log Mock New-ScheduledTaskPrincipal -MockWith { return @{ Kind = 'principal' } } Mock New-JobTrigger -MockWith { return @{ Kind = 'trigger' } } Mock New-ScheduledTask -MockWith { return @{ Kind = 'definition' } } @@ -107,7 +106,6 @@ Describe 'Register-CACertificatesRefreshTask' { Describe 'Should-InstallCACertificatesRefreshTask' { BeforeEach { - Mock Write-Log } It 'returns true for legacy regions without calling the opt-in endpoint' { @@ -143,7 +141,6 @@ Describe 'Should-InstallCACertificatesRefreshTask' { Describe 'Get-CACertificates' { BeforeEach { - Mock Write-Log Mock Create-Directory -MockWith { param($FullPath, $DirectoryUsage) if (-not (Test-Path $FullPath)) { From df59b94d424bfc60798f8340e9084d2791b884b4 Mon Sep 17 00:00:00 2001 From: Ramkumar Chinchani Date: Thu, 19 Mar 2026 23:27:10 -0700 Subject: [PATCH 08/11] feat: update tests for certificate endpoint mode handling and refresh schedule installation --- .../artifacts/init_aks_custom_cloud_spec.sh | 12 ++++++++++-- staging/cse/windows/kubernetesfunc.tests.ps1 | 7 +++++-- 2 files changed, 15 insertions(+), 4 deletions(-) diff --git a/spec/parts/linux/cloud-init/artifacts/init_aks_custom_cloud_spec.sh b/spec/parts/linux/cloud-init/artifacts/init_aks_custom_cloud_spec.sh index 8b54975d51b..58812659856 100644 --- a/spec/parts/linux/cloud-init/artifacts/init_aks_custom_cloud_spec.sh +++ b/spec/parts/linux/cloud-init/artifacts/init_aks_custom_cloud_spec.sh @@ -17,26 +17,34 @@ Describe 'init-aks-custom-cloud.sh refresh mode wiring' It 'always derives cert endpoint mode from refresh_location' When run grep -Eq '^location_normalized="\$\{refresh_location,,\}"$' "$script_path" The status should eq 0 + End + It 'maps ussec/usnat locations to legacy cert endpoint mode' When run grep -Eq 'ussec\*|usnat\*\) cert_endpoint_mode="legacy"' "$script_path" The status should eq 0 End - It 'installs refresh schedule only for legacy mode or opted-in rcv1p mode' + It 'initializes refresh schedule installation as disabled' When run grep -Eq '^install_ca_refresh_schedule=0$' "$script_path" The status should eq 0 + End + It 'enables refresh schedule installation for eligible certificate modes' When run grep -Eq '^[[:space:]]*install_ca_refresh_schedule=1$' "$script_path" The status should eq 0 + End + It 'gates refresh schedule installation on install_ca_refresh_schedule' When run grep -Eq '^[[:space:]]*if \[ "\$install_ca_refresh_schedule" -eq 1 \]; then$' "$script_path" The status should eq 0 End - It 'exits early in ca-refresh mode after certificate refresh logic' + It 'checks for ca-refresh mode after certificate refresh logic' When run grep -Eq '^if \[ "\$action" = "ca-refresh" \]; then$' "$script_path" The status should eq 0 + End + It 'exits early in ca-refresh mode after certificate refresh logic' When run grep -Eq '^[[:space:]]*exit$' "$script_path" The status should eq 0 End diff --git a/staging/cse/windows/kubernetesfunc.tests.ps1 b/staging/cse/windows/kubernetesfunc.tests.ps1 index 8b062a273d0..42e15c4fc25 100644 --- a/staging/cse/windows/kubernetesfunc.tests.ps1 +++ b/staging/cse/windows/kubernetesfunc.tests.ps1 @@ -48,8 +48,11 @@ function Register-ScheduledTask { param($TaskName, $InputObject) } -. $PSScriptRoot\..\..\..\parts\windows\windowscsehelper.ps1 -. $PSCommandPath.Replace('.tests.ps1', '.ps1') +$helperScriptPath = Join-Path $PSScriptRoot '..\..\..\parts\windows\windowscsehelper.ps1' +$scriptUnderTestPath = Join-Path $PSScriptRoot 'kubernetesfunc.ps1' + +. $helperScriptPath +. $scriptUnderTestPath Describe 'Get-CustomCloudCertEndpointModeFromLocation' { It 'returns legacy for ussec regions' { From 28bf1ccf7344deabb0ef077540fee47f06300f8d Mon Sep 17 00:00:00 2001 From: Ramkumar Chinchani Date: Fri, 20 Mar 2026 07:42:47 -0700 Subject: [PATCH 09/11] feat: refactor test setup functions for improved readability and consistency --- staging/cse/windows/kubernetesfunc.tests.ps1 | 102 +++++++++---------- 1 file changed, 51 insertions(+), 51 deletions(-) diff --git a/staging/cse/windows/kubernetesfunc.tests.ps1 b/staging/cse/windows/kubernetesfunc.tests.ps1 index 42e15c4fc25..3f9f403666b 100644 --- a/staging/cse/windows/kubernetesfunc.tests.ps1 +++ b/staging/cse/windows/kubernetesfunc.tests.ps1 @@ -1,58 +1,64 @@ -if (-not (Get-PSDrive -Name C -ErrorAction SilentlyContinue)) { - New-PSDrive -Name C -PSProvider FileSystem -Root ([System.IO.Path]::GetTempPath()) | Out-Null -} +BeforeAll { + if (-not (Get-PSDrive -Name C -ErrorAction SilentlyContinue)) { + New-PSDrive -Name C -PSProvider FileSystem -Root ([System.IO.Path]::GetTempPath()) | Out-Null + } -function Write-Log { - param($Message) - Write-Host "$Message" -} + function Write-Log { + param($Message) + Write-Host "$Message" + } -function Logs-To-Event { - param($TaskName, $TaskMessage) - Write-Host "$TaskName $TaskMessage" -} + function Logs-To-Event { + param($TaskName, $TaskMessage) + Write-Host "$TaskName $TaskMessage" + } -function Set-ExitCode { - param($ExitCode, $ErrorMessage) - throw "Unexpected Set-ExitCode: $ExitCode $ErrorMessage" -} + function Set-ExitCode { + param($ExitCode, $ErrorMessage) + throw "Unexpected Set-ExitCode: $ExitCode $ErrorMessage" + } -function Create-Directory { - param($FullPath, $DirectoryUsage) - if (-not (Test-Path $FullPath)) { - New-Item -Path $FullPath -ItemType Directory -Force | Out-Null + function Create-Directory { + param($FullPath, $DirectoryUsage) + if (-not (Test-Path $FullPath)) { + New-Item -Path $FullPath -ItemType Directory -Force | Out-Null + } } -} -function Get-ScheduledTask { - param($TaskName, $ErrorAction) -} + function Get-ScheduledTask { + param($TaskName, $ErrorAction) + } -function New-ScheduledTaskAction { - param($Execute, $Argument) -} + function New-ScheduledTaskAction { + param($Execute, $Argument) + } -function New-ScheduledTaskPrincipal { - param($UserId, $LogonType, $RunLevel) -} + function New-ScheduledTaskPrincipal { + param($UserId, $LogonType, $RunLevel) + } -function New-JobTrigger { - param([switch]$Daily, $At, $DaysInterval) -} + function New-JobTrigger { + param([switch]$Daily, $At, $DaysInterval) + } -function New-ScheduledTask { - param($Action, $Principal, $Trigger, $Description) -} + function New-ScheduledTask { + param($Action, $Principal, $Trigger, $Description) + } -function Register-ScheduledTask { - param($TaskName, $InputObject) -} + function Register-ScheduledTask { + param($TaskName, $InputObject) + } + + function Retry-Command { + param($Command, $Args, $Retries, $RetryDelaySeconds) + } -$helperScriptPath = Join-Path $PSScriptRoot '..\..\..\parts\windows\windowscsehelper.ps1' -$scriptUnderTestPath = Join-Path $PSScriptRoot 'kubernetesfunc.ps1' + $helperScriptPath = Join-Path $PSScriptRoot '..\..\..\parts\windows\windowscsehelper.ps1' + $scriptUnderTestPath = Join-Path $PSScriptRoot 'kubernetesfunc.ps1' -. $helperScriptPath -. $scriptUnderTestPath + . $helperScriptPath + . $scriptUnderTestPath +} Describe 'Get-CustomCloudCertEndpointModeFromLocation' { It 'returns legacy for ussec regions' { @@ -76,11 +82,11 @@ Describe 'Register-CACertificatesRefreshTask' { BeforeEach { $script:lastScheduledTaskArgument = $null - Mock Logs-To-Event + Mock Logs-To-Event -MockWith { } Mock New-ScheduledTaskPrincipal -MockWith { return @{ Kind = 'principal' } } Mock New-JobTrigger -MockWith { return @{ Kind = 'trigger' } } Mock New-ScheduledTask -MockWith { return @{ Kind = 'definition' } } - Mock Register-ScheduledTask + Mock Register-ScheduledTask -MockWith { } Mock New-ScheduledTaskAction -MockWith { param($Execute, $Argument) $script:lastScheduledTaskArgument = $Argument @@ -109,6 +115,7 @@ Describe 'Register-CACertificatesRefreshTask' { Describe 'Should-InstallCACertificatesRefreshTask' { BeforeEach { + Mock Retry-Command -MockWith { } } It 'returns true for legacy regions without calling the opt-in endpoint' { @@ -144,13 +151,6 @@ Describe 'Should-InstallCACertificatesRefreshTask' { Describe 'Get-CACertificates' { BeforeEach { - Mock Create-Directory -MockWith { - param($FullPath, $DirectoryUsage) - if (-not (Test-Path $FullPath)) { - New-Item -Path $FullPath -ItemType Directory -Force | Out-Null - } - } - if (Test-Path 'C:\ca') { Remove-Item -Path 'C:\ca' -Recurse -Force } From 5b0108bf1ef9b2662c71d454859e71eeae8216bb Mon Sep 17 00:00:00 2001 From: Ramkumar Chinchani Date: Fri, 20 Mar 2026 08:52:23 -0700 Subject: [PATCH 10/11] feat: update Get-CustomCloudCertEndpointModeFromLocation to clarify endpoint mode handling for legacy and rcv1p regions --- staging/cse/windows/kubernetesfunc.ps1 | 2 ++ staging/cse/windows/kubernetesfunc.tests.ps1 | 20 ++++++++++---------- 2 files changed, 12 insertions(+), 10 deletions(-) diff --git a/staging/cse/windows/kubernetesfunc.ps1 b/staging/cse/windows/kubernetesfunc.ps1 index 76b59d2e584..1dcfc6ed6c0 100644 --- a/staging/cse/windows/kubernetesfunc.ps1 +++ b/staging/cse/windows/kubernetesfunc.ps1 @@ -249,11 +249,13 @@ function Get-CustomCloudCertEndpointModeFromLocation { $Location ) + # ussec/usnat regions still use the legacy certificate endpoint contract. $normalizedLocation = $Location.ToLowerInvariant() if ($normalizedLocation.StartsWith("ussec") -or $normalizedLocation.StartsWith("usnat")) { return "legacy" } + # All other regions use the rcv1p endpoint mode with opt-in gating. return "rcv1p" } diff --git a/staging/cse/windows/kubernetesfunc.tests.ps1 b/staging/cse/windows/kubernetesfunc.tests.ps1 index 3f9f403666b..2e95cef1338 100644 --- a/staging/cse/windows/kubernetesfunc.tests.ps1 +++ b/staging/cse/windows/kubernetesfunc.tests.ps1 @@ -62,19 +62,19 @@ BeforeAll { Describe 'Get-CustomCloudCertEndpointModeFromLocation' { It 'returns legacy for ussec regions' { - Get-CustomCloudCertEndpointModeFromLocation -Location 'ussecwest' | Should Be 'legacy' + Get-CustomCloudCertEndpointModeFromLocation -Location 'ussecwest' | Should -Be 'legacy' } It 'returns legacy for usnat regions' { - Get-CustomCloudCertEndpointModeFromLocation -Location 'usnatcentral' | Should Be 'legacy' + Get-CustomCloudCertEndpointModeFromLocation -Location 'usnatcentral' | Should -Be 'legacy' } It 'returns rcv1p for public regions' { - Get-CustomCloudCertEndpointModeFromLocation -Location 'southcentralus' | Should Be 'rcv1p' + Get-CustomCloudCertEndpointModeFromLocation -Location 'southcentralus' | Should -Be 'rcv1p' } It 'handles mixed-case input' { - Get-CustomCloudCertEndpointModeFromLocation -Location 'UsSeCeast' | Should Be 'legacy' + Get-CustomCloudCertEndpointModeFromLocation -Location 'UsSeCeast' | Should -Be 'legacy' } } @@ -109,7 +109,7 @@ Describe 'Register-CACertificatesRefreshTask' { Register-CACertificatesRefreshTask -Location 'southcentralus' Assert-MockCalled -CommandName Register-ScheduledTask -Exactly -Times 1 - $script:lastScheduledTaskArgument | Should Match ([regex]::Escape("Get-CACertificates -Location 'southcentralus'")) + $script:lastScheduledTaskArgument | Should -Match ([regex]::Escape("Get-CACertificates -Location 'southcentralus'")) } } @@ -123,7 +123,7 @@ Describe 'Should-InstallCACertificatesRefreshTask' { $result = Should-InstallCACertificatesRefreshTask -Location 'ussecwest' - $result | Should Be $true + $result | Should -Be $true Assert-MockCalled -CommandName Retry-Command -Exactly -Times 0 } @@ -134,7 +134,7 @@ Describe 'Should-InstallCACertificatesRefreshTask' { $result = Should-InstallCACertificatesRefreshTask -Location 'southcentralus' - $result | Should Be $true + $result | Should -Be $true Assert-MockCalled -CommandName Retry-Command -Exactly -Times 1 -ParameterFilter { $Args.Uri -eq 'http://168.63.129.16/acms/isOptedInForRootCerts' } } @@ -145,7 +145,7 @@ Describe 'Should-InstallCACertificatesRefreshTask' { $result = Should-InstallCACertificatesRefreshTask -Location 'southcentralus' - $result | Should Be $false + $result | Should -Be $false } } @@ -166,7 +166,7 @@ Describe 'Get-CACertificates' { $result = Get-CACertificates -Location 'ussecwest' - $result | Should Be $true + $result | Should -Be $true Assert-MockCalled -CommandName Retry-Command -Exactly -Times 1 -ParameterFilter { $Args.Uri -eq 'http://168.63.129.16/machine?comp=acmspackage&type=cacertificates&ext=json' } Assert-MockCalled -CommandName Retry-Command -Exactly -Times 0 -ParameterFilter { $Args.Uri -eq 'http://168.63.129.16/acms/isOptedInForRootCerts' } } @@ -178,6 +178,6 @@ Describe 'Get-CACertificates' { $result = Get-CACertificates -Location 'southcentralus' - $result | Should Be $false + $result | Should -Be $false } } From e330307ea44ca2f775f5a7e0873ea9ad63889bf2 Mon Sep 17 00:00:00 2001 From: Ramkumar Chinchani Date: Fri, 20 Mar 2026 09:52:25 -0700 Subject: [PATCH 11/11] feat: enhance tests for Should-InstallCACertificatesRefreshTask and Get-CACertificates to verify URI handling --- staging/cse/windows/kubernetesfunc.tests.ps1 | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/staging/cse/windows/kubernetesfunc.tests.ps1 b/staging/cse/windows/kubernetesfunc.tests.ps1 index 2e95cef1338..8186bfabc4c 100644 --- a/staging/cse/windows/kubernetesfunc.tests.ps1 +++ b/staging/cse/windows/kubernetesfunc.tests.ps1 @@ -128,14 +128,18 @@ Describe 'Should-InstallCACertificatesRefreshTask' { } It 'returns true for rcv1p regions when opt-in is enabled' { + $script:lastRetryUri = $null Mock Retry-Command -MockWith { + param($Command, $Args, $Retries, $RetryDelaySeconds) + $script:lastRetryUri = $PSBoundParameters['Args'].Uri return [PSCustomObject]@{ Content = 'IsOptedInForRootCerts=true' } } $result = Should-InstallCACertificatesRefreshTask -Location 'southcentralus' $result | Should -Be $true - Assert-MockCalled -CommandName Retry-Command -Exactly -Times 1 -ParameterFilter { $Args.Uri -eq 'http://168.63.129.16/acms/isOptedInForRootCerts' } + Assert-MockCalled -CommandName Retry-Command -Exactly -Times 1 + $script:lastRetryUri | Should -Be 'http://168.63.129.16/acms/isOptedInForRootCerts' } It 'returns false for rcv1p regions when opt-in is disabled' { @@ -157,8 +161,10 @@ Describe 'Get-CACertificates' { } It 'uses the legacy endpoint when location is a ussec/usnat region' { + $script:retryUris = @() Mock Retry-Command -MockWith { param($Command, $Args, $Retries, $RetryDelaySeconds) + $script:retryUris += $PSBoundParameters['Args'].Uri return [PSCustomObject]@{ Content = '{"Certificates":[{"Name":"legacy.crt","CertBody":"legacy-body"}]}' } @@ -167,8 +173,9 @@ Describe 'Get-CACertificates' { $result = Get-CACertificates -Location 'ussecwest' $result | Should -Be $true - Assert-MockCalled -CommandName Retry-Command -Exactly -Times 1 -ParameterFilter { $Args.Uri -eq 'http://168.63.129.16/machine?comp=acmspackage&type=cacertificates&ext=json' } - Assert-MockCalled -CommandName Retry-Command -Exactly -Times 0 -ParameterFilter { $Args.Uri -eq 'http://168.63.129.16/acms/isOptedInForRootCerts' } + Assert-MockCalled -CommandName Retry-Command -Exactly -Times 1 + $script:retryUris | Should -Contain 'http://168.63.129.16/machine?comp=acmspackage&type=cacertificates&ext=json' + $script:retryUris | Should -Not -Contain 'http://168.63.129.16/acms/isOptedInForRootCerts' } It 'returns false when certificate retrieval throws' {