Delete EKS Cluster v1.25 and update to v1.32

eks_infrastructure/build_param.json

@@ -1,12 +1,12 @@
 {
-    "operation": "new_operation",
+    "operation": "delete",
     "contexts": [
         "MAINLINE"
     ],
     "eks_clusters": [
         "dlc-pytorch",
         "dlc-tensorflow"
     ],
-    "eks_version": "1.25",
-    "cluster_autoscalar_image_version": "v1.25.0"
+    "eks_version": "1.32",
+    "cluster_autoscalar_image_version": "v1.32.0"
 }

eks_infrastructure/cluster-autoscalar-autodiscover.yaml

@@ -138,7 +138,7 @@ spec:
     spec:
       serviceAccountName: cluster-autoscaler
       containers:
-        - image: k8s.gcr.io/autoscaling/cluster-autoscaler:<VERSION>
+        - image: registry.k8s.io/autoscaling/cluster-autoscaler:<VERSION>
           name: cluster-autoscaler
           resources:
             limits:
@@ -166,4 +166,4 @@ spec:
       volumes:
         - name: ssl-certs
           hostPath:
-            path: "/etc/ssl/certs/ca-bundle.crt"
+            path: "/etc/ssl/certs/ca-bundle.crt"

eks_infrastructure/create_cluster.sh

@@ -35,8 +35,9 @@ function create_eks_cluster() {
 function create_node_group() {
 
   STATIC_NODEGROUP_INSTANCE_TYPE="m5.large"
-  GPU_NODEGROUP_INSTANCE_TYPE="p3.16xlarge"
+  GPU_NODEGROUP_INSTANCE_TYPE="g5.24xlarge"
   INF_NODEGROUP_INSTANCE_TYPE="inf1.xlarge"
+  GRAVITON_NODEGROUP_INSTANCE_TYPE="c6g.4xlarge"
 
   # static nodegroup
   eksctl create nodegroup \
@@ -78,6 +79,36 @@ function create_node_group() {
     --ssh-access \
     --ssh-public-key "${3}"
 
+  # dynamic graviton nodegroup
+  eksctl create nodegroup \
+    --name ${1}-graviton-nodegroup-${2/./-} \
+    --cluster ${1} \
+    --node-type ${GRAVITON_NODEGROUP_INSTANCE_TYPE} \
+    --nodes-min 0 \
+    --nodes-max 100 \
+    --node-volume-size 80 \
+    --node-labels "test_type=graviton" \
+    --tags "k8s.io/cluster-autoscaler/node-template/label/test_type=graviton" \
+    --asg-access \
+    --managed=true \
+    --ssh-access \
+    --ssh-public-key "${3}"
+}
+
+#Function to upgrade core k8s components
+function update_eksctl_utils() {
+  LIST_ADDONS=$(eksctl get addon --cluster ${CLUSTER}  -o json | jq -r '.[].Name')
+
+  if [ -n "${LIST_ADDONS}" ]; then
+    for ADDONS in ${LIST_ADDONS}; do
+      eksctl update addon \
+        --name ${ADDONS} \
+        --cluster ${1} \
+        --region ${2}
+    done
+  else
+    echo "No addons present in the EKS cluster ${CLUSTER}"
+  fi
 }
 
 # Attach IAM policy to nodegroup IAM role
@@ -148,6 +179,10 @@ function add_tags_asg() {
         ResourceId=${asg_name},ResourceType=auto-scaling-group,Key=k8s.io/cluster-autoscaler/node-template/resources/hugepages-2Mi,Value=256Mi,PropagateAtLaunch=true
     fi
 
+    if [[ ${nodegroup_name} == *"graviton"* ]]; then
+      aws autoscaling create-or-update-tags \
+        --tags ResourceId=${asg_name},ResourceType=auto-scaling-group,Key=k8s.io/cluster-autoscaler/node-template/label/test_type,Value=graviton,PropagateAtLaunch=true
+    fi
   done
 
 }
@@ -185,4 +220,5 @@ create_eks_cluster ${CLUSTER} ${EKS_VERSION} ${AWS_REGION}
 create_node_group ${CLUSTER} ${EKS_VERSION} ${EC2_KEY_PAIR_NAME}
 add_tags_asg ${CLUSTER} ${AWS_REGION}
 add_iam_permissions_nodegroup ${CLUSTER} ${AWS_REGION}
-create_namespaces
+create_namespaces
+update_eksctl_utils ${CLUSTER} ${AWS_REGION}

eks_infrastructure/env_setup.sh

@@ -4,11 +4,11 @@
 set -ex
 
 # The below url/version is based on EKS v1.25.6. The same needs to be updated for EKS version upgrade.
-KUBECTL_CLIENT="https://s3.us-west-2.amazonaws.com/amazon-eks/1.25.6/2023-01-30/bin/linux/amd64/kubectl"
-EKSCTL_CLIENT="https://github.com/weaveworks/eksctl/releases/latest/download/eksctl_$(uname -s)_amd64.tar.gz"
-AWS_IAM_AUTHENTICATOR="https://github.com/kubernetes-sigs/aws-iam-authenticator/releases/download/v0.6.2/aws-iam-authenticator_0.6.2_linux_amd64"
+KUBECTL_CLIENT="https://s3.us-west-2.amazonaws.com/amazon-eks/1.32.0/2024-12-20/bin/linux/amd64/kubectl"
+EKSCTL_CLIENT="https://github.com/eksctl-io/eksctl/releases/latest/download/eksctl_$(uname -s)_amd64.tar.gz"
+AWS_IAM_AUTHENTICATOR="https://github.com/kubernetes-sigs/aws-iam-authenticator/releases/download/v0.6.29/aws-iam-authenticator_0.6.29_linux_amd64"
 
-LATEST_KUBECTL_CLIENT_VERSION=1.25
+LATEST_KUBECTL_CLIENT_VERSION=1.32
 
 function install_kubectl_client() {
     curl --silent --location ${KUBECTL_CLIENT} -o /usr/local/bin/kubectl
@@ -38,7 +38,7 @@ else
 fi
 
 #kubectl version
-kubectl version --short --client
+kubectl version --client
 
 # install eksctl
 if ! [ -x "$(command -v eksctl)" ]; then

eks_infrastructure/helper.sh

@@ -35,18 +35,27 @@ function create_cluster() {
 # 4. Upgrade core k8s components
 # 5. Scale cluster autoscalar back to 1
 function upgrade_cluster() {
+  # Initialize error log and ensure log does not exist
+  ERROR_LOG="failed_nodegroups.log"
+  rm -f ${ERROR_LOG}
+
   TARGET="CLUSTER"
   for CONTEXT in "${CONTEXTS[@]}"; do
     for CLUSTER in "${EKS_CLUSTERS[@]}"; do
       CLUSTER_NAME=${CLUSTER}-${CONTEXT}
       if check_cluster_status $CLUSTER_NAME; then
-        ./upgrade_operation.sh $TARGET $CLUSTER_NAME $EKS_VERSION $CLUSTER_AUTOSCALAR_IMAGE_VERSION
+        ./upgrade_operation.sh $TARGET $CLUSTER_NAME $EKS_VERSION $ERROR_LOG $CLUSTER_AUTOSCALAR_IMAGE_VERSION
       else
         echo "EKS Cluster :: ${CLUSTER_NAME} :: does not exists. Skipping upgrade operation."
       fi
     done
   done
 
+  if [ -f ${ERROR_LOG} ]; then
+    echo "The following nodegroups failed to upgrade."
+    cat ${ERROR_LOG}
+    exit 1
+  fi
 }
 
 # Upgrade nodegroup operation function

eks_infrastructure/new_operation.sh

@@ -105,20 +105,18 @@ function create_graviton_node_group() {
 
 #Function to upgrade core k8s components
 function update_eksctl_utils() {
-  eksctl utils update-kube-proxy \
-    --cluster ${1} \
-    --region ${2} \
-    --approve
-
-  eksctl utils update-aws-node \
-    --cluster ${1} \
-    --region ${2} \
-    --approve
-
-  eksctl utils update-coredns \
-    --cluster ${1} \
-    --region ${2} \
-    --approve
+  LIST_ADDONS=$(eksctl get addon --cluster ${CLUSTER}  -o json | jq -r '.[].Name')
+
+  if [ -n "${LIST_ADDONS}" ]; then
+    for ADDONS in ${LIST_ADDONS}; do
+      eksctl update addon \
+        --name ${ADDONS} \
+        --cluster ${1} \
+        --region ${2}
+    done
+  else
+    echo "No addons present in the EKS cluster ${CLUSTER}"
+  fi
 }
 
 if [ $# -ne 2 ]; then
@@ -155,4 +153,4 @@ fi
 # update_eksctl_utils ${CLUSTER} ${AWS_REGION}
 create_graviton_node_group ${CLUSTER} ${EKS_VERSION} ${EC2_KEY_PAIR_NAME}
 add_tags_asg ${CLUSTER} ${AWS_REGION}
-add_iam_permissions_nodegroup ${CLUSTER} ${AWS_REGION}
+add_iam_permissions_nodegroup ${CLUSTER} ${AWS_REGION}

eks_infrastructure/upgrade_operation.sh

@@ -66,20 +66,18 @@ function upgrade_nodegroups() {
 
 #Function to upgrade core k8s components
 function update_eksctl_utils() {
-  eksctl utils update-kube-proxy \
-    --cluster ${1} \
-    --region ${2} \
-    --approve
-
-  eksctl utils update-aws-node \
-    --cluster ${1} \
-    --region ${2} \
-    --approve
-
-  eksctl utils update-coredns \
-    --cluster ${1} \
-    --region ${2} \
-    --approve
+  LIST_ADDONS=$(eksctl get addon --cluster ${CLUSTER}  -o json | jq -r '.[].Name')
+
+  if [ -n "${LIST_ADDONS}" ]; then
+    for ADDONS in ${LIST_ADDONS}; do
+      eksctl update addon \
+        --name ${ADDONS} \
+        --cluster ${1} \
+        --region ${2}
+    done
+  else
+    echo "No addons present in the EKS cluster ${CLUSTER}"
+  fi
 }
 
 if [ $# -lt 3 ]; then
@@ -95,11 +93,8 @@ fi
 TARGET=${1}
 CLUSTER=${2}
 EKS_VERSION=${3}
-if [ "${TARGET}" = "CLUSTER" ]; then
-  CLUSTER_AUTOSCALAR_IMAGE_VERSION=${4}
-elif [ "${TARGET}" = "NODEGROUP" ]; then
-  ERROR_LOG=${4}
-fi
+ERROR_LOG=${4}
+CLUSTER_AUTOSCALAR_IMAGE_VERSION=${5}
 
 if [ -n "${EKS_CLUSTER_MANAGER_ROLE}" ]; then
   update_kubeconfig ${CLUSTER} ${EKS_CLUSTER_MANAGER_ROLE} ${AWS_REGION}
@@ -110,7 +105,7 @@ if [ "${TARGET}" = "CLUSTER" ]; then
   scale_cluster_autoscalar 0
   upgrade_autoscalar_image ${CLUSTER_AUTOSCALAR_IMAGE_VERSION}
   upgrade_eks_control_plane ${CLUSTER} ${EKS_VERSION}
-  upgrade_nodegroups ${CLUSTER} ${EKS_VERSION} ${AWS_REGION}
+  upgrade_nodegroups ${CLUSTER} ${EKS_VERSION} ${AWS_REGION} ${ERROR_LOG}
   update_eksctl_utils ${CLUSTER} ${AWS_REGION}
   #scale back to 1
   scale_cluster_autoscalar 1

test/dlc_tests/eks/eks_manifest_templates/kubeflow/install_kubeflow.sh

@@ -1,13 +1,13 @@
 #!/bin/bash
-#/ Usage: 
+#/ Usage:
 #/ cluster name is required. operation is optional
 #/ ./install_kubeflow.sh eks_cluster_name [operation]
 
 set -ex
 
 # Function to install kustomize
 install_kustomize(){
-    KUSTOMIZE_VERSION="v4.5.7"
+    KUSTOMIZE_VERSION="v5.5.0"
     KUSTOMIZE_URL="https://github.com/kubernetes-sigs/kustomize/releases/download/kustomize%2F${KUSTOMIZE_VERSION}/kustomize_${KUSTOMIZE_VERSION}_linux_amd64.tar.gz"
 
     if ! command -v kustomize &> /dev/null
@@ -42,17 +42,17 @@ uninstall_kubeflow(){
 
 # Function to create directory and download kubeflow components
 setup_kubeflow(){
-    KUBEFLOW_VERSION="v1.7.0"
+    KUBEFLOW_VERSION="v1.9.1"
     local EKS_CLUSTER_NAME=$1
     DIRECTORY="${HOME}/${EKS_CLUSTER_NAME}"
 
     if [ -d "${DIRECTORY}" ]; then
         rm -rf ${DIRECTORY};
     fi
-        
-    mkdir ${DIRECTORY} 
+
+    mkdir ${DIRECTORY}
     cd ${DIRECTORY}
-    
+
     # clones manifests from kubeflow github into a folder named manifests
     git clone -b ${KUBEFLOW_VERSION} --single-branch https://github.com/kubeflow/manifests.git
 }

test/test_utils/eks.py

@@ -42,9 +42,9 @@
 LOGGER.addHandler(logging.StreamHandler(sys.stderr))
 
 
-EKS_VERSION = "1.20.4"
-EKSCTL_VERSION = "0.53.0"
-KUBETAIL_VERSION = "1.6.7"
+EKS_VERSION = "1.32.0"
+EKSCTL_VERSION = "0.203.0"
+KUBETAIL_VERSION = "1.6.20"
 
 
 def get_aws_secret_yml_path():
@@ -152,7 +152,7 @@ def setup_eksctl():
 
     platform = run("uname -s", echo=True).stdout.strip()
     eksctl_download_command = (
-        f"curl --silent --location https://github.com/weaveworks/eksctl/releases/download/"
+        f"curl --silent --location https://github.com/eksctl-io/eksctl/releases/download/v"
         f"{EKSCTL_VERSION}/eksctl_{platform}_amd64.tar.gz | tar xz -C /tmp"
     )
     run(eksctl_download_command, echo=True)
@@ -169,7 +169,7 @@ def eks_setup():
 
     # Run a quick check that the binaries are available in the PATH by listing the 'version'
     run_out = run(
-        "eksctl version && kubectl version --short --client && aws-iam-authenticator version",
+        "eksctl version && kubectl version --client && aws-iam-authenticator version",
         warn=True,
     )
 
@@ -182,12 +182,12 @@ def eks_setup():
 
     kubectl_download_command = (
         f"curl --silent --location https://amazon-eks.s3-us-west-2.amazonaws.com/"
-        f"{EKS_VERSION}/2021-04-12/bin/{platform.lower()}/amd64/kubectl -o /usr/local/bin/kubectl"
+        f"{EKS_VERSION}/2024-12-20/bin/{platform.lower()}/amd64/kubectl -o /usr/local/bin/kubectl"
     )
 
     aws_iam_authenticator_download_command = (
         f"curl --silent --location https://amazon-eks.s3-us-west-2.amazonaws.com/"
-        f"{EKS_VERSION}/2021-04-12/bin/{platform.lower()}/amd64/aws-iam-authenticator "
+        f"{EKS_VERSION}/2024-12-20/bin/{platform.lower()}/amd64/aws-iam-authenticator "
         f"-o /usr/local/bin/aws-iam-authenticator"
     )
 
@@ -210,7 +210,7 @@ def eks_setup():
 
     # Run a quick check that the binaries are available in the PATH by listing the 'version'
     run("eksctl version", echo=True)
-    run("kubectl version --short --client", echo=True)
+    run("kubectl version --client", echo=True)
     run("aws-iam-authenticator version", echo=True)