eks-performance-cluster-scaling.yml

# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
# SPDX-License-Identifier: MIT
name: EKS Cluster Scaling

on:
  schedule:
    - cron: '0 9 * * 0'  # Scale up: Runs every Sunday at 9:00 AM
    - cron: '0 21 * * 1'  # Scale down: Runs every Monday at 9:00 PM
  workflow_dispatch:
    inputs:
      region:
        description: 'AWS Region'
        required: true
        type: string
        default: 'us-west-2'
      cluster_name:
        description: 'EKS Cluster Name'
        required: true
        type: string
        default: 'eks-performance'
      desired_capacity_per_nodegroup:
        description: 'Desired capacity for each node group'
        required: true
        type: number
        default: 500
      node_group_count:
        description: 'Count of node groups'
        type: number
        default: 10
      leader_node_desired_capacity:
        description: 'Desired capacity for leader node group (manual execution only)'
        type: number
        default: 1

env:
  AWS_REGION: ${{ inputs.region || 'us-west-2' }}
  CLUSTER_NAME: ${{ inputs.cluster_name || 'eks-performance' }}
  NODE_GROUP_COUNT: ${{ inputs.node_group_count || 10 }}
  DESIRED_CAPACITY_PER_NODEGROUP: ${{ inputs.desired_capacity_per_nodegroup || 500 }}
  LEADER_NODE_DESIRED_CAPACITY: ${{ inputs.leader_node_desired_capacity || 1 }}
  TERRAFORM_AWS_ASSUME_ROLE: ${{ vars.TERRAFORM_AWS_ASSUME_ROLE }}
  TERRAFORM_AWS_ASSUME_ROLE_DURATION: 14400  # 4 hour duration
  CWA_GITHUB_TEST_REPO_NAME: "aws/amazon-cloudwatch-agent-test"
  CWA_GITHUB_TEST_REPO_URL: "https://github.com/aws/amazon-cloudwatch-agent-test.git"
  CWA_GITHUB_TEST_REPO_BRANCH: "main"

jobs:
  scale-eks-cluster:
    runs-on: ubuntu-latest
    permissions:
      id-token: write
      contents: read
    steps:
      - uses: actions/checkout@v3
        with:
          repository: ${{ env.CWA_GITHUB_TEST_REPO_NAME }}
          ref: ${{ env.CWA_GITHUB_TEST_REPO_BRANCH }}

      - name: Configure AWS Credentials
        uses: aws-actions/configure-aws-credentials@v4
        with:
          role-to-assume: ${{ env.TERRAFORM_AWS_ASSUME_ROLE }}
          aws-region: ${{ inputs.region || 'us-west-2' }}
          role-duration-seconds: ${{ env.TERRAFORM_AWS_ASSUME_ROLE_DURATION }}

      - name: Install kubectl
        uses: azure/setup-kubectl@v3
        with:
          version: 'latest'

      - name: Update kubeconfig for EKS cluster
        run: |
          aws eks update-kubeconfig --name $CLUSTER_NAME --region $AWS_REGION

      - name: Scale up node groups (Sunday)
        if: github.event.schedule == '0 9 * * 0'
        run: |
          echo "Starting scale UP operation with desired capacity: $DESIRED_CAPACITY_PER_NODEGROUP"
          
          # Scale leader node to 1
          echo "Scaling leader node group: $CLUSTER_NAME-leader-node to 1"
          aws eks update-nodegroup-config \
            --cluster-name $CLUSTER_NAME \
            --nodegroup-name $CLUSTER_NAME-leader-node \
            --region $AWS_REGION \
            --scaling-config desiredSize=1
          
          echo "Waiting 1 minute before scaling regular node groups..."
          sleep 60
          
          for i in $(seq 1 $NODE_GROUP_COUNT); do
            echo "Scaling node group: $CLUSTER_NAME-node-${i} to $DESIRED_CAPACITY_PER_NODEGROUP"
            aws eks update-nodegroup-config \
              --cluster-name $CLUSTER_NAME \
              --nodegroup-name $CLUSTER_NAME-node-${i} \
              --region $AWS_REGION \
              --scaling-config desiredSize=$DESIRED_CAPACITY_PER_NODEGROUP
          
            echo "Waiting 1 minute before scaling next node group..."
            sleep 60
          done

      - name: Scale down node groups (Monday)
        if: github.event.schedule == '0 21 * * 1'
        run: |
          echo "Starting scale DOWN operation with desired capacity: 0"
          
          # Scale leader node to 0
          echo "Scaling leader node group: $CLUSTER_NAME-leader-node to 0"
          aws eks update-nodegroup-config \
            --cluster-name $CLUSTER_NAME \
            --nodegroup-name $CLUSTER_NAME-leader-node \
            --region $AWS_REGION \
            --scaling-config desiredSize=0
          
          echo "Waiting 1 minute before scaling regular node groups..."
          sleep 60
          
          for i in $(seq 1 $NODE_GROUP_COUNT); do
            echo "Scaling node group: $CLUSTER_NAME-node-${i} to 0"
            aws eks update-nodegroup-config \
              --cluster-name $CLUSTER_NAME \
              --nodegroup-name $CLUSTER_NAME-node-${i} \
              --region $AWS_REGION \
              --scaling-config desiredSize=0
          
            echo "Waiting 1 minute before scaling next node group..."
            sleep 60
          done

      - name: Scale node groups (Manual)
        if: github.event_name == 'workflow_dispatch'
        run: |
          echo "Starting manual scaling operation with desired capacity: $DESIRED_CAPACITY_PER_NODEGROUP"
          echo "Leader node desired capacity: $LEADER_NODE_DESIRED_CAPACITY"
          
          # Scale leader node to specified capacity
          echo "Scaling leader node group: $CLUSTER_NAME-leader-node to $LEADER_NODE_DESIRED_CAPACITY"
          aws eks update-nodegroup-config \
            --cluster-name $CLUSTER_NAME \
            --nodegroup-name $CLUSTER_NAME-leader-node \
            --region $AWS_REGION \
            --scaling-config desiredSize=$LEADER_NODE_DESIRED_CAPACITY
          
          echo "Waiting 1 minute before scaling regular node groups..."
          sleep 60
          
          for i in $(seq 1 $NODE_GROUP_COUNT); do
            echo "Scaling node group: $CLUSTER_NAME-node-${i} to $DESIRED_CAPACITY_PER_NODEGROUP"
            aws eks update-nodegroup-config \
              --cluster-name $CLUSTER_NAME \
              --nodegroup-name $CLUSTER_NAME-node-${i} \
              --region $AWS_REGION \
              --scaling-config desiredSize=$DESIRED_CAPACITY_PER_NODEGROUP
          
          done

      - name: Validate total node count
        run: |
          echo "Waiting 30 minutes for scaling operations to complete and stabilize..."
          sleep 1800
          
          echo "Validating total number of nodes in the cluster..."
          ACTUAL_NODE_COUNT=$(kubectl get nodes --no-headers | wc -l)
          
          # Determine expected count based on trigger type
          if [ "${{ github.event.schedule }}" = "0 21 * * 1" ]; then
            EXPECTED_NODE_COUNT=0
          else
            EXPECTED_NODE_COUNT=$(($NODE_GROUP_COUNT * $DESIRED_CAPACITY_PER_NODEGROUP + $LEADER_NODE_DESIRED_CAPACITY))
          fi
          
          echo "Expected total nodes: $EXPECTED_NODE_COUNT"
          echo "Actual total nodes: $ACTUAL_NODE_COUNT"
          
          if [ "$ACTUAL_NODE_COUNT" -eq "$EXPECTED_NODE_COUNT" ]; then
            echo "Validation successful! Node count matches expected value."
          else
            echo "Validation failed. Expected $EXPECTED_NODE_COUNT nodes but found $ACTUAL_NODE_COUNT nodes."
            exit 1
          fi