eks-performance-cluster-addon-install.yml

# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
# SPDX-License-Identifier: MIT
name: "Install/Remove Helm Charts After Scaling"
on:
  # Use workflow_run to trigger this workflow after the scaling workflow completes
  workflow_run:
    workflows: ["EKS Cluster Scaling"]
    types:
      - completed
    branches:
      - main

  # Keep the manual trigger option
  workflow_dispatch:
    inputs:
      # Required Core Settings
      cluster_name:
        description: 'EKS Cluster Name'
        required: true
        type: string
        default: 'eks-performance'
      region:
        description: 'AWS Region'
        required: true
        type: string
        default: 'us-west-2'

      # Optional Settings
      cloudwatch_agent_repository:
        description: 'CloudWatch Agent Repository'
        type: string
      cloudwatch_agent_tag:
        description: 'CloudWatch Agent Tag'
        type: string
      cloudwatch_agent_operator_repository:
        description: 'CloudWatch Agent Operator Repository'
        type: string
      cloudwatch_agent_operator_tag:
        description: 'CloudWatch Agent Operator Tag'
        type: string
      helm-charts-branch:
        description: 'Branch of the helm charts to test'
        type: string
        default: 'main'
      operator-branch:
        description: 'Branch of the operator to test'
        type: string
        default: 'main'
      terraform_assume_role:
        description: 'AWS IAM Role to assume'
        type: string

concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}
  cancel-in-progress: true

env:
  # Cluster environment variables
  AWS_REGION: ${{ inputs.region || 'us-west-2' }}
  CLUSTER_NAME: ${{ inputs.cluster_name || 'eks-performance' }}
  TERRAFORM_AWS_ASSUME_ROLE: ${{ inputs.terraform_assume_role || vars.TERRAFORM_AWS_ASSUME_ROLE }}
  TERRAFORM_AWS_ASSUME_ROLE_DURATION: 3600  # 1 hour duration

  # ECR repository environment variables
  AGENT_ECR_TEST_REPO: "cwagent-integration-test"
  OPERATOR_ECR_TEST_REPO: "cwagent-operator-pre-release"

  # Github repository environment variables
  OPERATOR_GITHUB_REPO_NAME: "aws/amazon-cloudwatch-agent-operator"
  CWA_GITHUB_TEST_REPO_NAME: "aws/amazon-cloudwatch-agent-test"
  CWA_GITHUB_TEST_REPO_BRANCH: "main"

jobs:
  # Check if this workflow should run
  check-trigger:
    runs-on: ubuntu-latest
    if: ${{ github.event_name == 'workflow_dispatch' || (github.event_name == 'workflow_run' && github.event.workflow_run.conclusion == 'success' && github.event.workflow_run.event == 'schedule') }}
    steps:
      - name: Check trigger type
        id: check-trigger
        run: |
          if [ "${{ github.event_name }}" == "workflow_run" ]; then
            echo "Triggered by workflow_run from a scheduled event"
          else
            echo "Triggered manually via workflow_dispatch"
          fi
    outputs:
      should_continue: ${{ github.event_name == 'workflow_dispatch' || (github.event_name == 'workflow_run' && github.event.workflow_run.conclusion == 'success' && github.event.workflow_run.event == 'schedule') }}

  # Get latest operator commit from github repo
  GetLatestOperatorCommitSHA:
    needs: check-trigger
    if: ${{ needs.check-trigger.outputs.should_continue == 'true' }}
    runs-on: ubuntu-latest
    outputs:
      operator_commit_sha: ${{steps.get_latest_sha.outputs.operator_sha}}
      operator_repo_name: ${{env.OPERATOR_GITHUB_REPO_NAME}}
    steps:
      - name: Checkout the target repo
        uses: actions/checkout@v4
        with:
          repository: ${{env.OPERATOR_GITHUB_REPO_NAME}}
          ref: ${{ inputs.operator-branch || 'main' }}
          path: operator-repo

      - name: Get latest commit SHA
        id: get_latest_sha
        run: |
          cd operator-repo
          latest_sha=$(git rev-parse HEAD)
          echo "operator_sha=$latest_sha" >> "$GITHUB_OUTPUT"

  # Build and upload agent image to ECR repo
  BuildAgent:
    needs: check-trigger
    if: ${{ needs.check-trigger.outputs.should_continue == 'true' }}
    uses: ./.github/workflows/build-test-artifacts.yml
    concurrency:
      group: "Build-Test-Artifacts-${{github.ref_name}}"
      cancel-in-progress: true
    secrets: inherit
    permissions:
      id-token: write
      contents: read
      actions: write
    with:
      run-tests: false

  # Build and upload operator image to ECR repo
  BuildOperator:
    needs: [ check-trigger, GetLatestOperatorCommitSHA ]
    if: ${{ needs.check-trigger.outputs.should_continue == 'true' }}
    uses: aws/amazon-cloudwatch-agent-operator/.github/workflows/build-and-upload.yml@main
    permissions:
      id-token: write
      contents: read
    concurrency:
      group: ${{ github.workflow }}-operator-${{ inputs.operator-branch || 'main' }}
      cancel-in-progress: true
    secrets: inherit
    with:
      tag: ${{needs.GetLatestOperatorCommitSHA.outputs.operator_commit_sha}}
      target-sha: ${{needs.GetLatestOperatorCommitSHA.outputs.operator_commit_sha}}
      repository: ${{needs.GetLatestOperatorCommitSHA.outputs.operator_repo_name}}
      test-image-before-upload: false

  install-helm:
    needs: [ check-trigger, BuildAgent, BuildOperator, GetLatestOperatorCommitSHA ]
    if: ${{ needs.check-trigger.outputs.should_continue == 'true' }}
    runs-on: ubuntu-latest
    permissions:
      id-token: write
      contents: read
    steps:
      - name: Configure AWS Credentials
        uses: aws-actions/configure-aws-credentials@v4
        with:
          role-to-assume: ${{ env.TERRAFORM_AWS_ASSUME_ROLE}}
          aws-region: ${{ env.AWS_REGION}}
          role-duration-seconds: ${{ env.TERRAFORM_AWS_ASSUME_ROLE_DURATION }}

      - name: Login ECR
        id: login-ecr
        uses: aws-actions/amazon-ecr-login@v2

      - name: Install kubectl
        uses: azure/setup-kubectl@v3
        with:
          version: 'latest'

      - name: Install Helm
        uses: azure/setup-helm@v3
        with:
          version: 'latest'

      - name: Update kubeconfig
        run: |
          aws eks update-kubeconfig --name $CLUSTER_NAME --region $AWS_REGION

      # TODO: Revert to using main helm branch when changes from leader-election are merged in
      - name: Clone Helm Charts Repository
        env:
          HELM_CHARTS_BRANCH: ${{ inputs.helm-charts-branch || 'sky333999/leader-election' }}
        run: |
          rm -rf ./helm-charts
          git clone -b "$HELM_CHARTS_BRANCH" https://github.com/aws-observability/helm-charts.git ./helm-charts

      - name: Clone Test Repo
        uses: actions/checkout@v4
        with:
          repository: ${{ env.CWA_GITHUB_TEST_REPO_NAME }}
          ref: ${{ env.CWA_GITHUB_TEST_REPO_BRANCH }}
          path: ./test-repo

      - name: Replace hostname in override files
        run: |
          HOSTNAME=$(kubectl get nodes -l eks.amazonaws.com/nodegroup=$CLUSTER_NAME-leader-node -o jsonpath='{.items[0].metadata.name}')
          for file in ./test-repo/test/performance/eks/resources/leader_election_overrides/*; do
            sed -i "s/<hostname>/$HOSTNAME/g" "$file"
          done

      # TODO: Revert to using workflow built agent image once required changes are made on main branch
      - name: Check node count and manage Helm chart
        env:
          INPUT_CLUSTER_NAME: ${{ inputs.cluster_name }}
          INPUT_REGION: ${{ inputs.region }}
          INPUT_AGENT_REPO: ${{ inputs.cloudwatch_agent_repository }}
          INPUT_AGENT_TAG: ${{ inputs.cloudwatch_agent_tag }}
          INPUT_OPERATOR_REPO: ${{ inputs.cloudwatch_agent_operator_repository }}
          INPUT_OPERATOR_TAG: ${{ inputs.cloudwatch_agent_operator_tag }}
          ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }}
          OPERATOR_SHA: ${{ needs.GetLatestOperatorCommitSHA.outputs.operator_commit_sha }}
          GITHUB_SHA_VAL: ${{ github.sha }}
        run: |
          NODE_COUNT=$(kubectl get nodes --no-headers | wc -l)

          if [ "$NODE_COUNT" -eq 0 ]; then
            echo "Node count is 0, removing Helm chart"
            helm uninstall amazon-cloudwatch-observability -n amazon-cloudwatch || echo "Chart not found or already removed"
          else
            echo "Node count is $NODE_COUNT, installing/updating Helm chart"
          
            # Set variables with defaults
            HELM_CLUSTER_NAME="${INPUT_CLUSTER_NAME:-$CLUSTER_NAME}"
            HELM_REGION="${INPUT_REGION:-$AWS_REGION}"
            HELM_AGENT_REPO="${INPUT_AGENT_REPO:-$AGENT_ECR_TEST_REPO}"
            HELM_AGENT_TAG="${INPUT_AGENT_TAG:-$GITHUB_SHA_VAL}"
            HELM_OPERATOR_REPO="${INPUT_OPERATOR_REPO:-$OPERATOR_ECR_TEST_REPO}"
            HELM_OPERATOR_TAG="${INPUT_OPERATOR_TAG:-$OPERATOR_SHA}"
          
            # Echo all variables being passed to helm
            echo "CLUSTER_NAME: $HELM_CLUSTER_NAME"
            echo "REGION: $HELM_REGION"
            echo "AGENT_REPOSITORY: $HELM_AGENT_REPO"
            echo "AGENT_TAG: $HELM_AGENT_TAG"
            echo "AGENT_REPOSITORY_DOMAIN: $ECR_REGISTRY"
            echo "MANAGER_REPOSITORY: $HELM_OPERATOR_REPO"
            echo "MANAGER_TAG: $HELM_OPERATOR_TAG"
            echo "MANAGER_REPOSITORY_DOMAIN: $ECR_REGISTRY"
          
            helm upgrade --install --wait amazon-cloudwatch-observability \
              ./helm-charts/charts/amazon-cloudwatch-observability \
              --namespace amazon-cloudwatch \
              --create-namespace \
              --set clusterName="$HELM_CLUSTER_NAME" \
              --set region="$HELM_REGION" \
              --set agent.image.repository="cloudwatch-agent" \
              --set agent.image.tag="latest" \
              --set agent.image.repositoryDomainMap.public="public.ecr.aws/q4e2d9n7" \
              --set manager.image.repository="$HELM_OPERATOR_REPO" \
              --set manager.image.tag="$HELM_OPERATOR_TAG" \
              --set manager.image.repositoryDomainMap.public="$ECR_REGISTRY" \
              --values ./test-repo/test/performance/eks/resources/leader_election_overrides/base-overrides.yml
          fi

  cleanup-on-failure:
    if: ${{ failure() || cancelled() }}
    runs-on: ubuntu-latest
    needs: [ install-helm ]
    permissions:
      id-token: write
      contents: read
    steps:
      - name: Configure AWS Credentials
        uses: aws-actions/configure-aws-credentials@v4
        with:
          role-to-assume: ${{ env.TERRAFORM_AWS_ASSUME_ROLE}}
          aws-region: ${{ env.AWS_REGION}}
          role-duration-seconds: ${{ env.TERRAFORM_AWS_ASSUME_ROLE_DURATION }}

      - name: Install kubectl
        uses: azure/setup-kubectl@v3
        with:
          version: 'latest'

      - name: Install Helm
        uses: azure/setup-helm@v3
        with:
          version: 'latest'

      - name: Update kubeconfig
        run: |
          aws eks update-kubeconfig --name $CLUSTER_NAME --region $AWS_REGION

      - name: Uninstall Helm chart
        run: |
          echo "Test was cancelled or failed. Cleaning up resources..."
          helm uninstall amazon-cloudwatch-observability -n amazon-cloudwatch || echo "Chart not found or already removed"
          echo "Cleanup completed"