Install/Remove Helm Charts After Scaling #104
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. | |
| # SPDX-License-Identifier: MIT | |
| name: "Install/Remove Helm Charts After Scaling" | |
| on: | |
| # Use workflow_run to trigger this workflow after the scaling workflow completes | |
| workflow_run: | |
| workflows: ["EKS Cluster Scaling"] | |
| types: | |
| - completed | |
| branches: | |
| - main | |
| # Keep the manual trigger option | |
| workflow_dispatch: | |
| inputs: | |
| # Required Core Settings | |
| cluster_name: | |
| description: 'EKS Cluster Name' | |
| required: true | |
| type: string | |
| default: 'eks-performance' | |
| region: | |
| description: 'AWS Region' | |
| required: true | |
| type: string | |
| default: 'us-west-2' | |
| # Optional Settings | |
| cloudwatch_agent_repository: | |
| description: 'CloudWatch Agent Repository' | |
| type: string | |
| cloudwatch_agent_tag: | |
| description: 'CloudWatch Agent Tag' | |
| type: string | |
| cloudwatch_agent_operator_repository: | |
| description: 'CloudWatch Agent Operator Repository' | |
| type: string | |
| cloudwatch_agent_operator_tag: | |
| description: 'CloudWatch Agent Operator Tag' | |
| type: string | |
| helm-charts-branch: | |
| description: 'Branch of the helm charts to test' | |
| type: string | |
| default: 'main' | |
| operator-branch: | |
| description: 'Branch of the operator to test' | |
| type: string | |
| default: 'main' | |
| terraform_assume_role: | |
| description: 'AWS IAM Role to assume' | |
| type: string | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.ref }} | |
| cancel-in-progress: true | |
| env: | |
| # Cluster environment variables | |
| AWS_REGION: ${{ inputs.region || 'us-west-2' }} | |
| CLUSTER_NAME: ${{ inputs.cluster_name || 'eks-performance' }} | |
| TERRAFORM_AWS_ASSUME_ROLE: ${{ inputs.terraform_assume_role || vars.TERRAFORM_AWS_ASSUME_ROLE }} | |
| TERRAFORM_AWS_ASSUME_ROLE_DURATION: 3600 # 1 hour duration | |
| # ECR repository environment variables | |
| AGENT_ECR_TEST_REPO: "cwagent-integration-test" | |
| OPERATOR_ECR_TEST_REPO: "cwagent-operator-pre-release" | |
| # Github repository environment variables | |
| OPERATOR_GITHUB_REPO_NAME: "aws/amazon-cloudwatch-agent-operator" | |
| CWA_GITHUB_TEST_REPO_NAME: "aws/amazon-cloudwatch-agent-test" | |
| CWA_GITHUB_TEST_REPO_BRANCH: "main" | |
| jobs: | |
| # Check if this workflow should run | |
| check-trigger: | |
| runs-on: ubuntu-latest | |
| if: ${{ github.event_name == 'workflow_dispatch' || (github.event_name == 'workflow_run' && github.event.workflow_run.conclusion == 'success' && github.event.workflow_run.event == 'schedule') }} | |
| steps: | |
| - name: Check trigger type | |
| id: check-trigger | |
| run: | | |
| if [ "${{ github.event_name }}" == "workflow_run" ]; then | |
| echo "Triggered by workflow_run from a scheduled event" | |
| else | |
| echo "Triggered manually via workflow_dispatch" | |
| fi | |
| outputs: | |
| should_continue: ${{ github.event_name == 'workflow_dispatch' || (github.event_name == 'workflow_run' && github.event.workflow_run.conclusion == 'success' && github.event.workflow_run.event == 'schedule') }} | |
| # Get latest operator commit from github repo | |
| GetLatestOperatorCommitSHA: | |
| needs: check-trigger | |
| if: ${{ needs.check-trigger.outputs.should_continue == 'true' }} | |
| runs-on: ubuntu-latest | |
| outputs: | |
| operator_commit_sha: ${{steps.get_latest_sha.outputs.operator_sha}} | |
| operator_repo_name: ${{env.OPERATOR_GITHUB_REPO_NAME}} | |
| steps: | |
| - name: Checkout the target repo | |
| uses: actions/checkout@v4 | |
| with: | |
| repository: ${{env.OPERATOR_GITHUB_REPO_NAME}} | |
| ref: ${{ inputs.operator-branch || 'main' }} | |
| path: operator-repo | |
| - name: Get latest commit SHA | |
| id: get_latest_sha | |
| run: | | |
| cd operator-repo | |
| latest_sha=$(git rev-parse HEAD) | |
| echo "operator_sha=$latest_sha" >> "$GITHUB_OUTPUT" | |
| # Build and upload agent image to ECR repo | |
| BuildAgent: | |
| needs: check-trigger | |
| if: ${{ needs.check-trigger.outputs.should_continue == 'true' }} | |
| uses: ./.github/workflows/build-test-artifacts.yml | |
| concurrency: | |
| group: "Build-Test-Artifacts-${{github.ref_name}}" | |
| cancel-in-progress: true | |
| secrets: inherit | |
| permissions: | |
| id-token: write | |
| contents: read | |
| actions: write | |
| with: | |
| run-tests: false | |
| # Build and upload operator image to ECR repo | |
| BuildOperator: | |
| needs: [ check-trigger, GetLatestOperatorCommitSHA ] | |
| if: ${{ needs.check-trigger.outputs.should_continue == 'true' }} | |
| uses: aws/amazon-cloudwatch-agent-operator/.github/workflows/build-and-upload.yml@main | |
| permissions: | |
| id-token: write | |
| contents: read | |
| concurrency: | |
| group: ${{ github.workflow }}-operator-${{ inputs.operator-branch || 'main' }} | |
| cancel-in-progress: true | |
| secrets: inherit | |
| with: | |
| tag: ${{needs.GetLatestOperatorCommitSHA.outputs.operator_commit_sha}} | |
| target-sha: ${{needs.GetLatestOperatorCommitSHA.outputs.operator_commit_sha}} | |
| repository: ${{needs.GetLatestOperatorCommitSHA.outputs.operator_repo_name}} | |
| test-image-before-upload: false | |
| install-helm: | |
| needs: [ check-trigger, BuildAgent, BuildOperator, GetLatestOperatorCommitSHA ] | |
| if: ${{ needs.check-trigger.outputs.should_continue == 'true' }} | |
| runs-on: ubuntu-latest | |
| permissions: | |
| id-token: write | |
| contents: read | |
| steps: | |
| - name: Configure AWS Credentials | |
| uses: aws-actions/configure-aws-credentials@v4 | |
| with: | |
| role-to-assume: ${{ env.TERRAFORM_AWS_ASSUME_ROLE}} | |
| aws-region: ${{ env.AWS_REGION}} | |
| role-duration-seconds: ${{ env.TERRAFORM_AWS_ASSUME_ROLE_DURATION }} | |
| - name: Login ECR | |
| id: login-ecr | |
| uses: aws-actions/amazon-ecr-login@v2 | |
| - name: Install kubectl | |
| uses: azure/setup-kubectl@v3 | |
| with: | |
| version: 'latest' | |
| - name: Install Helm | |
| uses: azure/setup-helm@v3 | |
| with: | |
| version: 'latest' | |
| - name: Update kubeconfig | |
| run: | | |
| aws eks update-kubeconfig --name $CLUSTER_NAME --region $AWS_REGION | |
| - name: Clone Helm Charts Repository | |
| env: | |
| HELM_CHARTS_BRANCH: ${{ inputs.helm-charts-branch || 'main' }} | |
| run: | | |
| rm -rf ./helm-charts | |
| git clone -b "$HELM_CHARTS_BRANCH" https://github.com/aws-observability/helm-charts.git ./helm-charts | |
| - name: Clone Test Repo | |
| uses: actions/checkout@v4 | |
| with: | |
| repository: ${{ env.CWA_GITHUB_TEST_REPO_NAME }} | |
| ref: ${{ env.CWA_GITHUB_TEST_REPO_BRANCH }} | |
| path: ./test-repo | |
| - name: Replace hostname in override files | |
| run: | | |
| HOSTNAME=$(kubectl get nodes -l eks.amazonaws.com/nodegroup=$CLUSTER_NAME-leader-node -o jsonpath='{.items[0].metadata.name}') | |
| for file in ./test-repo/test/performance/eks/resources/leader_election_overrides/*; do | |
| sed -i "s/<hostname>/$HOSTNAME/g" "$file" | |
| done | |
| # TODO: Revert to using workflow built agent image once required changes are made on main branch | |
| - name: Check node count and manage Helm chart | |
| env: | |
| INPUT_CLUSTER_NAME: ${{ inputs.cluster_name }} | |
| INPUT_REGION: ${{ inputs.region }} | |
| INPUT_AGENT_REPO: ${{ inputs.cloudwatch_agent_repository }} | |
| INPUT_AGENT_TAG: ${{ inputs.cloudwatch_agent_tag }} | |
| INPUT_OPERATOR_REPO: ${{ inputs.cloudwatch_agent_operator_repository }} | |
| INPUT_OPERATOR_TAG: ${{ inputs.cloudwatch_agent_operator_tag }} | |
| ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }} | |
| OPERATOR_SHA: ${{ needs.GetLatestOperatorCommitSHA.outputs.operator_commit_sha }} | |
| GITHUB_SHA_VAL: ${{ github.sha }} | |
| run: | | |
| NODE_COUNT=$(kubectl get nodes --no-headers | wc -l) | |
| if [ "$NODE_COUNT" -eq 0 ]; then | |
| echo "Node count is 0, removing Helm chart" | |
| helm uninstall amazon-cloudwatch-observability -n amazon-cloudwatch || echo "Chart not found or already removed" | |
| else | |
| echo "Node count is $NODE_COUNT, installing/updating Helm chart" | |
| # Set variables with defaults | |
| HELM_CLUSTER_NAME="${INPUT_CLUSTER_NAME:-$CLUSTER_NAME}" | |
| HELM_REGION="${INPUT_REGION:-$AWS_REGION}" | |
| HELM_AGENT_REPO="${INPUT_AGENT_REPO:-$AGENT_ECR_TEST_REPO}" | |
| HELM_AGENT_TAG="${INPUT_AGENT_TAG:-$GITHUB_SHA_VAL}" | |
| HELM_OPERATOR_REPO="${INPUT_OPERATOR_REPO:-$OPERATOR_ECR_TEST_REPO}" | |
| HELM_OPERATOR_TAG="${INPUT_OPERATOR_TAG:-$OPERATOR_SHA}" | |
| # Echo all variables being passed to helm | |
| echo "CLUSTER_NAME: $HELM_CLUSTER_NAME" | |
| echo "REGION: $HELM_REGION" | |
| echo "AGENT_REPOSITORY: $HELM_AGENT_REPO" | |
| echo "AGENT_TAG: $HELM_AGENT_TAG" | |
| echo "AGENT_REPOSITORY_DOMAIN: $ECR_REGISTRY" | |
| echo "MANAGER_REPOSITORY: $HELM_OPERATOR_REPO" | |
| echo "MANAGER_TAG: $HELM_OPERATOR_TAG" | |
| echo "MANAGER_REPOSITORY_DOMAIN: $ECR_REGISTRY" | |
| helm upgrade --install --wait amazon-cloudwatch-observability \ | |
| ./helm-charts/charts/amazon-cloudwatch-observability \ | |
| --namespace amazon-cloudwatch \ | |
| --create-namespace \ | |
| --set clusterName="$HELM_CLUSTER_NAME" \ | |
| --set region="$HELM_REGION" \ | |
| --set agent.image.repository="cloudwatch-agent" \ | |
| --set agent.image.tag="latest" \ | |
| --set agent.image.repositoryDomainMap.public="public.ecr.aws/q4e2d9n7" \ | |
| --set manager.image.repository="$HELM_OPERATOR_REPO" \ | |
| --set manager.image.tag="$HELM_OPERATOR_TAG" \ | |
| --set manager.image.repositoryDomainMap.public="$ECR_REGISTRY" \ | |
| --values ./test-repo/test/performance/eks/resources/leader_election_overrides/base-overrides.yml | |
| fi | |
| cleanup-on-failure: | |
| if: ${{ failure() || cancelled() }} | |
| runs-on: ubuntu-latest | |
| needs: [ install-helm ] | |
| permissions: | |
| id-token: write | |
| contents: read | |
| steps: | |
| - name: Configure AWS Credentials | |
| uses: aws-actions/configure-aws-credentials@v4 | |
| with: | |
| role-to-assume: ${{ env.TERRAFORM_AWS_ASSUME_ROLE}} | |
| aws-region: ${{ env.AWS_REGION}} | |
| role-duration-seconds: ${{ env.TERRAFORM_AWS_ASSUME_ROLE_DURATION }} | |
| - name: Install kubectl | |
| uses: azure/setup-kubectl@v3 | |
| with: | |
| version: 'latest' | |
| - name: Install Helm | |
| uses: azure/setup-helm@v3 | |
| with: | |
| version: 'latest' | |
| - name: Update kubeconfig | |
| run: | | |
| aws eks update-kubeconfig --name $CLUSTER_NAME --region $AWS_REGION | |
| - name: Uninstall Helm chart | |
| run: | | |
| echo "Test was cancelled or failed. Cleaning up resources..." | |
| helm uninstall amazon-cloudwatch-observability -n amazon-cloudwatch || echo "Chart not found or already removed" | |
| echo "Cleanup completed" |