PR-test.yml

# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
# SPDX-License-Identifier: MIT

name: PR Test
permissions:
  id-token: write
  contents: read
env:
  TERRAFORM_AWS_ASSUME_ROLE: ${{ vars.TERRAFORM_AWS_ASSUME_ROLE }}
  TERRAFORM_AWS_ASSUME_ROLE_DURATION: 14400 # 4 hours
  CWA_GITHUB_TEST_REPO_NAME: "aws/amazon-cloudwatch-agent-test"
  CWA_GITHUB_TEST_REPO_URL: "https://github.com/aws/amazon-cloudwatch-agent-test.git"
  CWA_GITHUB_TEST_REPO_BRANCH: "main"
  KEY_NAME: ${{ secrets.KEY_NAME }}
  PRIVATE_KEY: ${{ secrets.AWS_PRIVATE_KEY  }}
  S3_INTEGRATION_BUCKET: ${{ vars.S3_INTEGRATION_BUCKET }}

on:
  workflow_dispatch:
  pull_request:
    branches:
      - main*
      - feature*
      - feature/**
    types:
      - opened
      - synchronize
      - reopened
      - ready_for_review
      - labeled
      - unlabeled

concurrency:
  group: ${{ github.workflow }}-${{ github.ref_name }}
  cancel-in-progress: true

jobs:
  CheckLabel:
    runs-on: ubuntu-latest
    outputs:
      should_run: ${{ steps.check.outputs.has_label }}
    steps:
      - id: check
        env:
          PR_FORK: ${{ github.event.pull_request.head.repo.fork }}
          HAS_LABEL: ${{ contains(github.event.pull_request.labels.*.name, 'ready for testing') }}
        run: |
          # Fork PRs never receive secrets or id-token permissions, so the
          # integration tests cannot run even if the label is added. Maintainers
          # must push the branch to this repo to run them.
          if [[ "$PR_FORK" == "true" ]]; then
            echo "Fork PR - integration tests cannot run (no access to secrets)."
            echo "has_label=false" >> $GITHUB_OUTPUT
          elif [[ "$HAS_LABEL" == "true" ]]; then
            echo "has_label=true" >> $GITHUB_OUTPUT
          else
            echo "has_label=false" >> $GITHUB_OUTPUT
          fi

  BuildAndUpload:
    needs: [ CheckLabel ]
    if: needs.CheckLabel.outputs.should_run == 'true'
    uses: ./.github/workflows/test-build.yml
    secrets: inherit
    permissions:
      id-token: write
      contents: read
    with:
      BucketKey: "integration-test/binary/${{ github.sha }}"
      PackageBucketKey: "integration-test/packaging/${{ github.sha }}"
      TerraformAWSAssumeRole: ${{ vars.TERRAFORM_AWS_ASSUME_ROLE }}
      Bucket: ${{ vars.S3_INTEGRATION_BUCKET }}

  OutputEnvVariables:
    name: 'OutputEnvVariables'
    needs: [ CheckLabel ]
    if: needs.CheckLabel.outputs.should_run == 'true'
    runs-on: ubuntu-latest
    outputs:
      CWA_GITHUB_TEST_REPO_NAME: ${{ steps.set-outputs.outputs.CWA_GITHUB_TEST_REPO_NAME }}
      CWA_GITHUB_TEST_REPO_URL: ${{ steps.set-outputs.outputs.CWA_GITHUB_TEST_REPO_URL }}
      CWA_GITHUB_TEST_REPO_BRANCH: ${{ steps.set-outputs.outputs.CWA_GITHUB_TEST_REPO_BRANCH }}
    steps:
      - name: SetOutputs
        id: set-outputs
        run: |
          CWA_GITHUB_TEST_REPO_BRANCH=${{ env.CWA_GITHUB_TEST_REPO_BRANCH }}

          echo "CWA_GITHUB_TEST_REPO_NAME=${{ env.CWA_GITHUB_TEST_REPO_NAME }}" >> "$GITHUB_OUTPUT"
          echo "CWA_GITHUB_TEST_REPO_URL=${{ env.CWA_GITHUB_TEST_REPO_URL }}" >> "$GITHUB_OUTPUT"
          echo "CWA_GITHUB_TEST_REPO_BRANCH=${CWA_GITHUB_TEST_REPO_BRANCH:-${{ env.CWA_GITHUB_TEST_REPO_BRANCH }}}" >> "$GITHUB_OUTPUT"

      - name: Echo test variables
        env:
          GITHUB_SHA_VAL: ${{ github.sha }}
          OUT_REPO_NAME: ${{ steps.set-outputs.outputs.CWA_GITHUB_TEST_REPO_NAME }}
          OUT_REPO_URL: ${{ steps.set-outputs.outputs.CWA_GITHUB_TEST_REPO_URL }}
          OUT_REPO_BRANCH: ${{ steps.set-outputs.outputs.CWA_GITHUB_TEST_REPO_BRANCH }}
        run: |
          echo "build_id: $GITHUB_SHA_VAL"
          echo "CWA_GITHUB_TEST_REPO_NAME: $OUT_REPO_NAME"
          echo "CWA_GITHUB_TEST_REPO_URL: $OUT_REPO_URL"
          echo "CWA_GITHUB_TEST_REPO_BRANCH: $OUT_REPO_BRANCH"

      - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1
        with:
          repository: ${{ steps.set-outputs.outputs.CWA_GITHUB_TEST_REPO_NAME }}
          ref: ${{ steps.set-outputs.outputs.CWA_GITHUB_TEST_REPO_BRANCH }}

      - name: Set up Go 1.x
        uses: actions/setup-go@7b8cf10d4e4a01d4992d18a89f4d7dc5a3e6d6f4 # v4.3.0
        with:
          go-version: ~1.26.4

  StartLocalStack:
    name: 'StartLocalStack'
    needs: [ CheckLabel, OutputEnvVariables ]
    if: needs.CheckLabel.outputs.should_run == 'true'
    uses: ./.github/workflows/start-localstack.yml
    secrets: inherit
    permissions:
      id-token: write
      contents: read
    with:
      region: us-west-2
      test_repo_name: ${{ needs.OutputEnvVariables.outputs.CWA_GITHUB_TEST_REPO_NAME }}
      test_repo_branch: ${{ needs.OutputEnvVariables.outputs.CWA_GITHUB_TEST_REPO_BRANCH }}
      terraform_assume_role: ${{ vars.TERRAFORM_AWS_ASSUME_ROLE }}
      test_repo_url: ${{ needs.OutputEnvVariables.outputs.CWA_GITHUB_TEST_REPO_URL }}
      github_sha: ${{ github.sha }}
      s3_integration_bucket: ${{ vars.S3_INTEGRATION_BUCKET }}

  GenerateTestMatrix:
    name: 'GenerateTestMatrix'
    needs: [ CheckLabel, OutputEnvVariables ]
    if: needs.CheckLabel.outputs.should_run == 'true'
    runs-on: ubuntu-latest
    outputs:
      ec2_linux_matrix_0: ${{ steps.paginate-matrix.outputs.ec2_linux_matrix_0 }}
      ec2_linux_matrix_1: ${{ steps.paginate-matrix.outputs.ec2_linux_matrix_1 }}
      ec2_linux_matrix_2: ${{ steps.paginate-matrix.outputs.ec2_linux_matrix_2 }}
      ec2_linux_matrix_3: ${{ steps.paginate-matrix.outputs.ec2_linux_matrix_3 }}
      ec2_linux_matrix_4: ${{ steps.paginate-matrix.outputs.ec2_linux_matrix_4 }}
      ec2_linux_matrix_page_count: ${{ steps.paginate-matrix.outputs.ec2_linux_matrix_page_count }}
      ec2_selinux_matrix: ${{ steps.set-matrix.outputs.ec2_selinux_matrix }}

    steps:
      - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1
        with:
          repository: ${{env.CWA_GITHUB_TEST_REPO_NAME}}
          ref: ${{env.CWA_GITHUB_TEST_REPO_BRANCH}}

      - name: Set up Go 1.x
        uses: actions/setup-go@7b8cf10d4e4a01d4992d18a89f4d7dc5a3e6d6f4 # v4.3.0
        with:
          go-version: ~1.26.4

      - name: Install jq
        run: sudo apt-get install -y jq

      - name: Generate matrix
        id: set-matrix
        run: |
          go run --tags=generator generator/test_case_generator.go
          # Convert the output to valid JSON and filter out metric_value_benchmark tests
          jq -c '[.[] | select(.test_dir != "./test/metric_value_benchmark")]' generator/resources/ec2_linux_complete_test_matrix.json > filtered_matrix.json
          # Escape the JSON for GitHub Actions
          MATRIX=$(jq -c -r '@json' filtered_matrix.json)
          echo "ec2_linux_matrix=$MATRIX" >> $GITHUB_OUTPUT
           jq -c '[.[] | select(.test_dir != "./test/metric_value_benchmark")]' generator/resources/ec2_selinux_complete_test_matrix.json > filtered_matrix_selinux.json
          # Escape the JSON for GitHub Actions
          MATRIX=$(jq -c -r '@json' filtered_matrix_selinux.json)
          echo "ec2_selinux_matrix=$MATRIX" >> $GITHUB_OUTPUT

      - name: Paginate ec2_linux_matrix
        id: paginate-matrix
        run: |
          # GitHub Actions matrix limit is 256 jobs per workflow run.
          # Use 200 per page for headroom. Up to 5 pages supported (1000 tests).
          # Read the filtered matrix from the file written by the "Generate matrix"
          # step instead of a step-level env var: the matrix can exceed the
          # kernel's ARG_MAX (~2 MiB) on execve, which would prevent bash from
          # starting for this step.
          PAGE_SIZE=200
          FULL_MATRIX=$(cat filtered_matrix.json)
          TOTAL=$(echo "$FULL_MATRIX" | jq 'length')
          PAGE_COUNT=$(( (TOTAL + PAGE_SIZE - 1) / PAGE_SIZE ))

          if [ "$PAGE_COUNT" -gt 5 ]; then
            echo "::error::ec2_linux_matrix has $TOTAL entries requiring $PAGE_COUNT pages, but only 5 are supported. Add more static page slots."
            exit 1
          fi

          echo "ec2_linux total=$TOTAL pages=$PAGE_COUNT"
          echo "ec2_linux_matrix_page_count=${PAGE_COUNT}" >> "$GITHUB_OUTPUT"

          for (( i=0; i<PAGE_COUNT; i++ )); do
            START=$(( i * PAGE_SIZE ))
            PAGE=$(echo "$FULL_MATRIX" | jq -c ".[$START:$START+$PAGE_SIZE]" | jq -c -r '@json')
            echo "ec2_linux_matrix_${i}=${PAGE}" >> "$GITHUB_OUTPUT"
          done

      - name: Echo test plan matrix
        env:
          EC2_LINUX_PAGE_COUNT: ${{ steps.paginate-matrix.outputs.ec2_linux_matrix_page_count }}
          EC2_SELINUX_MATRIX: ${{ steps.set-matrix.outputs.ec2_selinux_matrix }}
        run: |
          echo "ec2_linux_matrix pages: $EC2_LINUX_PAGE_COUNT"
          echo "ec2_selinux_matrix: $EC2_SELINUX_MATRIX"


  EC2LinuxIntegrationTest-0:
    name: 'EC2LinuxTests-0'
    needs: [ CheckLabel, GenerateTestMatrix, OutputEnvVariables, StartLocalStack, BuildAndUpload ]
    if: needs.CheckLabel.outputs.should_run == 'true'
    uses:  ./.github/workflows/ec2-integration-test.yml
    permissions:
      id-token: write
      contents: read
    with:
      build_id: ${{ github.sha }}
      test_dir: terraform/ec2/linux
      job_id: ec2-linux-integration-test-0
      test_props: ${{ needs.GenerateTestMatrix.outputs.ec2_linux_matrix_0 }}
      test_repo_name: ${{ needs.OutputEnvVariables.outputs.CWA_GITHUB_TEST_REPO_NAME }}
      test_repo_url: ${{ needs.OutputEnvVariables.outputs.CWA_GITHUB_TEST_REPO_URL }}
      test_repo_branch: ${{ needs.OutputEnvVariables.outputs.CWA_GITHUB_TEST_REPO_BRANCH }}
      localstack_host: ${{ needs.StartLocalStack.outputs.local_stack_host_name }}
      region: us-west-2
      terraform_assume_role: ${{ vars.TERRAFORM_AWS_ASSUME_ROLE }}
      s3_integration_bucket: ${{ vars.S3_INTEGRATION_BUCKET }}
      is_selinux_test: false
    secrets: inherit

  EC2LinuxIntegrationTest-1:
    name: 'EC2LinuxTests-1'
    needs: [ CheckLabel, GenerateTestMatrix, OutputEnvVariables, StartLocalStack, BuildAndUpload ]
    if: ${{ needs.CheckLabel.outputs.should_run == 'true' && needs.GenerateTestMatrix.outputs.ec2_linux_matrix_page_count > 1 }}
    uses:  ./.github/workflows/ec2-integration-test.yml
    permissions:
      id-token: write
      contents: read
    with:
      build_id: ${{ github.sha }}
      test_dir: terraform/ec2/linux
      job_id: ec2-linux-integration-test-1
      test_props: ${{ needs.GenerateTestMatrix.outputs.ec2_linux_matrix_1 }}
      test_repo_name: ${{ needs.OutputEnvVariables.outputs.CWA_GITHUB_TEST_REPO_NAME }}
      test_repo_url: ${{ needs.OutputEnvVariables.outputs.CWA_GITHUB_TEST_REPO_URL }}
      test_repo_branch: ${{ needs.OutputEnvVariables.outputs.CWA_GITHUB_TEST_REPO_BRANCH }}
      localstack_host: ${{ needs.StartLocalStack.outputs.local_stack_host_name }}
      region: us-west-2
      terraform_assume_role: ${{ vars.TERRAFORM_AWS_ASSUME_ROLE }}
      s3_integration_bucket: ${{ vars.S3_INTEGRATION_BUCKET }}
      is_selinux_test: false
    secrets: inherit

  EC2LinuxIntegrationTest-2:
    name: 'EC2LinuxTests-2'
    needs: [ CheckLabel, GenerateTestMatrix, OutputEnvVariables, StartLocalStack, BuildAndUpload ]
    if: ${{ needs.CheckLabel.outputs.should_run == 'true' && needs.GenerateTestMatrix.outputs.ec2_linux_matrix_page_count > 2 }}
    uses:  ./.github/workflows/ec2-integration-test.yml
    permissions:
      id-token: write
      contents: read
    with:
      build_id: ${{ github.sha }}
      test_dir: terraform/ec2/linux
      job_id: ec2-linux-integration-test-2
      test_props: ${{ needs.GenerateTestMatrix.outputs.ec2_linux_matrix_2 }}
      test_repo_name: ${{ needs.OutputEnvVariables.outputs.CWA_GITHUB_TEST_REPO_NAME }}
      test_repo_url: ${{ needs.OutputEnvVariables.outputs.CWA_GITHUB_TEST_REPO_URL }}
      test_repo_branch: ${{ needs.OutputEnvVariables.outputs.CWA_GITHUB_TEST_REPO_BRANCH }}
      localstack_host: ${{ needs.StartLocalStack.outputs.local_stack_host_name }}
      region: us-west-2
      terraform_assume_role: ${{ vars.TERRAFORM_AWS_ASSUME_ROLE }}
      s3_integration_bucket: ${{ vars.S3_INTEGRATION_BUCKET }}
      is_selinux_test: false
    secrets: inherit

  EC2LinuxIntegrationTest-3:
    name: 'EC2LinuxTests-3'
    needs: [ CheckLabel, GenerateTestMatrix, OutputEnvVariables, StartLocalStack, BuildAndUpload ]
    if: ${{ needs.CheckLabel.outputs.should_run == 'true' && needs.GenerateTestMatrix.outputs.ec2_linux_matrix_page_count > 3 }}
    uses:  ./.github/workflows/ec2-integration-test.yml
    permissions:
      id-token: write
      contents: read
    with:
      build_id: ${{ github.sha }}
      test_dir: terraform/ec2/linux
      job_id: ec2-linux-integration-test-3
      test_props: ${{ needs.GenerateTestMatrix.outputs.ec2_linux_matrix_3 }}
      test_repo_name: ${{ needs.OutputEnvVariables.outputs.CWA_GITHUB_TEST_REPO_NAME }}
      test_repo_url: ${{ needs.OutputEnvVariables.outputs.CWA_GITHUB_TEST_REPO_URL }}
      test_repo_branch: ${{ needs.OutputEnvVariables.outputs.CWA_GITHUB_TEST_REPO_BRANCH }}
      localstack_host: ${{ needs.StartLocalStack.outputs.local_stack_host_name }}
      region: us-west-2
      terraform_assume_role: ${{ vars.TERRAFORM_AWS_ASSUME_ROLE }}
      s3_integration_bucket: ${{ vars.S3_INTEGRATION_BUCKET }}
      is_selinux_test: false
    secrets: inherit

  EC2LinuxIntegrationTest-4:
    name: 'EC2LinuxTests-4'
    needs: [ CheckLabel, GenerateTestMatrix, OutputEnvVariables, StartLocalStack, BuildAndUpload ]
    if: ${{ needs.CheckLabel.outputs.should_run == 'true' && needs.GenerateTestMatrix.outputs.ec2_linux_matrix_page_count > 4 }}
    uses:  ./.github/workflows/ec2-integration-test.yml
    permissions:
      id-token: write
      contents: read
    with:
      build_id: ${{ github.sha }}
      test_dir: terraform/ec2/linux
      job_id: ec2-linux-integration-test-4
      test_props: ${{ needs.GenerateTestMatrix.outputs.ec2_linux_matrix_4 }}
      test_repo_name: ${{ needs.OutputEnvVariables.outputs.CWA_GITHUB_TEST_REPO_NAME }}
      test_repo_url: ${{ needs.OutputEnvVariables.outputs.CWA_GITHUB_TEST_REPO_URL }}
      test_repo_branch: ${{ needs.OutputEnvVariables.outputs.CWA_GITHUB_TEST_REPO_BRANCH }}
      localstack_host: ${{ needs.StartLocalStack.outputs.local_stack_host_name }}
      region: us-west-2
      terraform_assume_role: ${{ vars.TERRAFORM_AWS_ASSUME_ROLE }}
      s3_integration_bucket: ${{ vars.S3_INTEGRATION_BUCKET }}
      is_selinux_test: false
    secrets: inherit

  EC2SELinuxIntegrationTest:
    name: 'EC2SELinuxTests'
    needs: [ CheckLabel, GenerateTestMatrix, OutputEnvVariables, StartLocalStack, BuildAndUpload ]
    if: needs.CheckLabel.outputs.should_run == 'true'
    uses:  ./.github/workflows/ec2-integration-test.yml
    permissions:
      id-token: write
      contents: read
    with:
      build_id: ${{ github.sha }}
      test_dir: terraform/ec2/linux
      job_id: ec2-linux-integration-test
      test_props: ${{ needs.GenerateTestMatrix.outputs.ec2_selinux_matrix }}
      test_repo_name: ${{ needs.OutputEnvVariables.outputs.CWA_GITHUB_TEST_REPO_NAME }}
      test_repo_url: ${{ needs.OutputEnvVariables.outputs.CWA_GITHUB_TEST_REPO_URL }}
      test_repo_branch: ${{ needs.OutputEnvVariables.outputs.CWA_GITHUB_TEST_REPO_BRANCH }}
      localstack_host: ${{ needs.StartLocalStack.outputs.local_stack_host_name }}
      region: us-west-2
      terraform_assume_role: ${{ vars.TERRAFORM_AWS_ASSUME_ROLE }}
      s3_integration_bucket: ${{ vars.S3_INTEGRATION_BUCKET }}
      is_selinux_test: true
    secrets: inherit

  StopLocalStack:
    name: 'StopLocalStack'
    needs: [ CheckLabel, StartLocalStack, EC2LinuxIntegrationTest-0, EC2LinuxIntegrationTest-1, EC2LinuxIntegrationTest-2, EC2LinuxIntegrationTest-3, EC2LinuxIntegrationTest-4, OutputEnvVariables ]
    if: ${{ always() && needs.StartLocalStack.result == 'success' && needs.CheckLabel.outputs.should_run == 'true'}}
    uses: ./.github/workflows/stop-localstack.yml
    secrets: inherit
    permissions:
      id-token: write
      contents: read
    with:
      region: us-west-2
      test_repo_name: ${{ needs.OutputEnvVariables.outputs.CWA_GITHUB_TEST_REPO_NAME }}
      test_repo_branch: ${{ needs.OutputEnvVariables.outputs.CWA_GITHUB_TEST_REPO_BRANCH }}
      terraform_assume_role: ${{ vars.TERRAFORM_AWS_ASSUME_ROLE }}
      github_sha: ${{ github.sha }}
      s3_integration_bucket: ${{ vars.S3_INTEGRATION_BUCKET }}

  IntegrationTestGate:
    name: Check ready for testing label
    runs-on: ubuntu-latest
    steps:
      - name: Check for ready for testing label
        env:
          PR_FORK: ${{ github.event.pull_request.head.repo.fork }}
          PR_DRAFT: ${{ github.event.pull_request.draft }}
          HAS_SKIP_TESTING: ${{ contains(github.event.pull_request.labels.*.name, 'skip testing') }}
          HAS_READY_FOR_TESTING: ${{ contains(github.event.pull_request.labels.*.name, 'ready for testing') }}
        run: |
          if [[ "$PR_FORK" == "true" ]]; then
            echo "Fork PR - integration tests skipped (no access to secrets). Push branch to this repo to run them."
            exit 0
          fi

          if [[ "$PR_DRAFT" == "true" ]]; then
            echo "Draft PR - skipping label check."
            exit 0
          fi

          if [[ "$HAS_SKIP_TESTING" == "true" ]]; then
            echo "'skip testing' label found - bypassing integration test requirement."
            exit 0
          fi

          if [[ "$HAS_READY_FOR_TESTING" != "true" ]]; then
            echo "Missing 'ready for testing' label. Please add before merging."
            exit 1
          fi

          echo "'ready for testing' label found!"

  verify-all:
    name: Verify All PR Test Jobs
    needs: [IntegrationTestGate, CheckLabel, BuildAndUpload, OutputEnvVariables, StartLocalStack,
            GenerateTestMatrix, EC2LinuxIntegrationTest-0, EC2LinuxIntegrationTest-1,
            EC2LinuxIntegrationTest-2, EC2LinuxIntegrationTest-3, EC2LinuxIntegrationTest-4,
            EC2SELinuxIntegrationTest, StopLocalStack]
    runs-on: ubuntu-latest
    if: always()
    steps:
      - name: Check Job Status
        env:
          NEEDS_JSON: ${{ toJSON(needs) }}
        run: |
          failed_jobs=()
          successful_jobs=()

          # Loop through all jobs in needs context
          for job in $(echo "$NEEDS_JSON" | jq -r 'keys[]'); do
            result=$(echo "$NEEDS_JSON" | jq -r ".[\"$job\"].result")

            if [[ "$result" == "failure" ]]; then
              failed_jobs+=("$job")
            elif [[ "$result" == "success" ]]; then
              successful_jobs+=("$job")
            fi
          done

          echo "Successfully validated jobs:"
          printf '%s\n' "${successful_jobs[@]}"

          if [ ${#failed_jobs[@]} -ne 0 ]; then
            echo -e "\nFailed jobs:"
            printf '%s\n' "${failed_jobs[@]}"
            exit 1
          fi

          echo -e "\nAll required jobs completed without failures!"