Merge remote-tracking branch 'origin/main' into larkirs/mysql-pipeline-translator

Author: Klarsen04

.github/workflows/PR-build.yml

@@ -35,8 +35,10 @@ jobs:
           filters: .github/config/file-filters.yml
 
       - name: List all updated files
+        env:
+          BUILD_FILES: ${{ steps.filter.outputs.build_files }}
         run: |
-          for file in ${{ steps.filter.outputs.build_files }}; do
+          for file in $BUILD_FILES; do
             echo "$file"
           done
 
@@ -142,7 +144,10 @@ jobs:
 
       - name: Build
         if: steps.cached_binaries.outputs.cache-hit != 'true' && needs.changes.outputs.build == 'true'
-        run: make amazon-cloudwatch-agent-${{ matrix.family }}
+        env:
+          MATRIX_FAMILY: ${{ matrix.family }}
+        shell: bash
+        run: make "amazon-cloudwatch-agent-$MATRIX_FAMILY"
 
       - name: Collect binary sizes
         if: steps.cached_binaries.outputs.cache-hit != 'true' && needs.changes.outputs.build == 'true' && matrix.family != 'darwin' && matrix.os != 'windows-latest' && github.event_name == 'pull_request'
@@ -221,16 +226,15 @@ jobs:
     if: ${{ !cancelled() }}
     steps:
       - name: Check Job Status
+        env:
+          NEEDS_JSON: ${{ toJSON(needs) }}
         run: |
-          # Convert needs context to JSON and process with jq
-          needs_json='${{ toJSON(needs) }}'
-
           failed_jobs=()
           successful_jobs=()
 
           # Loop through all jobs in needs context
-          for job in $(echo "$needs_json" | jq -r 'keys[]'); do
-            result=$(echo "$needs_json" | jq -r ".[\"$job\"].result")
+          for job in $(echo "$NEEDS_JSON" | jq -r 'keys[]'); do
+            result=$(echo "$NEEDS_JSON" | jq -r ".[\"$job\"].result")
 
             if [[ "$result" == "failure" ]]; then
               failed_jobs+=("$job")

.github/workflows/PR-test.yml

@@ -41,14 +41,17 @@ jobs:
       should_run: ${{ steps.check.outputs.has_label }}
     steps:
       - id: check
+        env:
+          PR_FORK: ${{ github.event.pull_request.head.repo.fork }}
+          HAS_LABEL: ${{ contains(github.event.pull_request.labels.*.name, 'ready for testing') }}
         run: |
           # Fork PRs never receive secrets or id-token permissions, so the
           # integration tests cannot run even if the label is added. Maintainers
           # must push the branch to this repo to run them.
-          if [[ "${{ github.event.pull_request.head.repo.fork }}" == "true" ]]; then
+          if [[ "$PR_FORK" == "true" ]]; then
             echo "Fork PR - integration tests cannot run (no access to secrets)."
             echo "has_label=false" >> $GITHUB_OUTPUT
-          elif [[ "${{ contains(github.event.pull_request.labels.*.name, 'ready for testing') }}" == "true" ]]; then
+          elif [[ "$HAS_LABEL" == "true" ]]; then
             echo "has_label=true" >> $GITHUB_OUTPUT
           else
             echo "has_label=false" >> $GITHUB_OUTPUT
@@ -88,11 +91,16 @@ jobs:
           echo "CWA_GITHUB_TEST_REPO_BRANCH=${CWA_GITHUB_TEST_REPO_BRANCH:-${{ env.CWA_GITHUB_TEST_REPO_BRANCH }}}" >> "$GITHUB_OUTPUT"
 
       - name: Echo test variables
+        env:
+          GITHUB_SHA_VAL: ${{ github.sha }}
+          OUT_REPO_NAME: ${{ steps.set-outputs.outputs.CWA_GITHUB_TEST_REPO_NAME }}
+          OUT_REPO_URL: ${{ steps.set-outputs.outputs.CWA_GITHUB_TEST_REPO_URL }}
+          OUT_REPO_BRANCH: ${{ steps.set-outputs.outputs.CWA_GITHUB_TEST_REPO_BRANCH }}
         run: |
-          echo "build_id: ${{ github.sha }}"
-          echo "CWA_GITHUB_TEST_REPO_NAME: ${{ steps.set-outputs.outputs.CWA_GITHUB_TEST_REPO_NAME }}"
-          echo "CWA_GITHUB_TEST_REPO_URL: ${{ steps.set-outputs.outputs.CWA_GITHUB_TEST_REPO_URL }}"
-          echo "CWA_GITHUB_TEST_REPO_BRANCH: ${{ steps.set-outputs.outputs.CWA_GITHUB_TEST_REPO_BRANCH }}"
+          echo "build_id: $GITHUB_SHA_VAL"
+          echo "CWA_GITHUB_TEST_REPO_NAME: $OUT_REPO_NAME"
+          echo "CWA_GITHUB_TEST_REPO_URL: $OUT_REPO_URL"
+          echo "CWA_GITHUB_TEST_REPO_BRANCH: $OUT_REPO_BRANCH"
 
       - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1
         with:
@@ -169,9 +177,12 @@ jobs:
         run: |
           # GitHub Actions matrix limit is 256 jobs per workflow run.
           # Use 200 per page for headroom. Up to 5 pages supported (1000 tests).
-          # The ec2_linux_matrix is @json-encoded (double-encoded), so decode first.
+          # Read the filtered matrix from the file written by the "Generate matrix"
+          # step instead of a step-level env var: the matrix can exceed the
+          # kernel's ARG_MAX (~2 MiB) on execve, which would prevent bash from
+          # starting for this step.
           PAGE_SIZE=200
-          FULL_MATRIX=$(echo '${{ steps.set-matrix.outputs.ec2_linux_matrix }}' | jq -r '.')
+          FULL_MATRIX=$(cat filtered_matrix.json)
           TOTAL=$(echo "$FULL_MATRIX" | jq 'length')
           PAGE_COUNT=$(( (TOTAL + PAGE_SIZE - 1) / PAGE_SIZE ))
 
@@ -190,9 +201,12 @@ jobs:
           done
 
       - name: Echo test plan matrix
+        env:
+          EC2_LINUX_PAGE_COUNT: ${{ steps.paginate-matrix.outputs.ec2_linux_matrix_page_count }}
+          EC2_SELINUX_MATRIX: ${{ steps.set-matrix.outputs.ec2_selinux_matrix }}
         run: |
-          echo "ec2_linux_matrix pages: ${{ steps.paginate-matrix.outputs.ec2_linux_matrix_page_count }}"
-          echo "ec2_selinux_matrix: ${{ steps.set-matrix.outputs.ec2_selinux_matrix }}"
+          echo "ec2_linux_matrix pages: $EC2_LINUX_PAGE_COUNT"
+          echo "ec2_selinux_matrix: $EC2_SELINUX_MATRIX"
 
 
   EC2LinuxIntegrationTest-0:
@@ -355,23 +369,28 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Check for ready for testing label
+        env:
+          PR_FORK: ${{ github.event.pull_request.head.repo.fork }}
+          PR_DRAFT: ${{ github.event.pull_request.draft }}
+          HAS_SKIP_TESTING: ${{ contains(github.event.pull_request.labels.*.name, 'skip testing') }}
+          HAS_READY_FOR_TESTING: ${{ contains(github.event.pull_request.labels.*.name, 'ready for testing') }}
         run: |
-          if [[ "${{ github.event.pull_request.head.repo.fork }}" == "true" ]]; then
+          if [[ "$PR_FORK" == "true" ]]; then
             echo "Fork PR - integration tests skipped (no access to secrets). Push branch to this repo to run them."
             exit 0
           fi
 
-          if [[ "${{ github.event.pull_request.draft }}" == "true" ]]; then
+          if [[ "$PR_DRAFT" == "true" ]]; then
             echo "Draft PR - skipping label check."
             exit 0
           fi
 
-          if [[ "${{ contains(github.event.pull_request.labels.*.name, 'skip testing') }}" == "true" ]]; then
+          if [[ "$HAS_SKIP_TESTING" == "true" ]]; then
             echo "'skip testing' label found - bypassing integration test requirement."
             exit 0
           fi
 
-          if [[ "${{ contains(github.event.pull_request.labels.*.name, 'ready for testing') }}" != "true" ]]; then
+          if [[ "$HAS_READY_FOR_TESTING" != "true" ]]; then
             echo "Missing 'ready for testing' label. Please add before merging."
             exit 1
           fi
@@ -388,16 +407,15 @@ jobs:
     if: always()
     steps:
       - name: Check Job Status
+        env:
+          NEEDS_JSON: ${{ toJSON(needs) }}
         run: |
-          # Convert needs context to JSON and process with jq
-          needs_json='${{ toJSON(needs) }}'
-
           failed_jobs=()
           successful_jobs=()
 
           # Loop through all jobs in needs context
-          for job in $(echo "$needs_json" | jq -r 'keys[]'); do
-            result=$(echo "$needs_json" | jq -r ".[\"$job\"].result")
+          for job in $(echo "$NEEDS_JSON" | jq -r 'keys[]'); do
+            result=$(echo "$NEEDS_JSON" | jq -r ".[\"$job\"].result")
 
             if [[ "$result" == "failure" ]]; then
               failed_jobs+=("$job")

.github/workflows/application-signals-e2e-test.yml

@@ -40,9 +40,12 @@ jobs:
             echo "Build SHA does not match test SHA"
             exit 1
           fi
-      - run: |
+      - env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          INPUT_BUILD_RUN_ID: ${{ inputs.build_run_id }}
+        run: |
           for i in {1..6}; do
-            conclusion=$(gh run view ${{ inputs.build_run_id }} --repo $GITHUB_REPOSITORY --json conclusion -q '.conclusion')
+            conclusion=$(gh run view "$INPUT_BUILD_RUN_ID" --repo "$GITHUB_REPOSITORY" --json conclusion -q '.conclusion')
             if [[ "$conclusion" == "success" ]]; then
               echo "Run succeeded"
               exit 0
@@ -55,8 +58,6 @@ jobs:
           done
           echo "Timed out waiting for workflow"
           exit 1
-        env:
-          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
 
   java-eks-e2e-test:
     needs: CheckBuildTestArtifacts

.github/workflows/build-test-artifacts.yml

@@ -128,9 +128,12 @@ jobs:
     permissions:
       actions: write
     steps:
-      - run: gh workflow run integration-test.yml --ref ${{ github.ref_name }} --repo $GITHUB_REPOSITORY -f build_run_id=${{ github.run_id }} -f build_sha=${{ github.sha }}
-        env:
+      - env:
           GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          REF_NAME: ${{ github.ref_name }}
+          RUN_ID: ${{ github.run_id }}
+          GITHUB_SHA_VAL: ${{ github.sha }}
+        run: gh workflow run integration-test.yml --ref "$REF_NAME" --repo "$GITHUB_REPOSITORY" -f "build_run_id=$RUN_ID" -f "build_sha=$GITHUB_SHA_VAL"
 
   StartApplicationSignalsE2ETests:
     needs: [ BuildAndUploadPackages, BuildAndUploadITAR, BuildAndUploadCN, BuildDocker, BuildDistributor ]
@@ -140,9 +143,12 @@ jobs:
     permissions:
       actions: write
     steps:
-      - run: gh workflow run application-signals-e2e-test.yml --ref ${{ github.ref_name }} --repo $GITHUB_REPOSITORY -f build_run_id=${{ github.run_id }} -f build_sha=${{ github.sha }}
-        env:
+      - env:
           GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          REF_NAME: ${{ github.ref_name }}
+          RUN_ID: ${{ github.run_id }}
+          GITHUB_SHA_VAL: ${{ github.sha }}
+        run: gh workflow run application-signals-e2e-test.yml --ref "$REF_NAME" --repo "$GITHUB_REPOSITORY" -f "build_run_id=$RUN_ID" -f "build_sha=$GITHUB_SHA_VAL"
 
   StartEKSE2ETests:
     needs: [ BuildAndUploadPackages, BuildAndUploadITAR, BuildAndUploadCN, BuildDocker, BuildDistributor ]
@@ -151,9 +157,11 @@ jobs:
     permissions:
       actions: write
     steps:
-      - run: gh workflow run e2e-test.yml --ref ${{ github.ref_name }} --repo $GITHUB_REPOSITORY -f build_sha=${{ github.sha }}
-        env:
+      - env:
           GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          REF_NAME: ${{ github.ref_name }}
+          GITHUB_SHA_VAL: ${{ github.sha }}
+        run: gh workflow run e2e-test.yml --ref "$REF_NAME" --repo "$GITHUB_REPOSITORY" -f "build_sha=$GITHUB_SHA_VAL"
 
   StartWorkloadDiscoveryIntegrationTests:
     needs: [ BuildAndUploadPackages, BuildAndUploadITAR, BuildAndUploadCN, BuildDocker, BuildDistributor ]
@@ -162,6 +170,9 @@ jobs:
     permissions:
       actions: write
     steps:
-      - run: gh workflow run wd-integration-test.yml --ref ${{ github.ref_name }} --repo $GITHUB_REPOSITORY -f build_run_id=${{ github.run_id }} -f build_sha=${{ github.sha }}
-        env:
+      - env:
           GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          REF_NAME: ${{ github.ref_name }}
+          RUN_ID: ${{ github.run_id }}
+          GITHUB_SHA_VAL: ${{ github.sha }}
+        run: gh workflow run wd-integration-test.yml --ref "$REF_NAME" --repo "$GITHUB_REPOSITORY" -f "build_run_id=$RUN_ID" -f "build_sha=$GITHUB_SHA_VAL"

.github/workflows/clean-aws-resources.yml

@@ -138,7 +138,9 @@ jobs:
 
       - name: Clean old host
         working-directory: tool/clean
-        run: go run ./clean_host/clean_host.go ${{ matrix.region }}
+        env:
+          MATRIX_REGION: ${{ matrix.region }}
+        run: go run ./clean_host/clean_host.go "$MATRIX_REGION"
 
   clean-hosts-china:
     runs-on: ubuntu-latest

.github/workflows/e2e-test.yml

@@ -123,13 +123,20 @@ jobs:
           echo "::set-output name=ECR_TARGET_ALLOCATOR_REPO::$(echo "${{ vars.ECR_TARGET_ALLOCATOR_STAGING_REPO }}" | awk -F'/' '{print $NF}')"
 
       - name: Echo test variables
+        env:
+          OUT_REPO_NAME: ${{ steps.set-outputs.outputs.CWA_GITHUB_TEST_REPO_NAME }}
+          OUT_REPO_URL: ${{ steps.set-outputs.outputs.CWA_GITHUB_TEST_REPO_URL }}
+          OUT_REPO_BRANCH: ${{ steps.set-outputs.outputs.CWA_GITHUB_TEST_REPO_BRANCH }}
+          OUT_ECR_INTEGRATION_TEST_REPO: ${{ steps.set-outputs.outputs.ECR_INTEGRATION_TEST_REPO }}
+          OUT_ECR_OPERATOR_REPO: ${{ steps.set-outputs.outputs.ECR_OPERATOR_REPO }}
+          OUT_ECR_TARGET_ALLOCATOR_REPO: ${{ steps.set-outputs.outputs.ECR_TARGET_ALLOCATOR_REPO }}
         run: |
-          echo "CWA_GITHUB_TEST_REPO_NAME: ${{ steps.set-outputs.outputs.CWA_GITHUB_TEST_REPO_NAME }}"
-          echo "CWA_GITHUB_TEST_REPO_URL: ${{ steps.set-outputs.outputs.CWA_GITHUB_TEST_REPO_URL }}"
-          echo "CWA_GITHUB_TEST_REPO_BRANCH: ${{ steps.set-outputs.outputs.CWA_GITHUB_TEST_REPO_BRANCH }}"
-          echo "ECR_INTEGRATION_TEST_REPO: ${{ steps.set-outputs.outputs.ECR_INTEGRATION_TEST_REPO }}"
-          echo "ECR_OPERATOR_REPO: ${{ steps.set-outputs.outputs.ECR_OPERATOR_REPO }}"
-          echo "ECR_TARGET_ALLOCATOR_REPO: ${{ steps.set-outputs.outputs.ECR_TARGET_ALLOCATOR_REPO }}"
+          echo "CWA_GITHUB_TEST_REPO_NAME: $OUT_REPO_NAME"
+          echo "CWA_GITHUB_TEST_REPO_URL: $OUT_REPO_URL"
+          echo "CWA_GITHUB_TEST_REPO_BRANCH: $OUT_REPO_BRANCH"
+          echo "ECR_INTEGRATION_TEST_REPO: $OUT_ECR_INTEGRATION_TEST_REPO"
+          echo "ECR_OPERATOR_REPO: $OUT_ECR_OPERATOR_REPO"
+          echo "ECR_TARGET_ALLOCATOR_REPO: $OUT_ECR_TARGET_ALLOCATOR_REPO"
 
   GenerateTestMatrix:
     needs: [BuildAgent, BuildOperator]
@@ -156,8 +163,10 @@ jobs:
           echo "::set-output name=eks_e2e_jmx_matrix::$(echo $(cat generator/resources/eks_e2e_jmx_complete_test_matrix.json))"
 
       - name: Echo test plan matrix
+        env:
+          EKS_E2E_JMX_MATRIX: ${{ steps.set-matrix.outputs.eks_e2e_jmx_matrix }}
         run: |
-          echo "eks_e2e_jmx_matrix: ${{ steps.set-matrix.outputs.eks_e2e_jmx_matrix }}"
+          echo "eks_e2e_jmx_matrix: $EKS_E2E_JMX_MATRIX"
 
   EKSE2EJVMTomcatTestHelm:
     needs: [ GetLatestOperatorCommitSHA, GenerateTestMatrix, OutputEnvVariables ]