ci: Add agent configurability and convenient local testing to performance tests

.github/actions/run-perf-test/action.yml

@@ -1,7 +1,7 @@
 name: Run Performance Test
 description: >
   Downloads the New Relic .NET agent (when requested) and runs the performance
-  test suite via run-perf-tests.sh, then uploads the results and log artifacts.
+  test suite via run-perf-test.py, then uploads the results and log artifacts.
 
 inputs:
   run-label:
@@ -37,9 +37,15 @@ inputs:
   collector-host:
     description: 'New Relic collector host (only used when attach-agent is true)'
     default: ''
+  agent-env:
+    description: 'Extra environment variables for the test app/agent, semicolon-separated NAME=VALUE pairs (e.g. VAR1=val1;VAR2=val2)'
+    default: ''
   github-token:
     description: 'GitHub token for downloading artifacts'
     required: true
+  agent-app-name:
+    description: 'New Relic application name to use in the test (only used when attach-agent is true)'
+    default: 'dotnet-agent-perf-test'
 
 runs:
   using: composite
@@ -78,7 +84,7 @@ runs:
           exit 1
         fi
         echo "Found workflow run ID: $WORKFLOW_RUN_ID"
-        echo "run_id=$WORKFLOW_RUN_ID" >> $GITHUB_OUTPUT
+        echo "run_id=$WORKFLOW_RUN_ID" >> "$GITHUB_OUTPUT"
 
     - name: Download agent homefolders from release build
       if: inputs.attach-agent == 'true' && inputs.agent-source == 'github_release'
@@ -142,16 +148,25 @@ runs:
       shell: bash
       working-directory: ${{ env.PERF_TEST_DIR }}
       run: |
-        bash run-perf-tests.sh \
+        EXTRA_ENV_ARGS=()
+        IFS=';' read -ra ENV_PAIRS <<< "${{ inputs.agent-env }}"
+        for pair in "${ENV_PAIRS[@]}"; do
+          [[ -z "$pair" ]] && continue
+          echo "Adding extra environment variable: $pair"
+          EXTRA_ENV_ARGS+=("--env" "$pair")
+        done
+
+        python3 run-perf-test.py \
           --attach-agent    "${{ inputs.attach-agent }}" \
           --agent-home      "${{ github.workspace }}/${{ env.PERF_TEST_DIR }}/agent-home" \
-          --app-name        "dotnet-agent-perf-test-${{ github.run_id }}-${{ inputs.run-label }}" \
+          --app-name        "${{ inputs.agent-app-name }}" \
           --test-duration   "${{ inputs.test-duration }}" \
           --locust-users    "${{ inputs.locust-users }}" \
           --locust-spawn-rate "${{ inputs.locust-spawn-rate }}" \
           --dotnet-version  "${{ inputs.dotnet-version }}" \
           --license-key     "${{ inputs.license-key }}" \
-          --collector-host  "${{ inputs.collector-host }}"
+          --collector-host  "${{ inputs.collector-host }}" \
+          "${EXTRA_ENV_ARGS[@]}"
 
     # -------------------------------------------------------------------------
     # Upload artifacts

.github/workflows/compare_performance.yml

@@ -4,6 +4,9 @@ name: Compare Performance
 # baseline plus up to 4 agent runs) and generates a comparative report with
 # ScottPlot charts and a markdown summary table.
 #
+# Nightly scheduled run compares the latest published release against the most
+# recent successful scheduled all_solutions.yml build (today's nightly build).
+#
 # Required secrets (when any agent run is included):
 #   PERF_TEST_ACCOUNT  - New Relic license key and collector host separated by a
 #                        comma.  Example: XXXXXXXXXXXX,collector.newrelic.com
@@ -22,6 +25,11 @@ on:
         type: boolean
         default: true
         required: false
+      no_agent_env:
+        description: 'Extra env vars for the no-agent run, semicolon-separated NAME=VALUE pairs (e.g. VAR1=val1;VAR2=val2)'
+        type: string
+        default: ''
+        required: false
 
       # --- Agent run 1 (always runs) ---
       run1_label:
@@ -34,6 +42,11 @@ on:
         type: string
         default: ''
         required: false
+      run1_agent_env:
+        description: 'Extra env vars for agent run 1, semicolon-separated NAME=VALUE pairs (e.g. VAR1=val1;VAR2=val2)'
+        type: string
+        default: ''
+        required: false
 
       # --- Agent run 2 (optional) ---
       include_run2:
@@ -51,6 +64,11 @@ on:
         type: string
         default: ''
         required: false
+      run2_agent_env:
+        description: 'Extra env vars for agent run 2, semicolon-separated NAME=VALUE pairs (e.g. VAR1=val1;VAR2=val2)'
+        type: string
+        default: ''
+        required: false
 
       # --- Agent run 3 (optional) ---
       include_run3:
@@ -68,6 +86,11 @@ on:
         type: string
         default: ''
         required: false
+      run3_agent_env:
+        description: 'Extra env vars for agent run 3, semicolon-separated NAME=VALUE pairs (e.g. VAR1=val1;VAR2=val2)'
+        type: string
+        default: ''
+        required: false
 
       # --- Agent run 4 (optional) ---
       include_run4:
@@ -85,6 +108,11 @@ on:
         type: string
         default: ''
         required: false
+      run4_agent_env:
+        description: 'Extra env vars for agent run 4, semicolon-separated NAME=VALUE pairs (e.g. VAR1=val1;VAR2=val2)'
+        type: string
+        default: ''
+        required: false
 
       # --- Shared test parameters ---
       test_duration:
@@ -109,8 +137,9 @@ on:
         required: false
 
   schedule:
-    # Run nightly on weekdays: no-agent baseline vs. latest published release
-    - cron: '0 6 * * 1-5'
+    # Run nightly on weekdays: latest published release vs. latest scheduled nightly build
+    # all_solutions.yml runs at 9:00 UTC and may take more than an hour, so start at 11:00 UTC
+    - cron: '0 11 * * 1-5'
 
 env:
   DOTNET_NOLOGO: true
@@ -123,6 +152,7 @@ concurrency:
 
 permissions:
   contents: read
+  actions: read
 
 jobs:
   # ---------------------------------------------------------------------------
@@ -158,6 +188,11 @@ jobs:
       run4_agent_source:   ${{ steps.inputs.outputs.run4_agent_source }}
       run4_agent_version:  ${{ steps.inputs.outputs.run4_agent_version }}
       run4_build_run_id:   ${{ steps.inputs.outputs.run4_build_run_id }}
+      no_agent_env:        ${{ steps.inputs.outputs.no_agent_env }}
+      run1_agent_env:      ${{ steps.inputs.outputs.run1_agent_env }}
+      run2_agent_env:      ${{ steps.inputs.outputs.run2_agent_env }}
+      run3_agent_env:      ${{ steps.inputs.outputs.run3_agent_env }}
+      run4_agent_env:      ${{ steps.inputs.outputs.run4_agent_env }}
       test_duration:       ${{ steps.inputs.outputs.test_duration }}
       locust_users:        ${{ steps.inputs.outputs.locust_users }}
       locust_spawn_rate:   ${{ steps.inputs.outputs.locust_spawn_rate }}
@@ -169,6 +204,7 @@ jobs:
         id: inputs
         env:
           PERF_TEST_ACCOUNT: ${{ secrets.PERF_TEST_ACCOUNT }}
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         run: |
           # Parse an agent string into agent_source / agent_version / build_run_id
           # and write the three outputs prefixed with $1.
@@ -177,59 +213,82 @@ jobs:
             local AGENT="$2"
             if echo "$AGENT" | grep -qE '^[0-9]+\.[0-9]'; then
               # Version string (e.g. "10.49.0") → github_release
-              echo "${PREFIX}_agent_source=github_release" >> $GITHUB_OUTPUT
-              echo "${PREFIX}_agent_version=$AGENT"        >> $GITHUB_OUTPUT
-              echo "${PREFIX}_build_run_id="               >> $GITHUB_OUTPUT
+              echo "${PREFIX}_agent_source=github_release" >> "$GITHUB_OUTPUT"
+              echo "${PREFIX}_agent_version=$AGENT"        >> "$GITHUB_OUTPUT"
+              echo "${PREFIX}_build_run_id="               >> "$GITHUB_OUTPUT"
             elif [ -n "$AGENT" ]; then
               # Pure digits → build_artifact run ID
-              echo "${PREFIX}_agent_source=build_artifact" >> $GITHUB_OUTPUT
-              echo "${PREFIX}_agent_version="              >> $GITHUB_OUTPUT
-              echo "${PREFIX}_build_run_id=$AGENT"         >> $GITHUB_OUTPUT
+              echo "${PREFIX}_agent_source=build_artifact" >> "$GITHUB_OUTPUT"
+              echo "${PREFIX}_agent_version="              >> "$GITHUB_OUTPUT"
+              echo "${PREFIX}_build_run_id=$AGENT"         >> "$GITHUB_OUTPUT"
             else
               # Empty → latest release
-              echo "${PREFIX}_agent_source=github_release" >> $GITHUB_OUTPUT
-              echo "${PREFIX}_agent_version="              >> $GITHUB_OUTPUT
-              echo "${PREFIX}_build_run_id="               >> $GITHUB_OUTPUT
+              echo "${PREFIX}_agent_source=github_release" >> "$GITHUB_OUTPUT"
+              echo "${PREFIX}_agent_version="              >> "$GITHUB_OUTPUT"
+              echo "${PREFIX}_build_run_id="               >> "$GITHUB_OUTPUT"
             fi
           }
 
           if [ "${{ github.event_name }}" = "schedule" ]; then
-            echo "include_no_agent=true"       >> $GITHUB_OUTPUT
-            echo "run1_label=latest-release"   >> $GITHUB_OUTPUT
+            # Find the most recent successful all_solutions.yml run triggered by schedule
+            NIGHTLY_RUN_ID=$(gh run list \
+              --workflow all_solutions.yml \
+              --event schedule \
+              --status success \
+              --limit 1 \
+              --json databaseId \
+              --jq '.[0].databaseId')
+            if [ -z "$NIGHTLY_RUN_ID" ]; then
+              echo "ERROR: Could not find a recent successful scheduled all_solutions.yml run" >&2
+              exit 1
+            fi
+            echo "Found nightly build run ID: $NIGHTLY_RUN_ID"
+            echo "include_no_agent=true"           >> "$GITHUB_OUTPUT"
+            echo "run1_label=latest-release"       >> "$GITHUB_OUTPUT"
             parse_agent "run1" ""
-            echo "include_run2=false"          >> $GITHUB_OUTPUT
-            echo "run2_label="                 >> $GITHUB_OUTPUT
-            parse_agent "run2" ""
-            echo "include_run3=false"          >> $GITHUB_OUTPUT
-            echo "run3_label="                 >> $GITHUB_OUTPUT
+            echo "include_run2=true"               >> "$GITHUB_OUTPUT"
+            echo "run2_label=nightly-build"        >> "$GITHUB_OUTPUT"
+            parse_agent "run2" "$NIGHTLY_RUN_ID"
+            echo "include_run3=false"              >> "$GITHUB_OUTPUT"
+            echo "run3_label="                     >> "$GITHUB_OUTPUT"
             parse_agent "run3" ""
-            echo "include_run4=false"          >> $GITHUB_OUTPUT
-            echo "run4_label="                 >> $GITHUB_OUTPUT
+            echo "include_run4=false"              >> "$GITHUB_OUTPUT"
+            echo "run4_label="                     >> "$GITHUB_OUTPUT"
             parse_agent "run4" ""
-            echo "test_duration=2m"            >> $GITHUB_OUTPUT
-            echo "locust_users=10"             >> $GITHUB_OUTPUT
-            echo "locust_spawn_rate=2"         >> $GITHUB_OUTPUT
-            echo "dotnet_version=10.0"         >> $GITHUB_OUTPUT
+            echo "no_agent_env="   >> "$GITHUB_OUTPUT"
+            echo "run1_agent_env=" >> "$GITHUB_OUTPUT"
+            echo "run2_agent_env=" >> "$GITHUB_OUTPUT"
+            echo "run3_agent_env=" >> "$GITHUB_OUTPUT"
+            echo "run4_agent_env=" >> "$GITHUB_OUTPUT"
+            echo "test_duration=2m"                >> "$GITHUB_OUTPUT"
+            echo "locust_users=10"                 >> "$GITHUB_OUTPUT"
+            echo "locust_spawn_rate=2"             >> "$GITHUB_OUTPUT"
+            echo "dotnet_version=10.0"             >> "$GITHUB_OUTPUT"
           else
-            echo "include_no_agent=${{ inputs.include_no_agent }}" >> $GITHUB_OUTPUT
-            echo "run1_label=${{ inputs.run1_label }}"             >> $GITHUB_OUTPUT
+            echo "include_no_agent=${{ inputs.include_no_agent }}" >> "$GITHUB_OUTPUT"
+            echo "run1_label=${{ inputs.run1_label }}"             >> "$GITHUB_OUTPUT"
             parse_agent "run1" "${{ inputs.run1_agent }}"
-            echo "include_run2=${{ inputs.include_run2 }}"         >> $GITHUB_OUTPUT
-            echo "run2_label=${{ inputs.run2_label }}"             >> $GITHUB_OUTPUT
+            echo "include_run2=${{ inputs.include_run2 }}"         >> "$GITHUB_OUTPUT"
+            echo "run2_label=${{ inputs.run2_label }}"             >> "$GITHUB_OUTPUT"
             parse_agent "run2" "${{ inputs.run2_agent }}"
-            echo "include_run3=${{ inputs.include_run3 }}"         >> $GITHUB_OUTPUT
-            echo "run3_label=${{ inputs.run3_label }}"             >> $GITHUB_OUTPUT
+            echo "include_run3=${{ inputs.include_run3 }}"         >> "$GITHUB_OUTPUT"
+            echo "run3_label=${{ inputs.run3_label }}"             >> "$GITHUB_OUTPUT"
             parse_agent "run3" "${{ inputs.run3_agent }}"
-            echo "include_run4=${{ inputs.include_run4 }}"         >> $GITHUB_OUTPUT
-            echo "run4_label=${{ inputs.run4_label }}"             >> $GITHUB_OUTPUT
+            echo "include_run4=${{ inputs.include_run4 }}"         >> "$GITHUB_OUTPUT"
+            echo "run4_label=${{ inputs.run4_label }}"             >> "$GITHUB_OUTPUT"
             parse_agent "run4" "${{ inputs.run4_agent }}"
-            echo "test_duration=${{ inputs.test_duration }}"       >> $GITHUB_OUTPUT
-            echo "locust_users=${{ inputs.locust_users }}"         >> $GITHUB_OUTPUT
-            echo "locust_spawn_rate=${{ inputs.locust_spawn_rate }}" >> $GITHUB_OUTPUT
-            echo "dotnet_version=${{ inputs.dotnet_version }}"     >> $GITHUB_OUTPUT
+            echo "no_agent_env=${{ inputs.no_agent_env }}"     >> "$GITHUB_OUTPUT"
+            echo "run1_agent_env=${{ inputs.run1_agent_env }}" >> "$GITHUB_OUTPUT"
+            echo "run2_agent_env=${{ inputs.run2_agent_env }}" >> "$GITHUB_OUTPUT"
+            echo "run3_agent_env=${{ inputs.run3_agent_env }}" >> "$GITHUB_OUTPUT"
+            echo "run4_agent_env=${{ inputs.run4_agent_env }}" >> "$GITHUB_OUTPUT"
+            echo "test_duration=${{ inputs.test_duration }}"       >> "$GITHUB_OUTPUT"
+            echo "locust_users=${{ inputs.locust_users }}"         >> "$GITHUB_OUTPUT"
+            echo "locust_spawn_rate=${{ inputs.locust_spawn_rate }}" >> "$GITHUB_OUTPUT"
+            echo "dotnet_version=${{ inputs.dotnet_version }}"     >> "$GITHUB_OUTPUT"
           fi
-          echo "license_key=${PERF_TEST_ACCOUNT%%,*}"   >> $GITHUB_OUTPUT
-          echo "collector_host=${PERF_TEST_ACCOUNT##*,}" >> $GITHUB_OUTPUT
+          echo "license_key=${PERF_TEST_ACCOUNT%%,*}"   >> "$GITHUB_OUTPUT"
+          echo "collector_host=${PERF_TEST_ACCOUNT##*,}" >> "$GITHUB_OUTPUT"
 
   # ---------------------------------------------------------------------------
   # Parallel test runs
@@ -253,6 +312,7 @@ jobs:
         with:
           run-label:         no-agent
           attach-agent:      'false'
+          agent-env:         ${{ needs.resolve-inputs.outputs.no_agent_env }}
           test-duration:     ${{ needs.resolve-inputs.outputs.test_duration }}
           locust-users:      ${{ needs.resolve-inputs.outputs.locust_users }}
           locust-spawn-rate: ${{ needs.resolve-inputs.outputs.locust_spawn_rate }}
@@ -282,6 +342,7 @@ jobs:
           agent-source:      ${{ needs.resolve-inputs.outputs.run1_agent_source }}
           agent-version:     ${{ needs.resolve-inputs.outputs.run1_agent_version }}
           build-run-id:      ${{ needs.resolve-inputs.outputs.run1_build_run_id }}
+          agent-env:         ${{ needs.resolve-inputs.outputs.run1_agent_env }}
           test-duration:     ${{ needs.resolve-inputs.outputs.test_duration }}
           locust-users:      ${{ needs.resolve-inputs.outputs.locust_users }}
           locust-spawn-rate: ${{ needs.resolve-inputs.outputs.locust_spawn_rate }}
@@ -314,6 +375,7 @@ jobs:
           agent-source:      ${{ needs.resolve-inputs.outputs.run2_agent_source }}
           agent-version:     ${{ needs.resolve-inputs.outputs.run2_agent_version }}
           build-run-id:      ${{ needs.resolve-inputs.outputs.run2_build_run_id }}
+          agent-env:         ${{ needs.resolve-inputs.outputs.run2_agent_env }}
           test-duration:     ${{ needs.resolve-inputs.outputs.test_duration }}
           locust-users:      ${{ needs.resolve-inputs.outputs.locust_users }}
           locust-spawn-rate: ${{ needs.resolve-inputs.outputs.locust_spawn_rate }}
@@ -346,6 +408,7 @@ jobs:
           agent-source:      ${{ needs.resolve-inputs.outputs.run3_agent_source }}
           agent-version:     ${{ needs.resolve-inputs.outputs.run3_agent_version }}
           build-run-id:      ${{ needs.resolve-inputs.outputs.run3_build_run_id }}
+          agent-env:         ${{ needs.resolve-inputs.outputs.run3_agent_env }}
           test-duration:     ${{ needs.resolve-inputs.outputs.test_duration }}
           locust-users:      ${{ needs.resolve-inputs.outputs.locust_users }}
           locust-spawn-rate: ${{ needs.resolve-inputs.outputs.locust_spawn_rate }}
@@ -378,6 +441,7 @@ jobs:
           agent-source:      ${{ needs.resolve-inputs.outputs.run4_agent_source }}
           agent-version:     ${{ needs.resolve-inputs.outputs.run4_agent_version }}
           build-run-id:      ${{ needs.resolve-inputs.outputs.run4_build_run_id }}
+          agent-env:         ${{ needs.resolve-inputs.outputs.run4_agent_env }}
           test-duration:     ${{ needs.resolve-inputs.outputs.test_duration }}
           locust-users:      ${{ needs.resolve-inputs.outputs.locust_users }}
           locust-spawn-rate: ${{ needs.resolve-inputs.outputs.locust_spawn_rate }}