test: add Q CLI and MCP integration testing

.github/actions/verify-mcp-usage/action.yml

@@ -0,0 +1,51 @@
+name: "Verify MCP Usage"
+description: "Verifies that both CloudWatch and Application Signals MCP tools were called"
+
+inputs:
+  artifact_name:
+    description: "Name of the artifact containing test outputs"
+    required: true
+  output_path:
+    description: "Path to download test outputs"
+    required: false
+    default: "./test-outputs"
+
+runs:
+  using: "composite"
+  steps:
+    - name: Download test outputs
+      uses: actions/download-artifact@v4
+      with:
+        name: ${{ inputs.artifact_name }}
+        path: ${{ inputs.output_path }}
+
+    - name: Verify MCP tool usage
+      shell: bash
+      run: |
+        echo "Checking for MCP tool usage..."
+
+        RAW_OUTPUT_FILE="${{ inputs.output_path }}/awsapm-raw-output.txt"
+
+        if [[ ! -f "$RAW_OUTPUT_FILE" ]]; then
+          echo "ERROR: Raw output file not found"
+          exit 1
+        fi
+
+        # Check for both MCP tool calls
+        if grep -qE "Using tool: list_monitored_services.*from mcp server.*applicationsignals" "$RAW_OUTPUT_FILE"; then
+          echo "PASSED - list_monitored_services tool was called from AWS Application Signals MCP"
+        else
+          echo "FAILED: list_monitored_services tool call not found"
+          echo "Raw output content:"
+          cat "$RAW_OUTPUT_FILE"
+          exit 1
+        fi
+
+        if grep -qE "Using tool: get_active_alarms.*from mcp server.*awslabs\.cloudwatch-mcp-server" "$RAW_OUTPUT_FILE"; then
+          echo "PASSED - get_active_alarms tool was called from CloudWatch MCP"
+        else
+          echo "FAILED: get_active_alarms tool call not found"
+          echo "Raw output content:"
+          cat "$RAW_OUTPUT_FILE"
+          exit 1
+        fi

.github/workflows/integ-test.yml

@@ -0,0 +1,57 @@
+name: Integration Test
+
+on:
+  push:
+    branches:
+      - integ-test
+      - main
+  workflow_dispatch:
+
+env:
+  AWS_DEFAULT_REGION: us-east-1
+
+jobs:
+  awsapm-integ-test:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: write
+      pull-requests: write
+      issues: write
+      id-token: write
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Configure AWS credentials
+        uses: aws-actions/configure-aws-credentials@v4
+        with:
+          role-to-assume: ${{ secrets.AWSAPM_ROLE_ARN }}
+          aws-region: ${{ env.AWS_DEFAULT_REGION }}
+
+      - name: Run Application Observability for AWS Investigation
+        uses: ./
+        with:
+          test_mode: "true"
+          custom_prompt: "Use the list_monitored_services tool to show me all services currently monitored by AWS Application Signals. Also use the get_active_alarms tool to show me all active CloudWatch alarms."
+
+      - name: Upload test outputs
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: integration-test-outputs
+          path: |
+            ${{ runner.temp }}/awsapm-output/
+          retention-days: 7
+
+  verify-mcp-usage:
+    needs: awsapm-integ-test
+    runs-on: ubuntu-latest
+    if: always()
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Verify MCP tool usage
+        uses: ./.github/actions/verify-mcp-usage
+        with:
+          artifact_name: integration-test-outputs

.github/workflows/soak-test.yml

@@ -0,0 +1,81 @@
+name: Soak Test
+
+on:
+  schedule:
+    - cron: '0 * * * *'  # every hour
+
+env:
+  AWS_DEFAULT_REGION: us-east-1
+
+jobs:
+  awsapm-soak-test:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: write
+      pull-requests: write
+      issues: write
+      id-token: write
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Configure AWS credentials
+        uses: aws-actions/configure-aws-credentials@v4
+        with:
+          role-to-assume: ${{ secrets.AWSAPM_ROLE_ARN }}
+          aws-region: ${{ env.AWS_DEFAULT_REGION }}
+
+      - name: Run Application Observability for AWS Investigation - Soak Test
+        # TODO: Change to use published action once v1 is released
+        # uses: aws-actions/application-observability-for-aws@v1
+        uses: ./
+        with:
+          test_mode: "true"
+          custom_prompt: "Use the list_monitored_services tool to show me all services currently monitored by AWS Application Signals. Also use the get_active_alarms tool to show me all active CloudWatch alarms."
+
+      - name: Upload test outputs
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: soak-test-outputs
+          path: |
+            ${{ runner.temp }}/awsapm-output/
+          retention-days: 7
+
+  verify-mcp-usage:
+    needs: awsapm-soak-test
+    runs-on: ubuntu-latest
+    if: always()
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Verify MCP tool usage
+        uses: ./.github/actions/verify-mcp-usage
+        with:
+          artifact_name: soak-test-outputs
+
+  publish-metric:
+    needs: [awsapm-soak-test, verify-mcp-usage]
+    runs-on: ubuntu-latest
+    if: always()
+    permissions:
+      id-token: write
+    steps:
+      - name: Configure AWS Credentials
+        uses: aws-actions/configure-aws-credentials@a03048d87541d1d9fcf2ecf528a4a65ba9bd7838 #5.0.0
+        with:
+          role-to-assume: ${{ secrets.MONITORING_ROLE_ARN }}
+          aws-region: ${{ env.AWS_DEFAULT_REGION }}
+
+      - name: Publish soak test status
+        run: |
+          if [[ "${{ needs.awsapm-soak-test.result }}" == "success" && "${{ needs.verify-mcp-usage.result }}" == "success" ]]; then
+            value="1.0"
+          else
+            value="0.0"
+          fi
+
+          aws cloudwatch put-metric-data \
+            --namespace 'ADOT/GithubActions' \
+            --metric-data MetricName=Success,Value=$value,Dimensions="[{Name=repository,Value=${{ github.repository }}},{Name=branch,Value=${{ github.ref_name }}},{Name=workflow,Value=soak_test}]"

action.yml

@@ -32,6 +32,10 @@ inputs:
     description: "Enable CloudWatch MCP server for metrics, alarms, and log insights"
     required: false
     default: "true"
+  test_mode:
+    description: "Enable integration test mode (internal use only)"
+    required: false
+    default: "false"
 
 outputs:
   execution_file:
@@ -47,6 +51,15 @@ outputs:
 runs:
   using: "composite"
   steps:
+    - name: Validate test mode usage
+      if: inputs.test_mode == 'true'
+      shell: bash
+      run: |
+        if [[ "${{ github.repository }}" != "aws-actions/application-observability-for-aws" || "${{ github.ref_name }}" != "main" ]]; then
+          echo "::error::test_mode can only be used from the aws-actions/application-observability-for-aws repository main branch"
+          exit 1
+        fi
+
     - name: Install Node.js
       uses: actions/setup-node@v4
       with:
@@ -72,6 +85,8 @@ runs:
         ALLOWED_NON_WRITE_USERS: ${{ inputs.allowed_non_write_users }}
         GITHUB_RUN_ID: ${{ github.run_id }}
         DEFAULT_WORKFLOW_TOKEN: ${{ github.token }}
+        TRACING_MODE: ${{ inputs.tracing_mode }}
+        TEST_MODE: ${{ inputs.test_mode }}
 
     - name: Install CLI Tools
       if: steps.init.outputs.contains_trigger == 'true'

src/execute.js

@@ -35,10 +35,16 @@ async function run() {
 
     // Run Amazon Q Developer CLI investigation
     let investigationResult = '';
+    let rawOutput = '';
 
     try {
       core.info('Running Amazon Q Developer CLI investigation...');
       const executor = new AmazonQCLIExecutor(null);
+
+      executor.onRawOutput = (output) => {
+        rawOutput = output;
+      };
+
       investigationResult = await executor.execute(promptContent);
       core.info('Amazon Q Developer CLI investigation completed');
     } catch (error) {
@@ -63,6 +69,12 @@ Please check the workflow logs for more details and ensure proper authentication
     const responseFile = path.join(outputDir, `awsapm-response-${runId}.txt`);
     fs.writeFileSync(responseFile, cleanedResult);
 
+    // Save raw output for integration test validation
+    if (rawOutput) {
+      const rawOutputFile = path.join(outputDir, 'awsapm-raw-output.txt');
+      fs.writeFileSync(rawOutputFile, rawOutput);
+    }
+
     // Set outputs
     core.setOutput('execution_file', responseFile);
     core.setOutput('conclusion', 'success');

src/executors/base-cli-executor.js

@@ -217,6 +217,10 @@ class BaseCLIExecutor {
         const output = stdoutData || stderrData;
         core.debug(`[SUMMARY] Using ${stdoutData ? 'stdout' : 'stderr'} as output source`);
 
+        if (this.onRawOutput) {
+          this.onRawOutput(output);
+        }
+
         resolve({ output, exitCode: code || 0 });
       });
 

src/init.js

@@ -20,6 +20,8 @@ async function run() {
     const targetBranch = process.env.TARGET_BRANCH || '';
     const allowedNonWriteUsers = process.env.ALLOWED_NON_WRITE_USERS || '';
     const customPrompt = process.env.CUSTOM_PROMPT || '';
+    const tracingMode = process.env.TRACING_MODE || 'false';
+    const testMode = process.env.TEST_MODE || 'false';
 
     // Function to check for bot name trigger phrase
     // Note: Phrases like "@awsapm-user" will be also considered valid.
@@ -74,6 +76,11 @@ async function run() {
         // We still want to search for existing result comments when editing
         commentId = null; // Explicitly set to null for clarity
       }
+    } else if (testMode === 'true') {
+      // In test mode, always trigger and use custom prompt
+      containsTrigger = true;
+      triggerText = customPrompt;
+      triggerUsername = 'integration-test';
     }
 
     // Set output for action.yml to check
@@ -233,25 +240,35 @@ async function run() {
     // Remove bot name from the user's request
     const cleanedUserRequest = triggerText.replace(new RegExp(botName, 'gi'), '').trim();
 
-    // Use the dynamic prompt generation with PR context
-    const { createGeneralPrompt } = require('./prompt-builder');
+    if (testMode === 'true') {
+      // for integration test, use custom prompt directly
+      try {
+        fs.writeFileSync(promptFile, customPrompt);
+      } catch (error) {
+        core.error(`Failed to write custom prompt to file: ${error.message}`);
+        process.exit(1);
+      }
+    } else {
+      // Use the dynamic prompt generation with PR context
+      const { createGeneralPrompt } = require('./prompt-builder');
 
     try {
       const finalPrompt = await createGeneralPrompt(context, repoInfo, cleanedUserRequest, githubToken, awsapmBranch);
       fs.writeFileSync(promptFile, finalPrompt);
     } catch (promptError) {
       core.error(`Failed to generate dynamic prompt: ${promptError.message}`);
 
-      // Fallback to basic prompt if dynamic generation fails
-      let fallbackPrompt = '';
-      if (customPrompt) {
-        fallbackPrompt = customPrompt + '\n\n';
-      }
-      fallbackPrompt += `Please analyze this ${isPR ? 'pull request' : 'issue'} using AI Agent for insights.\n\n`;
-      fallbackPrompt += `Original request: ${cleanedUserRequest}\n\n`;
-      fallbackPrompt += `Context: This is a ${context.eventName} event in ${context.repo.owner}/${context.repo.repo}`;
+        // Fallback to basic prompt if dynamic generation fails
+        let fallbackPrompt = '';
+        if (customPrompt) {
+          fallbackPrompt = customPrompt + '\n\n';
+        }
+        fallbackPrompt += `Please analyze this ${isPR ? 'pull request' : 'issue'} using AI Agent for insights.\n\n`;
+        fallbackPrompt += `Original request: ${cleanedUserRequest}\n\n`;
+        fallbackPrompt += `Context: This is a ${context.eventName} event in ${context.repo.owner}/${context.repo.repo}`;
 
-      fs.writeFileSync(promptFile, fallbackPrompt);
+        fs.writeFileSync(promptFile, fallbackPrompt);
+      }
     }
 
     // Set outputs
@@ -275,6 +292,11 @@ async function run() {
  * Check if user has write or admin permissions to the repository
  */
 async function checkUserPermissions(octokit, context, issueNumber, allowedNonWriteUsers) {
+  const testMode = process.env.TEST_MODE || 'false';
+  if (testMode === 'true') {
+    return true;
+  }
+
   const actor = context.actor;
   core.debug(`Checking permissions for actor: ${actor}`);