test: add Q CLI and MCP integration testing

.github/workflows/integ-test.yml

@@ -0,0 +1,77 @@
+name: Integration Test
+
+on:
+  schedule:
+    - cron: '0 */6 * * *'
+  push:
+    branches:
+      - integ-test
+
+jobs:
+  awsapm-investigation:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: write
+      pull-requests: write
+      issues: write
+      id-token: write
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 1
+
+      - name: Run Application observability for AWS Investigation
+        id: awsapm
+        uses: ./
+        with:
+          aws_access_key_id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          aws_secret_access_key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          aws_region: ${{ vars.AWS_REGION || 'us-east-1' }}
+          bot_name: "@awsapm"
+          tracing_mode: "true"
+          test_mode: "true"  # Enable integration test mode
+          custom_prompt: "Use the list_monitored_services tool to show me all services currently monitored by AWS Application Signals. I need to see the complete list of monitored services."
+
+      - name: Upload test outputs
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: integration-test-outputs
+          path: |
+            ${{ runner.temp }}/awsapm-output/
+          retention-days: 7
+
+  verify-mcp-usage:
+    needs: awsapm-investigation
+    runs-on: ubuntu-latest
+    if: always()
+    steps:
+      - name: Download test outputs
+        uses: actions/download-artifact@v4
+        with:
+          name: integration-test-outputs
+          path: ./test-outputs
+
+      - name: Verify MCP tool usage
+        shell: bash
+        run: |
+          echo "Checking for AWS Application Signals MCP usage..."
+
+          RAW_OUTPUT_FILE="./test-outputs/awsapm-raw-output.txt"
+
+          if [[ ! -f "$RAW_OUTPUT_FILE" ]]; then
+            echo "ERROR: Raw output file not found"
+            exit 1
+          fi
+
+          # Check for the specific tool call on one line (handling ANSI color codes)
+          if grep -qE "Using tool: list_monitored_services.*from mcp server.*awslabs\.cloudwatch-appsignals-mcp" "$RAW_OUTPUT_FILE"; then
+            echo "PASSED - list_monitored_services tool was called from AWS Application Signals MCP"
+          else
+            echo "FAILED: Expected tool call not found"
+            echo "Looking for: Using tool: list_monitored_services...from mcp server...awslabs.cloudwatch-appsignals-mcp"
+            echo "Raw output content:"
+            cat "$RAW_OUTPUT_FILE"
+            exit 1
+          fi

action.yml

@@ -45,6 +45,10 @@ inputs:
     description: "Enable tracing to show AI agent reasoning steps and MCP tool calls in workflow summary"
     required: false
     default: "true"
+  test_mode:
+    description: "Enable integration test mode (internal use only)"
+    required: false
+    default: "false"
 
 outputs:
   execution_file:
@@ -86,6 +90,7 @@ runs:
         GITHUB_RUN_ID: ${{ github.run_id }}
         DEFAULT_WORKFLOW_TOKEN: ${{ github.token }}
         TRACING_MODE: ${{ inputs.tracing_mode }}
+        TEST_MODE: ${{ inputs.test_mode }}
 
     - name: Install CLI Tools
       if: steps.init.outputs.contains_trigger == 'true'

src/execute.js

@@ -34,10 +34,16 @@ async function run() {
 
     // Run Amazon Q Developer CLI investigation
     let investigationResult = '';
+    let rawOutput = '';
 
     try {
       core.info('Running Amazon Q Developer CLI investigation...');
       const executor = new AmazonQCLIExecutor(null);
+
+      executor.onRawOutput = (output) => {
+        rawOutput = output;
+      };
+
       investigationResult = await executor.execute(promptContent);
       core.info('Amazon Q Developer CLI investigation completed');
     } catch (error) {
@@ -58,6 +64,12 @@ Please check the workflow logs for more details and ensure proper authentication
     const responseFile = path.join(outputDir, `awsapm-response-${runId}.txt`);
     fs.writeFileSync(responseFile, investigationResult);
 
+    // Save raw output for integration test validation
+    if (rawOutput) {
+      const rawOutputFile = path.join(outputDir, 'awsapm-raw-output.txt');
+      fs.writeFileSync(rawOutputFile, rawOutput);
+    }
+
     // Set outputs
     core.setOutput('execution_file', responseFile);
     core.setOutput('conclusion', 'success');

src/executors/base-cli-executor.js

@@ -217,6 +217,10 @@ class BaseCLIExecutor {
         const output = stdoutData || stderrData;
         core.debug(`[SUMMARY] Using ${stdoutData ? 'stdout' : 'stderr'} as output source`);
 
+        if (this.onRawOutput) {
+          this.onRawOutput(output);
+        }
+
         resolve({ output, exitCode: code || 0 });
       });
 

src/init.js

@@ -21,6 +21,7 @@ async function run() {
     const allowedNonWriteUsers = process.env.ALLOWED_NON_WRITE_USERS || '';
     const customPrompt = process.env.CUSTOM_PROMPT || '';
     const tracingMode = process.env.TRACING_MODE || 'false';
+    const testMode = process.env.TEST_MODE || 'false';
 
     // Function to check for bot name trigger phrase
     // Note: Phrases like "@awsapm-user" will be also considered valid.
@@ -72,6 +73,11 @@ async function run() {
         isEditEvent = payload.action === 'edited';
         triggerUsername = issue.user?.login || 'unknown';
       }
+    } else if (testMode === 'true') {
+      // In test mode, always trigger and use custom prompt
+      containsTrigger = true;
+      triggerText = customPrompt;
+      triggerUsername = 'integration-test';
     }
 
     // Set output for action.yml to check
@@ -223,25 +229,35 @@ async function run() {
     // Remove bot name from the user's request
     const cleanedUserRequest = triggerText.replace(new RegExp(botName, 'gi'), '').trim();
 
-    // Use the dynamic prompt generation with PR context
-    const { createGeneralPrompt } = require('./prompt-builder');
-
-    try {
-      const finalPrompt = await createGeneralPrompt(context, repoInfo, cleanedUserRequest, githubToken);
-      fs.writeFileSync(promptFile, finalPrompt);
-    } catch (promptError) {
-      core.error(`Failed to generate dynamic prompt: ${promptError.message}`);
-
-      // Fallback to basic prompt if dynamic generation fails
-      let fallbackPrompt = '';
-      if (customPrompt) {
-        fallbackPrompt = customPrompt + '\n\n';
+    if (testMode === 'true') {
+      // for integration test, use custom prompt directly
+      try {
+        fs.writeFileSync(promptFile, customPrompt);
+      } catch (error) {
+        core.error(`Failed to write custom prompt to file: ${error.message}`);
+        process.exit(1);
       }
-      fallbackPrompt += `Please analyze this ${isPR ? 'pull request' : 'issue'} using AI Agent for insights.\n\n`;
-      fallbackPrompt += `Original request: ${cleanedUserRequest}\n\n`;
-      fallbackPrompt += `Context: This is a ${context.eventName} event in ${context.repo.owner}/${context.repo.repo}`;
+    } else {
+      // Use the dynamic prompt generation with PR context
+      const { createGeneralPrompt } = require('./prompt-builder');
+
+      try {
+        const finalPrompt = await createGeneralPrompt(context, repoInfo, cleanedUserRequest, githubToken);
+        fs.writeFileSync(promptFile, finalPrompt);
+      } catch (promptError) {
+        core.error(`Failed to generate dynamic prompt: ${promptError.message}`);
+
+        // Fallback to basic prompt if dynamic generation fails
+        let fallbackPrompt = '';
+        if (customPrompt) {
+          fallbackPrompt = customPrompt + '\n\n';
+        }
+        fallbackPrompt += `Please analyze this ${isPR ? 'pull request' : 'issue'} using AI Agent for insights.\n\n`;
+        fallbackPrompt += `Original request: ${cleanedUserRequest}\n\n`;
+        fallbackPrompt += `Context: This is a ${context.eventName} event in ${context.repo.owner}/${context.repo.repo}`;
 
-      fs.writeFileSync(promptFile, fallbackPrompt);
+        fs.writeFileSync(promptFile, fallbackPrompt);
+      }
     }
 
     // Set outputs
@@ -266,6 +282,11 @@ async function run() {
  * Check if user has write or admin permissions to the repository
  */
 async function checkUserPermissions(octokit, context, issueNumber, allowedNonWriteUsers) {
+  const testMode = process.env.TEST_MODE || 'false';
+  if (testMode === 'true') {
+    return true;
+  }
+
   const actor = context.actor;
   core.debug(`Checking permissions for actor: ${actor}`);