diff --git a/.github/actions/verify-mcp-usage/action.yml b/.github/actions/verify-mcp-usage/action.yml new file mode 100644 index 0000000..6f3f6a1 --- /dev/null +++ b/.github/actions/verify-mcp-usage/action.yml @@ -0,0 +1,51 @@ +name: "Verify MCP Usage" +description: "Verifies that both CloudWatch and Application Signals MCP tools were called" + +inputs: + artifact_name: + description: "Name of the artifact containing test outputs" + required: true + output_path: + description: "Path to download test outputs" + required: false + default: "./test-outputs" + +runs: + using: "composite" + steps: + - name: Download test outputs + uses: actions/download-artifact@v4 + with: + name: ${{ inputs.artifact_name }} + path: ${{ inputs.output_path }} + + - name: Verify MCP tool usage + shell: bash + run: | + echo "Checking for MCP tool usage..." + + RAW_OUTPUT_FILE="${{ inputs.output_path }}/awsapm-raw-output.txt" + + if [[ ! -f "$RAW_OUTPUT_FILE" ]]; then + echo "ERROR: Raw output file not found" + exit 1 + fi + + # Check for both MCP tool calls + if grep -qE "Using tool: list_monitored_services.*from mcp server.*applicationsignals" "$RAW_OUTPUT_FILE"; then + echo "PASSED - list_monitored_services tool was called from AWS Application Signals MCP" + else + echo "FAILED: list_monitored_services tool call not found" + echo "Raw output content:" + cat "$RAW_OUTPUT_FILE" + exit 1 + fi + + if grep -qE "Using tool: get_active_alarms.*from mcp server.*awslabs\.cloudwatch-mcp-server" "$RAW_OUTPUT_FILE"; then + echo "PASSED - get_active_alarms tool was called from CloudWatch MCP" + else + echo "FAILED: get_active_alarms tool call not found" + echo "Raw output content:" + cat "$RAW_OUTPUT_FILE" + exit 1 + fi \ No newline at end of file diff --git a/.github/workflows/integ-test.yml b/.github/workflows/integ-test.yml new file mode 100644 index 0000000..327ae38 --- /dev/null +++ b/.github/workflows/integ-test.yml @@ -0,0 +1,56 @@ +name: Integration Test + +on: + push: + branches: + - main + workflow_dispatch: + +env: + AWS_DEFAULT_REGION: us-east-1 + +jobs: + awsapm-integ-test: + runs-on: ubuntu-latest + permissions: + contents: write + pull-requests: write + issues: write + id-token: write + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v4 + with: + role-to-assume: ${{ secrets.AWSAPM_ROLE_ARN }} + aws-region: ${{ env.AWS_DEFAULT_REGION }} + + - name: Run Application Observability for AWS Investigation + uses: ./ + with: + test_mode: "true" + custom_prompt: "Use the list_monitored_services tool to show me all services currently monitored by AWS Application Signals. Also use the get_active_alarms tool to show me all active CloudWatch alarms." + + - name: Upload test outputs + if: always() + uses: actions/upload-artifact@v4 + with: + name: integration-test-outputs + path: | + ${{ runner.temp }}/awsapm-output/ + retention-days: 7 + + verify-mcp-usage: + needs: awsapm-integ-test + runs-on: ubuntu-latest + if: always() + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Verify MCP tool usage + uses: ./.github/actions/verify-mcp-usage + with: + artifact_name: integration-test-outputs \ No newline at end of file diff --git a/.github/workflows/soak-test.yml b/.github/workflows/soak-test.yml new file mode 100644 index 0000000..627ff89 --- /dev/null +++ b/.github/workflows/soak-test.yml @@ -0,0 +1,81 @@ +name: Soak Test + +on: + schedule: + - cron: '0 * * * *' # every hour + +env: + AWS_DEFAULT_REGION: us-east-1 + +jobs: + awsapm-soak-test: + runs-on: ubuntu-latest + permissions: + contents: write + pull-requests: write + issues: write + id-token: write + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v4 + with: + role-to-assume: ${{ secrets.AWSAPM_ROLE_ARN }} + aws-region: ${{ env.AWS_DEFAULT_REGION }} + + - name: Run Application Observability for AWS Investigation - Soak Test + # TODO: Change to use published action once v1 is released + # uses: aws-actions/application-observability-for-aws@v1 + uses: ./ + with: + test_mode: "true" + custom_prompt: "Use the list_monitored_services tool to show me all services currently monitored by AWS Application Signals. Also use the get_active_alarms tool to show me all active CloudWatch alarms." + + - name: Upload test outputs + if: always() + uses: actions/upload-artifact@v4 + with: + name: soak-test-outputs + path: | + ${{ runner.temp }}/awsapm-output/ + retention-days: 7 + + verify-mcp-usage: + needs: awsapm-soak-test + runs-on: ubuntu-latest + if: always() + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Verify MCP tool usage + uses: ./.github/actions/verify-mcp-usage + with: + artifact_name: soak-test-outputs + + publish-metric: + needs: [awsapm-soak-test, verify-mcp-usage] + runs-on: ubuntu-latest + if: always() + permissions: + id-token: write + steps: + - name: Configure AWS Credentials + uses: aws-actions/configure-aws-credentials@a03048d87541d1d9fcf2ecf528a4a65ba9bd7838 #5.0.0 + with: + role-to-assume: ${{ secrets.MONITORING_ROLE_ARN }} + aws-region: ${{ env.AWS_DEFAULT_REGION }} + + - name: Publish soak test status + run: | + if [[ "${{ needs.awsapm-soak-test.result }}" == "success" && "${{ needs.verify-mcp-usage.result }}" == "success" ]]; then + value="1.0" + else + value="0.0" + fi + + aws cloudwatch put-metric-data \ + --namespace 'ADOT/GithubActions' \ + --metric-data MetricName=Success,Value=$value,Dimensions="[{Name=repository,Value=${{ github.repository }}},{Name=branch,Value=${{ github.ref_name }}},{Name=workflow,Value=soak_test}]" \ No newline at end of file diff --git a/action.yml b/action.yml index ce2605a..214ec23 100644 --- a/action.yml +++ b/action.yml @@ -32,6 +32,10 @@ inputs: description: "Enable CloudWatch MCP server for metrics, alarms, and log insights" required: false default: "true" + test_mode: + description: "Enable integration test mode (internal use only)" + required: false + default: "false" outputs: execution_file: @@ -47,6 +51,15 @@ outputs: runs: using: "composite" steps: + - name: Validate test mode usage + if: inputs.test_mode == 'true' + shell: bash + run: | + if [[ "${{ github.repository }}" != "aws-actions/application-observability-for-aws" || "${{ github.ref_name }}" != "main" ]]; then + echo "::error::test_mode can only be used from the aws-actions/application-observability-for-aws repository main branch" + exit 1 + fi + - name: Install Node.js uses: actions/setup-node@v4 with: @@ -72,6 +85,7 @@ runs: ALLOWED_NON_WRITE_USERS: ${{ inputs.allowed_non_write_users }} GITHUB_RUN_ID: ${{ github.run_id }} DEFAULT_WORKFLOW_TOKEN: ${{ github.token }} + TEST_MODE: ${{ inputs.test_mode }} - name: Install CLI Tools if: steps.init.outputs.contains_trigger == 'true' @@ -118,6 +132,7 @@ runs: GITHUB_TOKEN: ${{ steps.init.outputs.GITHUB_TOKEN }} ENABLE_CLOUDWATCH_MCP: ${{ inputs.enable_cloudwatch_mcp }} INPUT_PROMPT_FILE: ${{ runner.temp }}/awsapm-prompts/awsapm-prompt.txt + TEST_MODE: ${{ inputs.test_mode }} - name: Update comment with results if: steps.init.outputs.contains_trigger == 'true' && steps.init.outputs.awsapm_comment_id && always() diff --git a/src/execute.js b/src/execute.js index 78764ea..74add4e 100644 --- a/src/execute.js +++ b/src/execute.js @@ -35,10 +35,16 @@ async function run() { // Run Amazon Q Developer CLI investigation let investigationResult = ''; + let rawOutput = ''; try { core.info('Running Amazon Q Developer CLI investigation...'); const executor = new AmazonQCLIExecutor(null); + + executor.onRawOutput = (output) => { + rawOutput = output; + }; + investigationResult = await executor.execute(promptContent); core.info('Amazon Q Developer CLI investigation completed'); } catch (error) { @@ -63,6 +69,12 @@ Please check the workflow logs for more details and ensure proper authentication const responseFile = path.join(outputDir, `awsapm-response-${runId}.txt`); fs.writeFileSync(responseFile, cleanedResult); + // Save raw output for integration test validation (only in test mode) + if (rawOutput && process.env.TEST_MODE === 'true') { + const rawOutputFile = path.join(outputDir, 'awsapm-raw-output.txt'); + fs.writeFileSync(rawOutputFile, rawOutput); + } + // Set outputs core.setOutput('execution_file', responseFile); core.setOutput('conclusion', 'success'); diff --git a/src/executors/base-cli-executor.js b/src/executors/base-cli-executor.js index 28e676c..2277c02 100644 --- a/src/executors/base-cli-executor.js +++ b/src/executors/base-cli-executor.js @@ -217,6 +217,10 @@ class BaseCLIExecutor { const output = stdoutData || stderrData; core.debug(`[SUMMARY] Using ${stdoutData ? 'stdout' : 'stderr'} as output source`); + if (this.onRawOutput) { + this.onRawOutput(output); + } + resolve({ output, exitCode: code || 0 }); }); diff --git a/src/init.js b/src/init.js index 48b1c54..cf927df 100644 --- a/src/init.js +++ b/src/init.js @@ -20,6 +20,7 @@ async function run() { const targetBranch = process.env.TARGET_BRANCH || ''; const allowedNonWriteUsers = process.env.ALLOWED_NON_WRITE_USERS || ''; const customPrompt = process.env.CUSTOM_PROMPT || ''; + const testMode = process.env.TEST_MODE || 'false'; // Function to check for bot name trigger phrase // Note: Phrases like "@awsapm-user" will be also considered valid. @@ -74,6 +75,11 @@ async function run() { // We still want to search for existing result comments when editing commentId = null; // Explicitly set to null for clarity } + } else if (testMode === 'true') { + // In test mode, always trigger and use custom prompt + containsTrigger = true; + triggerText = customPrompt; + triggerUsername = 'integration-test'; } // Set output for action.yml to check @@ -233,25 +239,34 @@ async function run() { // Remove bot name from the user's request const cleanedUserRequest = triggerText.replace(new RegExp(botName, 'gi'), '').trim(); - // Use the dynamic prompt generation with PR context - const { createGeneralPrompt } = require('./prompt-builder'); - - try { - const finalPrompt = await createGeneralPrompt(context, repoInfo, cleanedUserRequest, githubToken, awsapmBranch); - fs.writeFileSync(promptFile, finalPrompt); - } catch (promptError) { - core.error(`Failed to generate dynamic prompt: ${promptError.message}`); - - // Fallback to basic prompt if dynamic generation fails - let fallbackPrompt = ''; - if (customPrompt) { - fallbackPrompt = customPrompt + '\n\n'; + if (testMode === 'true') { + // for integration test, use custom prompt directly + try { + fs.writeFileSync(promptFile, customPrompt); + } catch (error) { + core.error(`Failed to write custom prompt to file: ${error.message}`); + process.exit(1); } - fallbackPrompt += `Please analyze this ${isPR ? 'pull request' : 'issue'} using AI Agent for insights.\n\n`; - fallbackPrompt += `Original request: ${cleanedUserRequest}\n\n`; - fallbackPrompt += `Context: This is a ${context.eventName} event in ${context.repo.owner}/${context.repo.repo}`; + } else { + // Use the dynamic prompt generation with PR context + const { createGeneralPrompt } = require('./prompt-builder'); + + try { + const finalPrompt = await createGeneralPrompt(context, repoInfo, cleanedUserRequest, githubToken, awsapmBranch); + fs.writeFileSync(promptFile, finalPrompt); + } catch (promptError) { + core.error(`Failed to generate dynamic prompt: ${promptError.message}`); + // Fallback to basic prompt if dynamic generation fails + let fallbackPrompt = ''; + if (customPrompt) { + fallbackPrompt = customPrompt + '\n\n'; + } + fallbackPrompt += `Please analyze this ${isPR ? 'pull request' : 'issue'} using AI Agent for insights.\n\n`; + fallbackPrompt += `Original request: ${cleanedUserRequest}\n\n`; + fallbackPrompt += `Context: This is a ${context.eventName} event in ${context.repo.owner}/${context.repo.repo}`; - fs.writeFileSync(promptFile, fallbackPrompt); + fs.writeFileSync(promptFile, fallbackPrompt); + } } // Set outputs @@ -275,6 +290,11 @@ async function run() { * Check if user has write or admin permissions to the repository */ async function checkUserPermissions(octokit, context, issueNumber, allowedNonWriteUsers) { + const testMode = process.env.TEST_MODE || 'false'; + if (testMode === 'true') { + return true; + } + const actor = context.actor; core.debug(`Checking permissions for actor: ${actor}`);