compatibility-matrix-test.yml

name: Compatibility Matrix Test

on:
  # Run every morning at 9 AM UTC on the main branch
  schedule:
    - cron: '0 9 * * *'

  # Allow manual trigger from any branch
  workflow_dispatch:
    inputs:
      combination:
        description: 'Test specific combination (leave empty for all)'
        required: false
        type: string
      clean_build:
        description: 'Clean build between tests'
        required: false
        type: boolean
        default: false

jobs:
  # Read the compatibility matrix and set up the test matrix
  setup:
    name: Setup Test Matrix
    runs-on: ubuntu-latest
    outputs:
      matrix: ${{ steps.set-matrix.outputs.matrix }}
    steps:
      - name: Checkout code
        uses: actions/checkout@v4

      - name: Set up matrix from JSON
        id: set-matrix
        run: |
          # Read the compatibility matrix JSON and filter out disabled combinations
          MATRIX=$(cat .github/workflows/compatibility-matrix.json | jq -c '[.combinations[] | select(.enabled // true == true)]')
          echo "matrix=$MATRIX" >> $GITHUB_OUTPUT

          # Print the matrix for debugging
          echo "Enabled test combinations:"
          echo "$MATRIX" | jq -r '.[] | "  - \(.name): AGP \(.agp), Gradle \(.gradle), Java \(.java), Kotlin \(.kotlin)"'

          # Print disabled combinations for reference
          DISABLED=$(cat .github/workflows/compatibility-matrix.json | jq -c '[.combinations[] | select(.enabled == false)]')
          DISABLED_COUNT=$(echo "$DISABLED" | jq 'length')
          if [ "$DISABLED_COUNT" -gt 0 ]; then
            echo ""
            echo "Disabled combinations (future versions):"
            echo "$DISABLED" | jq -r '.[] | "  - \(.name): \(.note // "Future version")"'
          fi

  # Build the agent once and cache it for all test jobs
  build-agent:
    name: Build and Publish Agent
    runs-on: ubuntu-latest
    steps:
      - name: Checkout code
        uses: actions/checkout@v4

      - name: Set up JDK 11
        uses: actions/setup-java@v4
        with:
          java-version: 11
          distribution: 'temurin'

      - name: Setup Gradle
        uses: gradle/gradle-build-action@v3
        with:
          cache-read-only: false

      - name: Grant execute permission for gradlew
        run: chmod +x gradlew

      - name: Publish Agent to Local Maven
        run: |
          ./gradlew publishToMavenLocal -x test -x functionalTests -x integrationTests
        env:
          JAVA_HOME: ${{ env.JAVA_HOME }}

      - name: Upload Maven Local Repository
        uses: actions/upload-artifact@v4
        with:
          name: maven-local-repository
          path: ~/.m2/repository/com/newrelic/agent/android/
          retention-days: 1

  # Run tests for each combination in the matrix
  test:
    name: Test ${{ matrix.combination.name }}
    runs-on: ubuntu-latest
    needs: [setup, build-agent]
    timeout-minutes: 30
    strategy:
      fail-fast: false  # Continue testing other combinations even if one fails
      max-parallel: 1   # Run tests sequentially to avoid conflicts with shared files
      matrix:
        combination: ${{ fromJson(needs.setup.outputs.matrix) }}

    steps:
      - name: Checkout code
        uses: actions/checkout@v4

      - name: Set up JDK ${{ matrix.combination.java }}
        uses: actions/setup-java@v4
        with:
          java-version: ${{ matrix.combination.java }}
          distribution: 'temurin'

      - name: Setup Gradle
        uses: gradle/gradle-build-action@v3
        with:
          gradle-version: ${{ matrix.combination.gradle }}
          cache-read-only: false

      - name: Cache Android SDK
        uses: actions/cache@v4
        with:
          path: |
            ~/.android/sdk
            ~/.android/avd
          key: android-sdk-${{ runner.os }}-${{ hashFiles('**/build.gradle*', '**/gradle-wrapper.properties') }}
          restore-keys: |
            android-sdk-${{ runner.os }}-

      - name: Download Pre-built Agent from Maven
        uses: actions/download-artifact@v4
        with:
          name: maven-local-repository
          path: ~/.m2/repository/com/newrelic/agent/android/

      - name: Grant execute permission for gradlew and test script
        run: |
          chmod +x gradlew
          chmod +x scripts/test-compatibility-matrix.sh

      - name: Build Test App with ${{ matrix.combination.name }}
        id: build
        run: |
          echo "::group::Building with configuration"
          echo "AGP Version: ${{ matrix.combination.agp }}"
          echo "Gradle Version: ${{ matrix.combination.gradle }}"
          echo "Java Version: ${{ matrix.combination.java }}"
          echo "Kotlin Version: ${{ matrix.combination.kotlin }}"
          echo "::endgroup::"

          # Use the test script to properly handle version switching
          ./scripts/test-compatibility-matrix.sh --combination "${{ matrix.combination.name }}" --verbose
        env:
          JAVA_HOME: ${{ env.JAVA_HOME }}

      - name: Check APK was generated
        if: success()
        working-directory: samples/agent-test-app
        run: |
          APK_PATH="build/outputs/apk/release/agent-test-app-release-unsigned.apk"
          if [ -f "$APK_PATH" ]; then
            APK_SIZE=$(du -h "$APK_PATH" | cut -f1)
            echo "✓ APK generated successfully: $APK_SIZE"
            echo "apk_generated=true" >> $GITHUB_OUTPUT
            echo "apk_size=$APK_SIZE" >> $GITHUB_OUTPUT
          else
            echo "✗ APK not found at $APK_PATH"
            echo "apk_generated=false" >> $GITHUB_OUTPUT
            exit 1
          fi

      - name: Upload APK as artifact
        if: success()
        uses: actions/upload-artifact@v4
        with:
          name: apk-${{ matrix.combination.name }}-agp${{ matrix.combination.agp }}
          path: samples/agent-test-app/build/outputs/apk/release/*.apk
          retention-days: 7
          if-no-files-found: error

      - name: Upload build logs on failure
        if: failure()
        uses: actions/upload-artifact@v4
        with:
          name: build-logs-${{ matrix.combination.name }}
          path: |
            samples/agent-test-app/build/reports/
            samples/agent-test-app/build/outputs/logs/
          retention-days: 7
          if-no-files-found: ignore

  # Generate a summary report
  report:
    name: Generate Test Report
    runs-on: ubuntu-latest
    needs: [setup, test]
    if: always()  # Run even if some tests failed
    steps:
      - name: Checkout code
        uses: actions/checkout@v4

      - name: Generate Summary
        run: |
          cat >> $GITHUB_STEP_SUMMARY << 'EOF'
          # 📊 Compatibility Matrix Test Results

          ## Test Configuration
          - **Triggered by**: ${{ github.event_name }}
          - **Run ID**: ${{ github.run_id }}
          - **Date**: $(date -u +"%Y-%m-%d %H:%M:%S UTC")

          ## Test Matrix

          | Configuration | AGP | Gradle | Java | Kotlin | Status |
          |---------------|-----|--------|------|--------|--------|
          EOF

          # Parse matrix and add results
          MATRIX='${{ needs.setup.outputs.matrix }}'
          echo "$MATRIX" | jq -r '.[] |
            "| \(.name) | \(.agp) | \(.gradle) | \(.java) | \(.kotlin) | ⏳ Running |"
          ' >> $GITHUB_STEP_SUMMARY

          cat >> $GITHUB_STEP_SUMMARY << 'EOF'

          ## 📝 Notes
          - ✅ Green: Build succeeded and APK generated
          - ❌ Red: Build failed
          - ⏭️ Skipped: Dependency failure

          EOF

  # Comment on PR with results (only runs if workflow is triggered by a pull_request event)
  # Note: This job is disabled by default since the workflow runs on schedule/manual trigger
  # Re-enable by adding 'pull_request:' trigger if needed
  comment:
    name: Post PR Comment
    runs-on: ubuntu-latest
    needs: [setup, test, build-agent]
    if: github.event_name == 'pull_request' && always()
    permissions:
      pull-requests: write
    steps:
      - name: Generate Comment Body
        uses: actions/github-script@v7
        id: generate-comment
        with:
          script: |
            const matrix = ${{ needs.setup.outputs.matrix }};

            // Fetch all workflow jobs to get individual test results
            const { data: jobs } = await github.rest.actions.listJobsForWorkflowRun({
              owner: context.repo.owner,
              repo: context.repo.repo,
              run_id: context.runId,
            });

            // Map test job results by combination name
            const testResults = {};
            jobs.jobs.forEach(job => {
              // Match jobs like "Test AGP 7.4 - Stable"
              const match = job.name.match(/^Test (.+)$/);
              if (match) {
                const combinationName = match[1];
                testResults[combinationName] = {
                  conclusion: job.conclusion,
                  status: job.status,
                  html_url: job.html_url
                };
              }
            });

            // Generate markdown table
            let comment = `## 🤖 Compatibility Matrix Test Results\n\n`;
            comment += `The agent was tested against multiple AGP/Gradle/Java/Kotlin combinations:\n\n`;
            comment += `| Configuration | AGP | Gradle | Java | Kotlin | Result |\n`;
            comment += `|---------------|-----|--------|------|--------|--------|\n`;

            matrix.forEach(combo => {
              const result = testResults[combo.name];
              let statusIcon = '⏳';
              let statusText = 'Running';

              if (result) {
                if (result.conclusion === 'success') {
                  statusIcon = '✅';
                  statusText = 'Passed';
                } else if (result.conclusion === 'failure') {
                  statusIcon = '❌';
                  statusText = 'Failed';
                } else if (result.conclusion === 'cancelled') {
                  statusIcon = '🚫';
                  statusText = 'Cancelled';
                } else if (result.conclusion === 'skipped') {
                  statusIcon = '⏭️';
                  statusText = 'Skipped';
                }
              }

              comment += `| ${combo.name} | ${combo.agp} | ${combo.gradle} | ${combo.java} | ${combo.kotlin} | ${statusIcon} ${statusText} |\n`;
            });

            // Add summary
            const totalTests = matrix.length;
            const passedTests = Object.values(testResults).filter(r => r.conclusion === 'success').length;
            const failedTests = Object.values(testResults).filter(r => r.conclusion === 'failure').length;

            comment += `\n### Summary\n`;
            comment += `- **Total**: ${totalTests} combinations\n`;
            comment += `- **Passed**: ✅ ${passedTests}\n`;
            comment += `- **Failed**: ❌ ${failedTests}\n`;

            if (failedTests > 0) {
              comment += `\n⚠️ **Some tests failed.** Please check the workflow logs for details.\n`;
            } else if (passedTests === totalTests) {
              comment += `\n🎉 **All tests passed!**\n`;
            }

            comment += `\n**View full results**: [Workflow Run](${context.payload.repository.html_url}/actions/runs/${context.runId})\n`;

            return comment;

      - name: Comment on PR
        uses: actions/github-script@v7
        with:
          script: |
            const comment = ${{ steps.generate-comment.outputs.result }};

            // Find existing comment to update
            const { data: comments } = await github.rest.issues.listComments({
              issue_number: context.issue.number,
              owner: context.repo.owner,
              repo: context.repo.repo,
            });

            const botComment = comments.find(c =>
              c.user.type === 'Bot' &&
              c.body.includes('🤖 Compatibility Matrix Test Results')
            );

            if (botComment) {
              // Update existing comment
              await github.rest.issues.updateComment({
                owner: context.repo.owner,
                repo: context.repo.repo,
                comment_id: botComment.id,
                body: comment
              });
            } else {
              // Create new comment
              await github.rest.issues.createComment({
                issue_number: context.issue.number,
                owner: context.repo.owner,
                repo: context.repo.repo,
                body: comment
              });
            }