Skip to content
Merged
Show file tree
Hide file tree
Changes from 11 commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
352 changes: 352 additions & 0 deletions .github/workflows/compatibility-matrix-test.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,352 @@
name: Compatibility Matrix Test

on:
# Run every morning at 9 AM UTC on the main branch
schedule:
- cron: '0 9 * * *'

# Allow manual trigger from any branch
workflow_dispatch:
inputs:
combination:
description: 'Test specific combination (leave empty for all)'
required: false
type: string
clean_build:
description: 'Clean build between tests'
required: false
type: boolean
default: false

jobs:
# Read the compatibility matrix and set up the test matrix
setup:
name: Setup Test Matrix
runs-on: ubuntu-latest
outputs:
matrix: ${{ steps.set-matrix.outputs.matrix }}
steps:
- name: Checkout code
uses: actions/checkout@v4

- name: Set up matrix from JSON
id: set-matrix
run: |
# Read the compatibility matrix JSON and filter out disabled combinations
MATRIX=$(cat .github/workflows/compatibility-matrix.json | jq -c '[.combinations[] | select(.enabled // true == true)]')
echo "matrix=$MATRIX" >> $GITHUB_OUTPUT

# Print the matrix for debugging
echo "Enabled test combinations:"
echo "$MATRIX" | jq -r '.[] | " - \(.name): AGP \(.agp), Gradle \(.gradle), Java \(.java), Kotlin \(.kotlin)"'

# Print disabled combinations for reference
DISABLED=$(cat .github/workflows/compatibility-matrix.json | jq -c '[.combinations[] | select(.enabled == false)]')
DISABLED_COUNT=$(echo "$DISABLED" | jq 'length')
if [ "$DISABLED_COUNT" -gt 0 ]; then
echo ""
echo "Disabled combinations (future versions):"
echo "$DISABLED" | jq -r '.[] | " - \(.name): \(.note // "Future version")"'
fi

# Build the agent once and cache it for all test jobs
build-agent:
name: Build and Publish Agent
runs-on: ubuntu-latest
steps:
- name: Checkout code
uses: actions/checkout@v4

- name: Set up JDK 11
uses: actions/setup-java@v4
with:
java-version: 11
distribution: 'temurin'

- name: Setup Gradle
uses: gradle/gradle-build-action@v3
with:
cache-read-only: false

- name: Grant execute permission for gradlew
run: chmod +x gradlew

- name: Publish Agent to Local Maven
run: |
./gradlew publishToMavenLocal -x test -x functionalTests -x integrationTests
env:
JAVA_HOME: ${{ env.JAVA_HOME }}

- name: Upload Maven Local Repository
uses: actions/upload-artifact@v4
with:
name: maven-local-repository
path: ~/.m2/repository/com/newrelic/agent/android/
retention-days: 1

# Run tests for each combination in the matrix
test:
name: Test ${{ matrix.combination.name }}
runs-on: ubuntu-latest
needs: [setup, build-agent]
timeout-minutes: 30
strategy:
fail-fast: false # Continue testing other combinations even if one fails
max-parallel: 1 # Run tests sequentially to avoid conflicts with shared files
matrix:
combination: ${{ fromJson(needs.setup.outputs.matrix) }}

steps:
- name: Checkout code
uses: actions/checkout@v4

- name: Set up JDK ${{ matrix.combination.java }}
uses: actions/setup-java@v4
with:
java-version: ${{ matrix.combination.java }}
distribution: 'temurin'

- name: Setup Gradle
uses: gradle/gradle-build-action@v3
with:
gradle-version: ${{ matrix.combination.gradle }}
cache-read-only: false

- name: Cache Android SDK
uses: actions/cache@v4
with:
path: |
~/.android/sdk
~/.android/avd
key: android-sdk-${{ runner.os }}-${{ hashFiles('**/build.gradle*', '**/gradle-wrapper.properties') }}
restore-keys: |
android-sdk-${{ runner.os }}-

- name: Download Pre-built Agent from Maven
uses: actions/download-artifact@v4
with:
name: maven-local-repository
path: ~/.m2/repository/com/newrelic/agent/android/

- name: Grant execute permission for gradlew and test script
run: |
chmod +x gradlew
chmod +x scripts/test-compatibility-matrix.sh

- name: Build Test App with ${{ matrix.combination.name }}
id: build
run: |
echo "::group::Building with configuration"
echo "AGP Version: ${{ matrix.combination.agp }}"
echo "Gradle Version: ${{ matrix.combination.gradle }}"
echo "Java Version: ${{ matrix.combination.java }}"
echo "Kotlin Version: ${{ matrix.combination.kotlin }}"
echo "::endgroup::"

# Use the test script to properly handle version switching
./scripts/test-compatibility-matrix.sh --combination "${{ matrix.combination.name }}" --verbose
env:
JAVA_HOME: ${{ env.JAVA_HOME }}

- name: Check APK was generated
if: success()
working-directory: samples/agent-test-app
run: |
APK_PATH="build/outputs/apk/release/agent-test-app-release-unsigned.apk"
if [ -f "$APK_PATH" ]; then
APK_SIZE=$(du -h "$APK_PATH" | cut -f1)
echo "✓ APK generated successfully: $APK_SIZE"
echo "apk_generated=true" >> $GITHUB_OUTPUT
echo "apk_size=$APK_SIZE" >> $GITHUB_OUTPUT
else
echo "✗ APK not found at $APK_PATH"
echo "apk_generated=false" >> $GITHUB_OUTPUT
exit 1
fi

- name: Upload APK as artifact
if: success()
uses: actions/upload-artifact@v4
with:
name: apk-${{ matrix.combination.name }}-agp${{ matrix.combination.agp }}
path: samples/agent-test-app/build/outputs/apk/release/*.apk
retention-days: 7
if-no-files-found: error

- name: Upload build logs on failure
if: failure()
uses: actions/upload-artifact@v4
with:
name: build-logs-${{ matrix.combination.name }}
path: |
samples/agent-test-app/build/reports/
samples/agent-test-app/build/outputs/logs/
retention-days: 7
if-no-files-found: ignore

# Generate a summary report
report:
name: Generate Test Report
runs-on: ubuntu-latest
needs: [setup, test]
if: always() # Run even if some tests failed
steps:
- name: Checkout code
uses: actions/checkout@v4

- name: Generate Summary
run: |
cat >> $GITHUB_STEP_SUMMARY << 'EOF'
# 📊 Compatibility Matrix Test Results

## Test Configuration
- **Triggered by**: ${{ github.event_name }}
- **Run ID**: ${{ github.run_id }}
- **Date**: $(date -u +"%Y-%m-%d %H:%M:%S UTC")

## Test Matrix

| Configuration | AGP | Gradle | Java | Kotlin | Status |
|---------------|-----|--------|------|--------|--------|
EOF

# Parse matrix and add results
MATRIX='${{ needs.setup.outputs.matrix }}'
echo "$MATRIX" | jq -r '.[] |
"| \(.name) | \(.agp) | \(.gradle) | \(.java) | \(.kotlin) | ⏳ Running |"
' >> $GITHUB_STEP_SUMMARY

cat >> $GITHUB_STEP_SUMMARY << 'EOF'

## 📝 Notes
- ✅ Green: Build succeeded and APK generated
- ❌ Red: Build failed
- ⏭️ Skipped: Dependency failure

EOF

# Comment on PR with results (only runs if workflow is triggered by a pull_request event)
# Note: This job is disabled by default since the workflow runs on schedule/manual trigger
# Re-enable by adding 'pull_request:' trigger if needed
comment:
name: Post PR Comment
runs-on: ubuntu-latest
needs: [setup, test, build-agent]
if: github.event_name == 'pull_request' && always()
permissions:
pull-requests: write
steps:
- name: Generate Comment Body
uses: actions/github-script@v7
id: generate-comment
with:
script: |
const matrix = ${{ needs.setup.outputs.matrix }};

// Fetch all workflow jobs to get individual test results
const { data: jobs } = await github.rest.actions.listJobsForWorkflowRun({
owner: context.repo.owner,
repo: context.repo.repo,
run_id: context.runId,
});

// Map test job results by combination name
const testResults = {};
jobs.jobs.forEach(job => {
// Match jobs like "Test AGP 7.4 - Stable"
const match = job.name.match(/^Test (.+)$/);
if (match) {
const combinationName = match[1];
testResults[combinationName] = {
conclusion: job.conclusion,
status: job.status,
html_url: job.html_url
};
}
});

// Generate markdown table
let comment = `## 🤖 Compatibility Matrix Test Results\n\n`;
comment += `The agent was tested against multiple AGP/Gradle/Java/Kotlin combinations:\n\n`;
comment += `| Configuration | AGP | Gradle | Java | Kotlin | Result |\n`;
comment += `|---------------|-----|--------|------|--------|--------|\n`;

matrix.forEach(combo => {
const result = testResults[combo.name];
let statusIcon = '⏳';
let statusText = 'Running';

if (result) {
if (result.conclusion === 'success') {
statusIcon = '✅';
statusText = 'Passed';
} else if (result.conclusion === 'failure') {
statusIcon = '❌';
statusText = 'Failed';
} else if (result.conclusion === 'cancelled') {
statusIcon = '🚫';
statusText = 'Cancelled';
} else if (result.conclusion === 'skipped') {
statusIcon = '⏭️';
statusText = 'Skipped';
}
}

comment += `| ${combo.name} | ${combo.agp} | ${combo.gradle} | ${combo.java} | ${combo.kotlin} | ${statusIcon} ${statusText} |\n`;
});

// Add summary
const totalTests = matrix.length;
const passedTests = Object.values(testResults).filter(r => r.conclusion === 'success').length;
const failedTests = Object.values(testResults).filter(r => r.conclusion === 'failure').length;

comment += `\n### Summary\n`;
comment += `- **Total**: ${totalTests} combinations\n`;
comment += `- **Passed**: ✅ ${passedTests}\n`;
comment += `- **Failed**: ❌ ${failedTests}\n`;

if (failedTests > 0) {
comment += `\n⚠️ **Some tests failed.** Please check the workflow logs for details.\n`;
} else if (passedTests === totalTests) {
comment += `\n🎉 **All tests passed!**\n`;
}

comment += `\n**View full results**: [Workflow Run](${context.payload.repository.html_url}/actions/runs/${context.runId})\n`;

return comment;

- name: Comment on PR
uses: actions/github-script@v7
with:
script: |
const comment = ${{ steps.generate-comment.outputs.result }};

// Find existing comment to update
const { data: comments } = await github.rest.issues.listComments({
issue_number: context.issue.number,
owner: context.repo.owner,
repo: context.repo.repo,
});

const botComment = comments.find(c =>
c.user.type === 'Bot' &&
c.body.includes('🤖 Compatibility Matrix Test Results')
);

if (botComment) {
// Update existing comment
await github.rest.issues.updateComment({
owner: context.repo.owner,
repo: context.repo.repo,
comment_id: botComment.id,
body: comment
});
} else {
// Create new comment
await github.rest.issues.createComment({
issue_number: context.issue.number,
owner: context.repo.owner,
repo: context.repo.repo,
body: comment
});
}
Loading
Loading