Mobile Tests #63
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| --- | |
| name: Mobile Tests | |
| "on": | |
| schedule: | |
| # Run every day at 4 AM UTC (off-peak for AWS Device Farm) | |
| - cron: '0 4 * * *' | |
| workflow_dispatch: | |
| inputs: | |
| device_pools: | |
| description: | | |
| Device pools (comma-separated for parallel execution). Available options: | |
| 00-android-base, | |
| android-google-pixel-6a, | |
| android-google-pixel-7, | |
| android-google-pixel-8, | |
| android-google-pixel-9-pro, | |
| android-samsung-a14, | |
| android-samsung-a34, | |
| android-samsung-s23, | |
| android-samsung-s24-ultra | |
| required: false | |
| default: 'android-samsung-a14,android-samsung-s24-ultra' | |
| type: string | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.ref }} | |
| cancel-in-progress: true | |
| jobs: | |
| prepare-matrix: | |
| runs-on: ubuntu-latest | |
| outputs: | |
| matrix: ${{ steps.set-matrix.outputs.matrix }} | |
| steps: | |
| - id: set-matrix | |
| shell: python | |
| run: | | |
| import json | |
| import os | |
| pools = "${{ github.event.inputs.device_pools || 'android-samsung-a14,android-samsung-s24-ultra' }}" | |
| device_list = [p.strip() for p in pools.split(',')] | |
| matrix = json.dumps({"device-pool": device_list}) | |
| print(f"Matrix JSON: {matrix}") | |
| with open(os.environ['GITHUB_OUTPUT'], 'a') as f: | |
| f.write(f"matrix={matrix}\n") | |
| build-apk: | |
| runs-on: c7a-4xl-android | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Checkout mobilebench | |
| uses: actions/checkout@v4 | |
| with: | |
| repository: IrreducibleOSS/mobilebench | |
| token: ${{ secrets.MOBILEBENCH_ACCESS_TOKEN }} | |
| path: mobilebench | |
| - name: Check versions and environment | |
| run: | | |
| echo "=== Build Environment ===" | |
| echo "Python: $(python --version)" | |
| echo "Java: $(java -version 2>&1 | head -1)" | |
| echo "Rust: $(rustc --version)" | |
| echo "Cargo: $(cargo --version)" | |
| echo "Gradle: $(gradle --version | grep Gradle)" | |
| echo "Android SDK: $ANDROID_HOME" | |
| echo "Android NDK: $ANDROID_NDK_HOME" | |
| echo "" | |
| echo "=== NDK Toolchain Check ===" | |
| # Check for x86_64 NDK toolchain (not aarch64) | |
| ls -la $ANDROID_NDK_HOME/toolchains/llvm/prebuilt/linux-x86_64/bin/aarch64-linux-android*-clang | head -5 || echo "NDK toolchain not found at expected path" | |
| echo "" | |
| echo "=== Rust Android Targets ===" | |
| rustup target list --installed | grep android || echo "No Android targets installed" | |
| - name: Setup environment | |
| run: | | |
| # Python dependencies | |
| python -m pip install --upgrade pip | |
| pip install -r mobilebench/scripts/requirements.txt | |
| # Add mobilebench to PATH | |
| echo "${{ github.workspace }}/mobilebench" >> $GITHUB_PATH | |
| - name: Verify mobilebench | |
| run: | | |
| mobilebench --help | |
| - name: Cache Rust dependencies | |
| uses: Swatinem/rust-cache@v2 | |
| with: | |
| key: mobile-test-android | |
| cache-on-failure: true | |
| - name: Build tests | |
| run: | | |
| mobilebench build rust --build-type test --target aarch64-linux-android | |
| ls -la mobile/android_binaries/*/ | |
| - name: Remove subprocess-dependent tests | |
| run: | | |
| # Remove ceck testsuite tests that require subprocess execution | |
| rm -f mobile/android_binaries/*/test.testsuite-* | |
| ls -la mobile/android_binaries/*/ | |
| - name: Cache Gradle dependencies | |
| uses: actions/cache@v4 | |
| with: | |
| path: | | |
| ~/.gradle/caches | |
| ~/.gradle/wrapper | |
| mobilebench/android/.gradle | |
| mobilebench/android/build | |
| key: gradle-${{ runner.os }}-${{ hashFiles('**/*.gradle*', '**/gradle-wrapper.properties') }} | |
| restore-keys: gradle-${{ runner.os }}- | |
| - name: Build APK | |
| run: | | |
| mobilebench build apk | |
| ls -la mobile/android_apk/ | |
| - name: Upload APK artifact | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: mobile-test-apk | |
| path: mobile/android_apk/ | |
| retention-days: 1 | |
| run-tests: | |
| needs: [prepare-matrix, build-apk] | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 30 | |
| permissions: | |
| id-token: write # For AWS OIDC authentication | |
| contents: read # For checking out code | |
| strategy: | |
| fail-fast: false | |
| matrix: ${{ fromJSON(needs.prepare-matrix.outputs.matrix) }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Checkout mobilebench | |
| uses: actions/checkout@v4 | |
| with: | |
| repository: IrreducibleOSS/mobilebench | |
| token: ${{ secrets.MOBILEBENCH_ACCESS_TOKEN }} | |
| path: mobilebench | |
| - name: Download APK artifact | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: mobile-test-apk | |
| path: mobile/android_apk/ | |
| - name: Setup Python | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: '3.13' | |
| - name: Setup environment | |
| run: | | |
| # Python dependencies | |
| python -m pip install --upgrade pip | |
| pip install -r mobilebench/scripts/requirements.txt | |
| # Add mobilebench to PATH | |
| echo "${{ github.workspace }}/mobilebench" >> $GITHUB_PATH | |
| - name: Configure AWS | |
| uses: aws-actions/configure-aws-credentials@v4 | |
| with: | |
| role-to-assume: ${{ secrets.AWS_UPLOAD_ROLE }} | |
| role-session-name: github-actions-mobile-test-${{ matrix.device-pool }} | |
| aws-region: us-west-2 | |
| - name: Verify mobilebench | |
| run: | | |
| mobilebench --help | |
| ls -la mobile/android_apk/ | |
| - name: Run on Device Farm - ${{ matrix.device-pool }} | |
| run: | | |
| mobilebench aws run --device-pool "${{ matrix.device-pool }}" | |
| RESULT_DIR=$(find ./mobile/results -type d -mindepth 1 -maxdepth 1 | head -n 1) | |
| echo "RESULT_DIR=${RESULT_DIR}" >> $GITHUB_ENV | |
| - name: Process results - ${{ matrix.device-pool }} | |
| if: always() | |
| run: | | |
| set +e # Disable automatic exit on error to properly capture exit codes | |
| if [ -n "$RESULT_DIR" ] && [ -d "$RESULT_DIR" ]; then | |
| mobilebench results parse "$RESULT_DIR" --format console --fail-on-error > "$RESULT_DIR/test_results_summary.txt" | |
| PARSE_EXIT_CODE=$? | |
| mobilebench results parse "$RESULT_DIR" --format json > "$RESULT_DIR/test_results.json" | |
| mobilebench results parse "$RESULT_DIR" --format github > "$RESULT_DIR/test_results_github.md" | |
| if [ $PARSE_EXIT_CODE -ne 0 ]; then | |
| echo "::error::Some tests failed on ${{ matrix.device-pool }}" | |
| exit 1 | |
| fi | |
| else | |
| echo "::error::No results found for ${{ matrix.device-pool }}" | |
| exit 1 | |
| fi | |
| - name: Upload to S3 - ${{ matrix.device-pool }} | |
| if: always() && env.RESULT_DIR != '' | |
| run: | | |
| # Create metadata file with context | |
| cat > "$RESULT_DIR/github_context.json" << EOF | |
| { | |
| "workflow_run_id": "${{ github.run_id }}", | |
| "workflow_run_url": "${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}", | |
| "repository": "${{ github.repository }}", | |
| "branch": "${{ github.ref_name }}", | |
| "commit_sha": "${{ github.sha }}", | |
| "pr_number": "${{ github.event.pull_request.number }}", | |
| "pr_title": "${{ github.event.pull_request.title }}", | |
| "device_pool": "${{ matrix.device-pool }}", | |
| "timestamp": "$(date -u +%Y-%m-%dT%H:%M:%SZ)", | |
| "triggered_by": "${{ github.event_name }}", | |
| "actor": "${{ github.actor }}" | |
| } | |
| EOF | |
| REPO_NAME="${{ github.event.repository.name }}" | |
| DATETIME=$(date +%Y-%m-%d-%H%M%S) | |
| COMMIT_SHORT="${{ github.sha }}" && COMMIT_SHORT="${COMMIT_SHORT:0:8}" | |
| if [ "${{ github.event_name }}" = "pull_request" ]; then | |
| S3_PATH="s3://mobilebench-7f4a9d2c/${REPO_NAME}-tests/pr/${{ github.event.pull_request.number }}/${DATETIME}-${COMMIT_SHORT}/${{ matrix.device-pool }}" | |
| else | |
| S3_PATH="s3://mobilebench-7f4a9d2c/${REPO_NAME}-tests/${{ github.ref_name }}/${DATETIME}-${COMMIT_SHORT}/${{ matrix.device-pool }}" | |
| fi | |
| aws s3 sync "$RESULT_DIR" "${S3_PATH}/" --no-progress | |
| echo "S3_RESULTS_PATH=${S3_PATH}" >> $GITHUB_ENV | |
| - name: Upload artifacts - ${{ matrix.device-pool }} | |
| uses: actions/upload-artifact@v4 | |
| if: always() && env.RESULT_DIR != '' | |
| with: | |
| name: test-results-${{ matrix.device-pool }}-${{ github.run_id }} | |
| path: mobile/results/ | |
| retention-days: 90 | |
| - name: Summary - ${{ matrix.device-pool }} | |
| if: always() | |
| run: | | |
| echo "## 📱 Mobile Test Results - ${{ matrix.device-pool }}" >> $GITHUB_STEP_SUMMARY | |
| echo "" >> $GITHUB_STEP_SUMMARY | |
| if [ -f "$RESULT_DIR/test_results_github.md" ]; then | |
| cat "$RESULT_DIR/test_results_github.md" >> $GITHUB_STEP_SUMMARY | |
| elif [ -f "$RESULT_DIR/test_results_summary.txt" ]; then | |
| echo '```' >> $GITHUB_STEP_SUMMARY | |
| head -n 50 "$RESULT_DIR/test_results_summary.txt" >> $GITHUB_STEP_SUMMARY | |
| echo '```' >> $GITHUB_STEP_SUMMARY | |
| else | |
| echo "⚠️ No results found" >> $GITHUB_STEP_SUMMARY | |
| fi | |
| echo "" >> $GITHUB_STEP_SUMMARY | |
| echo "<details><summary>📋 Details</summary>" >> $GITHUB_STEP_SUMMARY | |
| echo "" >> $GITHUB_STEP_SUMMARY | |
| echo "- Device Pool: ${{ matrix.device-pool }}" >> $GITHUB_STEP_SUMMARY | |
| echo "- S3: \`${S3_RESULTS_PATH:-Not uploaded}\`" >> $GITHUB_STEP_SUMMARY | |
| echo "</details>" >> $GITHUB_STEP_SUMMARY |