diff --git a/.github/scripts/test_report_comment.dart b/.github/scripts/test_report_comment.dart new file mode 100644 index 00000000..b94df2df --- /dev/null +++ b/.github/scripts/test_report_comment.dart @@ -0,0 +1,225 @@ +// Generates a markdown PR comment summarising test results. +// +// Usage: dart .github/scripts/test_report_comment.dart +// +// Reads every *.json file in (dart/flutter test +// `--file-reporter=json` output, one JSON event per line) and writes a +// markdown comment to : +// +// * a [!CAUTION] callout when at least one test failed, [!NOTE] otherwise +// * a per-package summary table (test count and success rate) +// * for packages with failures only: a collapsible per-file breakdown +// listing each test in files that contain at least one failing test + +import 'dart:convert'; +import 'dart:io'; + +void main(List args) { + if (args.length != 2) { + stderr.writeln( + 'Usage: dart test_report_comment.dart ', + ); + exit(64); + } + + final reportFiles = Directory(args.first) + .listSync() + .whereType() + .where((f) => f.path.endsWith('.json')) + .toList() + ..sort((a, b) => a.path.compareTo(b.path)); + + final packages = [ + for (final file in reportFiles) _PackageReport.parse(file), + ]; + + File(args.last).writeAsStringSync(_render(packages)); +} + +class _TestResult { + const _TestResult(this.name, this.result, this.skipped); + + final String name; + final String result; // success | failure | error + final bool skipped; + + bool get failed => result != 'success' && !skipped; +} + +class _PackageReport { + _PackageReport(this.name); + + /// Package name, derived from the report file name + /// (e.g. `clerk_auth.json` -> `clerk_auth`). + final String name; + + /// Test results keyed by display path of the test file. + final byFile = >{}; + + int get total => byFile.values.fold(0, (sum, tests) => sum + tests.length); + + int get failed => byFile.values + .fold(0, (sum, tests) => sum + tests.where((t) => t.failed).length); + + int get successRate => total == 0 ? 100 : ((total - failed) * 100) ~/ total; + + static _PackageReport parse(File file) { + final fileName = file.uri.pathSegments.last; + final report = + _PackageReport(fileName.substring(0, fileName.length - '.json'.length)); + + final suitePaths = {}; + final groupNames = {}; + final testsById = >{}; + + for (final line in file.readAsLinesSync()) { + if (line.trim().isEmpty) continue; + + final Map event; + try { + event = json.decode(line) as Map; + } on FormatException { + continue; // tolerate non-JSON noise in the report file + } + + switch (event['type']) { + case 'suite': + final suite = event['suite'] as Map; + suitePaths[suite['id'] as int] = + _displayPath(suite['path'] as String? ?? ''); + + case 'group': + final group = event['group'] as Map; + groupNames[group['id'] as int] = group['name'] as String? ?? ''; + + case 'testStart': + final test = event['test'] as Map; + testsById[test['id'] as int] = test; + + case 'testDone': + // Hidden tests are bookkeeping entries (e.g. suite loading). + if (event['hidden'] == true) continue; + final test = testsById[event['testID']]; + if (test == null) continue; + final path = suitePaths[test['suiteID']] ?? '(unknown file)'; + report.byFile.putIfAbsent(path, () => []).add( + _TestResult( + _displayName(test, groupNames), + event['result'] as String? ?? 'error', + event['skipped'] == true, + ), + ); + } + } + + return report; + } + + /// Path of a test file relative to its package root. + static String _displayPath(String path) { + for (final marker in ['integration_test/', 'test/']) { + if (path.startsWith(marker)) return path; + final index = path.indexOf('/$marker'); + if (index >= 0) return path.substring(index + 1); + } + return path.split('/').last; + } + + /// Test name with its group prefix separated by ` · `. + /// + /// Group names and test names are cumulative in the JSON reporter output + /// (each includes its ancestors' names), so successive prefixes are + /// stripped to recover the bare name of each level. + static String _displayName( + Map test, + Map groupNames, + ) { + final segments = []; + var cumulative = ''; + + for (final id in (test['groupIDs'] as List?)?.cast() ?? const []) { + final fullName = groupNames[id] ?? ''; + if (fullName.isEmpty) continue; + var segment = fullName; + if (cumulative.isNotEmpty && fullName.startsWith(cumulative)) { + segment = fullName.substring(cumulative.length).trim(); + } + if (segment.isNotEmpty) segments.add(segment); + cumulative = fullName; + } + + var bareName = test['name'] as String? ?? ''; + if (cumulative.isNotEmpty && bareName.startsWith(cumulative)) { + bareName = bareName.substring(cumulative.length).trim(); + } + + return [...segments, if (bareName.isNotEmpty) bareName].join(' · '); + } +} + +String _render(List<_PackageReport> packages) { + final anyFailures = packages.any((p) => p.failed > 0); + final lines = [ + if (anyFailures) ...[ + '[!CAUTION]', + 'At least one test failed. Please review the test results below.', + ] else ...[ + '[!NOTE]', + 'All tests passed.', + ], + '', + '
', + 'View Test Results', + '', + '| Package | Number of Tests | Success Rate |', + '| :--- | :--- | :--- |', + for (final package in packages) + '| ${_cell(package.name)} | ${package.total} | ${package.successRate}% |', + for (final package in packages.where((p) => p.failed > 0)) + ..._renderPackageBreakdown(package), + '', + '
', + ]; + + return lines.map((line) => line.isEmpty ? '>' : '> $line').join('\n'); +} + +Iterable _renderPackageBreakdown(_PackageReport package) sync* { + yield ''; + yield '
'; + yield '📂 ${_cell(package.name)}'; + + // Only files containing at least one failing test are listed, so the + // breakdown stays skimmable. + final failingFiles = package.byFile.entries + .where((entry) => entry.value.any((test) => test.failed)) + .toList() + ..sort((a, b) => a.key.compareTo(b.key)); + + for (final entry in failingFiles) { + yield ''; + yield '### ${entry.key}'; + yield ''; + yield '| Test | Result |'; + yield '| :--- | :--- |'; + for (final test in entry.value) { + final name = test.failed ? _redText(test.name) : _cell(test.name); + final icon = test.failed ? '❌' : (test.skipped ? '⏭️' : '✅'); + yield '| $name | $icon |'; + } + } + + yield ''; + yield '
'; +} + +/// Escapes a value for use inside a markdown table cell. +String _cell(String value) => value.replaceAll('|', r'\|'); + +/// Renders a failing test name in red via GitHub's LaTeX support. +String _redText(String value) { + final escaped = value + .replaceAll(RegExp(r'[\\~^|]'), ' ') + .replaceAllMapped(RegExp(r'[#$%&_{}]'), (m) => '\\${m[0]}'); + return '\$\\color{red}{\\text{$escaped}}\$'; +} diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml index 4f3a7f5a..33d289c0 100644 --- a/.github/workflows/main.yaml +++ b/.github/workflows/main.yaml @@ -6,6 +6,11 @@ on: pull_request: branches: [ "main" ] +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + # Cancel superseded runs on PR branches, but let runs on main complete. + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + jobs: pr_check: name: PR Check Build @@ -68,10 +73,101 @@ jobs: run: flutter test --file-reporter=json:../../reports/clerk_flutter.json working-directory: packages/clerk_flutter + - name: Upload test reports + uses: actions/upload-artifact@v4 + if: always() # upload reports even if tests failed + with: + name: reports-unit + path: reports/*.json + + integration_test: + name: Integration Tests (iOS Simulator) + runs-on: macos-latest + timeout-minutes: 45 + + steps: + - name: Checkout repo (self) + uses: actions/checkout@v4 + + - name: Install flutter + uses: flutter-actions/setup-flutter@v4 + with: + version: '3.41.9' + + - name: 'Install tools: melos and fvm' + run: dart pub global activate melos && dart pub global activate fvm + + - name: Install FVM + run: fvm install + + - name: Install top-level dependencies + run: dart pub get + + - name: Run melos + run: melos bs && melos run get + + - name: Boot iPhone simulator + id: simulator + run: | + UDID=$(xcrun simctl list devices available --json \ + | jq -r '[.devices | to_entries[] | select(.key | contains("iOS")) | .value[] | select(.name | startswith("iPhone"))] | last | .udid') + if [ -z "$UDID" ] || [ "$UDID" = "null" ]; then + echo "No available iPhone simulator found" >&2 + xcrun simctl list devices available + exit 1 + fi + echo "udid=$UDID" >> "$GITHUB_OUTPUT" + xcrun simctl bootstatus "$UDID" -b + + - name: Run integration tests in clerk_flutter_example + run: melos run test:integration -- -d ${{ steps.simulator.outputs.udid }} --file-reporter=json:$GITHUB_WORKSPACE/reports/clerk_flutter_integration.json + + - name: Upload test reports + uses: actions/upload-artifact@v4 + if: always() # upload reports even if tests failed + with: + name: reports-integration + path: reports/*.json + + test_report: + name: Test Report + runs-on: ubuntu-latest + needs: [ pr_check, integration_test ] + if: ${{ !cancelled() }} # report even if a test job failed + permissions: + contents: read + checks: write # dorny/test-reporter creates a check run + pull-requests: write # sticky comment on the PR + + steps: + - name: Checkout repo (self) + uses: actions/checkout@v4 + + - name: Download test reports + uses: actions/download-artifact@v4 + with: + pattern: reports-* + merge-multiple: true + path: reports + - name: Test Report uses: dorny/test-reporter@v1 - if: success() || failure() # run this step even if previous step failed with: name: 'Test Report' # Name of the check run which will be created path: reports/*.json # Path to test results reporter: flutter-json # Format of test results + + - name: Install dart + uses: dart-lang/setup-dart@v1 + if: ${{ !cancelled() }} # comment even if tests failed + + - name: Generate test summary comment + if: ${{ !cancelled() }} + run: dart .github/scripts/test_report_comment.dart reports test_summary_comment.md + + - name: Post test summary comment + uses: marocchino/sticky-pull-request-comment@v2 + if: ${{ !cancelled() && github.event_name == 'pull_request' }} + with: + header: test-results + path: test_summary_comment.md