|
1 | | -"""Testing / quality metrics module (stub). |
| 1 | +"""Testing / quality metrics module. |
2 | 2 |
|
3 | | -Emits the ``testing_quality_*`` spreadsheet columns. Values are placeholders for now. |
| 3 | +Emits the ``testing_classifier_*`` spreadsheet columns from a single scan of the AI |
| 4 | +test-classifier's PR comments. The classifier posts one comment per CI run with findings, |
| 5 | +each leading with the ``test-classifier:`` Conventional-Comment label and embedding a JSON |
| 6 | +verdict block (one entry per failing test). We expand those into per-verdict records, |
| 7 | +attach each comment's 👍/👎 reaction totals, and aggregate the week into: |
4 | 8 |
|
5 | | -.. todo:: |
6 | | - Source these from the GitHub API / CI, e.g. the thumbs-up rate on AI test-suggestion |
7 | | - comments, the merge rate of accepted suggestions, and time to workflow-run completion. |
8 | | - Set :pyattr:`requires_github_token` to ``True`` when implemented. |
| 9 | +* per-verdict counts (``app_bug`` / ``test_bug`` / ``flaky_failure`` / ``environment_issue``), |
| 10 | +* total comment-level reactions, and |
| 11 | +* the 👍-rate -- the classifier's tuning signal. |
| 12 | +
|
| 13 | +Source: :func:`metricsai.sources.github.fetch_classifier_comments`. |
9 | 14 | """ |
10 | 15 |
|
11 | 16 | from __future__ import annotations |
12 | 17 |
|
13 | 18 | from metricsai.context import RunContext |
14 | | -from metricsai.models import MetricValue |
| 19 | +from metricsai.models import MetricValue, week_window |
15 | 20 | from metricsai.modules import register |
16 | 21 | from metricsai.modules.base import MetricsModule |
| 22 | +from metricsai.sources.github import Classification, fetch_classifier_comments |
| 23 | + |
| 24 | +#: Verdicts the classifier emits, mapped to their spreadsheet column suffix. Records whose |
| 25 | +#: verdict is absent/unparseable (``""``) count toward the comment total but no bucket. |
| 26 | +_VERDICT_COLUMNS = { |
| 27 | + "APPLICATION_BUG": "testing_classifier_app_bug", |
| 28 | + "TEST_BUG": "testing_classifier_test_bug", |
| 29 | + "FLAKY_FAILURE": "testing_classifier_flaky_failure", |
| 30 | + "ENVIRONMENT_ISSUE": "testing_classifier_environment_issue", |
| 31 | +} |
17 | 32 |
|
18 | 33 |
|
19 | 34 | class TestingModule(MetricsModule): |
20 | | - """Gathers testing / quality metrics (currently stubbed).""" |
| 35 | + """Gathers AI test-classifier precision metrics.""" |
21 | 36 |
|
22 | 37 | name = "testing" |
23 | | - requires_github_token = False |
| 38 | + requires_github_token = True |
24 | 39 |
|
25 | 40 | def gather(self, ctx: RunContext) -> dict[str, MetricValue]: |
26 | | - """Return placeholder testing/quality metrics. |
| 41 | + """Collect the classifier metrics for the reporting week. |
27 | 42 |
|
28 | | - :param ctx: The shared per-run context (unused while stubbed). |
29 | | - :returns: Stub metric key/value pairs keyed by spreadsheet column. |
| 43 | + :param ctx: The shared per-run context. |
| 44 | + :returns: The ``testing_classifier_*`` key/value pairs. |
| 45 | + :raises ValueError: If no target repositories are configured. |
30 | 46 | """ |
31 | | - return { |
32 | | - "testing_quality_comment_thumbs_up_rate_pct": 0.0, |
33 | | - "testing_quality_suggestion_merge_rate_pct": 0.0, |
34 | | - "testing_quality_time_to_workflow_run_completion": 0.0, |
35 | | - } |
| 47 | + settings = ctx.settings |
| 48 | + repos = settings.testing_repos |
| 49 | + if not repos: |
| 50 | + raise ValueError( |
| 51 | + "No repositories configured. Set METRICSAI_TESTING_GITHUB_REPOS " |
| 52 | + "(or METRICSAI_GITHUB_REPOS), or pass --repo." |
| 53 | + ) |
| 54 | + |
| 55 | + start, end = week_window(ctx.week_ending_date) |
| 56 | + classifications = fetch_classifier_comments( |
| 57 | + token=ctx.get_github_token(), |
| 58 | + base_url=settings.github_base_url, |
| 59 | + repos=repos, |
| 60 | + authors=settings.testing_authors, |
| 61 | + start=start, |
| 62 | + end=end, |
| 63 | + ) |
| 64 | + return _aggregate_classifications(classifications) |
| 65 | + |
| 66 | + |
| 67 | +def _aggregate_classifications(records: list[Classification]) -> dict[str, MetricValue]: |
| 68 | + """Aggregate per-verdict records into the ``testing_classifier_*`` columns. |
| 69 | +
|
| 70 | + The 👍-rate is rounded to one decimal and is ``0.0`` when there are no reactions, so the |
| 71 | + column is always numeric. Per-comment reactions are repeated across that comment's |
| 72 | + records, so reaction totals are summed over verdict entries exactly as the comments |
| 73 | + carry them. |
| 74 | +
|
| 75 | + :param records: One entry per classification (see |
| 76 | + :class:`~metricsai.sources.github.Classification`). |
| 77 | + :returns: The ``testing_classifier_*`` metrics. |
| 78 | + """ |
| 79 | + counts = dict.fromkeys(_VERDICT_COLUMNS.values(), 0) |
| 80 | + thumbs_up = 0 |
| 81 | + thumbs_down = 0 |
| 82 | + for record in records: |
| 83 | + thumbs_up += record.thumbs_up |
| 84 | + thumbs_down += record.thumbs_down |
| 85 | + column = _VERDICT_COLUMNS.get(record.verdict) |
| 86 | + if column is not None: |
| 87 | + counts[column] += 1 |
| 88 | + |
| 89 | + total_reactions = thumbs_up + thumbs_down |
| 90 | + up_rate = round(100.0 * thumbs_up / total_reactions, 1) if total_reactions else 0.0 |
| 91 | + |
| 92 | + return { |
| 93 | + "testing_classifier_total_classifications": len(records), |
| 94 | + "testing_classifier_thumbs_ups": thumbs_up, |
| 95 | + "testing_classifier_thumbs_downs": thumbs_down, |
| 96 | + "testing_classifier_thumbs_up_rate_pct": up_rate, |
| 97 | + **counts, |
| 98 | + } |
36 | 99 |
|
37 | 100 |
|
38 | 101 | register(TestingModule()) |
0 commit comments