feat(metricsai): honor --all-authors for testing — count local-run classifier comments (#51)

* feat(metricsai): honor --all-authors in the testing module

The security module already lets --all-authors lift the author allowlist so
comments are matched on the Conventional-Comment label alone (mirrors the shell
harvester's TARGET_USER=ANY). The testing module did not: it only counted
classifier comments authored by METRICSAI_TESTING_GITHUB_AUTHORS (default
github-actions[bot]), so a classifier comment posted under a developer's
identity — the local/Path-A path — was silently dropped.

Thread match_all_authors through the classifier source chain
(fetch_classifier_comments → _scan_repo_classifier → _classify_classifier),
mirroring the existing security chain, and wire settings.all_authors into the
testing module's gather(). --all-authors now applies to both modules.

Tests mirror the security match_all_authors coverage in test_github_source.py.

* docs(test-classifier): note TARGET_USER=ANY / --all-authors for local-run metrics

Local --post-comment runs post under the developer's identity, so the default
author gate drops them. Note the escape hatch in the metrics README next to the
TARGET_USER description (TARGET_USER=ANY for the shell harvester, --all-authors
for metricsai), matching how security keeps author-matching in the metrics doc
rather than the developer setup doc.

Author: fg-nava

metricsai/src/metricsai/cli.py

@@ -125,8 +125,8 @@ def build_parser() -> argparse.ArgumentParser:
     parser.add_argument(
         "--all-authors",
         action="store_true",
-        help="Count security-reviewer comments from any author, ignoring the --author / "
-        "METRICSAI_SECURITY_GITHUB_AUTHORS allowlist (overrides METRICSAI_ALL_AUTHORS).",
+        help="Count comments from any author (security and testing), matching on the "
+        "Conventional-Comment label alone (overrides METRICSAI_ALL_AUTHORS).",
     )
     parser.add_argument(
         "--skip-sechub",

metricsai/src/metricsai/modules/testing.py

@@ -60,6 +60,7 @@ def gather(self, ctx: RunContext) -> dict[str, MetricValue]:
             authors=settings.testing_authors,
             start=start,
             end=end,
+            match_all_authors=settings.all_authors,
         )
         return _aggregate_classifications(classifications)
 

metricsai/src/metricsai/sources/github.py

@@ -152,6 +152,7 @@ def fetch_classifier_comments(
     authors: Iterable[str],
     start: datetime,
     end: datetime,
+    match_all_authors: bool = False,
 ) -> list[Classification]:
     """Fetch the AI test-classifier's comments and expand them to per-verdict records.
 
@@ -167,6 +168,7 @@ def fetch_classifier_comments(
     :param authors: Comment author logins to keep (case-insensitive).
     :param start: Inclusive window start (timezone-aware UTC).
     :param end: Inclusive window end (timezone-aware UTC).
+    :param match_all_authors: When ``True``, accept any author and ignore ``authors``.
     :returns: One record per classification across all matching comments.
     """
     gh = Github(base_url=base_url, auth=Auth.Token(token))
@@ -176,14 +178,25 @@ def fetch_classifier_comments(
         logger.debug("Scanning %s for classifier comments", full_name)
         out.extend(
             _scan_repo_classifier(
-                gh.get_repo(full_name), full_name, authors_lower, start=start, end=end
+                gh.get_repo(full_name),
+                full_name,
+                authors_lower,
+                start=start,
+                end=end,
+                match_all_authors=match_all_authors,
             )
         )
     return out
 
 
 def _scan_repo_classifier(
-    repo: object, full_name: str, authors_lower: set[str], *, start: datetime, end: datetime
+    repo: object,
+    full_name: str,
+    authors_lower: set[str],
+    *,
+    start: datetime,
+    end: datetime,
+    match_all_authors: bool = False,
 ) -> list[Classification]:
     """Scan one repository's PR issue + review comments for classifier verdicts.
 
@@ -197,14 +210,21 @@ def _scan_repo_classifier(
     :param authors_lower: Lower-cased author logins to keep.
     :param start: Inclusive window start (UTC).
     :param end: Inclusive window end (UTC).
+    :param match_all_authors: When ``True``, accept any author and ignore ``authors_lower``.
     :returns: The classifications harvested from this repository's classifier comments.
     """
     matched: list[Classification] = []
     counts = {"issue": [0, 0], "review": [0, 0]}
 
     def record(source: str, obj: object) -> None:
         counts[source][0] += 1
-        records = _classify_classifier(obj, authors_lower=authors_lower, start=start, end=end)
+        records = _classify_classifier(
+            obj,
+            authors_lower=authors_lower,
+            start=start,
+            end=end,
+            match_all_authors=match_all_authors,
+        )
         if records:
             matched.extend(records)
             counts[source][1] += 1
@@ -223,7 +243,12 @@ def record(source: str, obj: object) -> None:
 
 
 def _classify_classifier(
-    obj: object, *, authors_lower: set[str], start: datetime, end: datetime
+    obj: object,
+    *,
+    authors_lower: set[str],
+    start: datetime,
+    end: datetime,
+    match_all_authors: bool = False,
 ) -> list[Classification]:
     """Expand one raw comment into its classifications, or ``[]`` if it does not count.
 
@@ -236,14 +261,15 @@ def _classify_classifier(
     :param authors_lower: Lower-cased author logins to keep.
     :param start: Inclusive window start (UTC).
     :param end: Inclusive window end (UTC).
+    :param match_all_authors: When ``True``, accept any author and ignore ``authors_lower``.
     :returns: One :class:`Classification` per verdict, or ``[]`` when the comment is skipped.
     """
     body = getattr(obj, "body", None)
     login = _login(obj)
     when = getattr(obj, "created_at", None)
     if not body or login is None or when is None:
         return []
-    if login.lower() not in authors_lower:
+    if not match_all_authors and login.lower() not in authors_lower:
         return []
     timestamp = _as_utc(when)
     if not (start <= timestamp <= end):

metricsai/tests/test_github_source.py

@@ -356,6 +356,25 @@ def test_classify_classifier_skips_wrong_label_author_window() -> None:
     assert github._classify_classifier(late, **gate) == []
 
 
+def test_classify_classifier_match_all_authors_accepts_any_author() -> None:
+    # A classifier comment posted under a developer's identity (Path A) still counts.
+    obj = _classifier_obj("someone-else", _classifier_body("APPLICATION_BUG"), _IN_WINDOW, "x")
+    out = github._classify_classifier(
+        obj, authors_lower={_CLASSIFIER}, start=_START, end=_END, match_all_authors=True
+    )
+    assert [c.verdict for c in out] == ["APPLICATION_BUG"]
+
+
+def test_classify_classifier_match_all_authors_still_enforces_window_and_label() -> None:
+    # The author gate is lifted, but the window and label gates still apply.
+    gate = {"authors_lower": {_CLASSIFIER}, "start": _START, "end": _END, "match_all_authors": True}
+    when = datetime(2026, 6, 30, tzinfo=UTC)
+    late = _classifier_obj("anyone", _classifier_body("TEST_BUG"), when, "x")
+    assert github._classify_classifier(late, **gate) == []
+    wrong_label = _classifier_obj("anyone", "security: leak", _IN_WINDOW, "x")
+    assert github._classify_classifier(wrong_label, **gate) == []
+
+
 def test_fetch_classifier_comments_scans_both_surfaces(monkeypatch) -> None:
     issue = _classifier_obj(
         _CLASSIFIER,

testing/metrics/README.md

@@ -63,6 +63,10 @@ the body starts with `test-classifier:`. Reaction counts are read from the
 inlined `.reactions` summary that both the issue-comments and pull-comments list
 endpoints return on each comment — no extra per-comment reactions API call.
 
+Set `TARGET_USER=ANY` to match on the `test-classifier:` label alone, for teams
+whose developers post the classifier locally (so the author is the developer, not
+a fixed bot). The `metricsai` tool exposes the same via `--all-authors`.
+
 ## Running it (TSV fallback — the default)
 
 ```bash