Fix advisor-coverage installation-token mint and harden backfill dispatch (#8585)

**Impact:** pytorch-advisor-coverage lambda only
**Risk:** low

## What
Replaces the broken scoped installation-token mint with one that
actually returns a usable token, re-mints the token ahead of its
60-minute expiry so long backfills don't die mid-run, and raises the
inter-dispatch gap floor for backfill mode.

## Why
The `actions:write`-scoped mint went through `Auth.AppInstallationAuth`,
whose `.token` only resolves once the auth object is handed to a
`github.Github(...)` client. Reading `.token` off a standalone instance
asserts, so every real dispatch run failed at runtime. It now mints via
`GithubIntegration.get_access_token`, which returns the full
authorization (token + `expires_at`).

Two follow-on gaps this exposes:
- A backfill walks months of history off a single startup mint and runs
well past the token's 60-minute lifetime. The dispatch path now re-mints
when the token nears GitHub's stated `expires_at` (judged against
`expires_at`, not elapsed process time, so a suspended-then-resumed run
doesn't dispatch with a dead token).
- Backfill sustains a dispatch rate the ongoing cron never reaches and
is the only mode that can trip GitHub's secondary rate limit on
`workflow_dispatch`, so its gap now floors at 5s (vs 1s for ongoing).
Env/event may still raise the gap, never lower it below the floor.

# Notes
- Mint failures now raise a `RuntimeError` naming `GITHUB_APP_ID` /
`GITHUB_INSTALLATION_ID`, which disambiguates the common "key registered
to a different App" error from a genuine GitHub outage.

---------

Signed-off-by: Jean Schmidt <contato@jschmidt.me>

Author: jeanschmidt

aws/lambda/pytorch-advisor-coverage/.env.example

@@ -29,6 +29,7 @@ AS_OF_STEP_HOURS=24
 
 # Throttle
 MAX_DISPATCHES_PER_RUN=10
+# Floored to 1s in ongoing mode, 5s in backfill mode (GitHub secondary rate limit).
 DISPATCH_GAP_SECONDS=3
 
 # Safety: DRY_RUN=true logs intended dispatches without POSTing. Set to false to

aws/lambda/pytorch-advisor-coverage/README.md

@@ -120,7 +120,11 @@ MODE=backfill AS_OF_START=2026-02-19 AS_OF_END=2026-08-18 \
 
 - `MAX_DISPATCHES_PER_RUN` (default 10) — per invocation, clamped by a compiled
   `HARD_CAP` (100) and the Lambda timeout budget; env/event may only LOWER it.
-- `DISPATCH_GAP_SECONDS` (default 3) — sleep between dispatches (floored to 1s).
+- `DISPATCH_GAP_SECONDS` (default 3) — sleep between dispatches, floored to 1s in
+  ongoing mode and to 5s in backfill mode. Backfill sustains a dispatch rate the
+  ongoing cron never reaches, so it is the only mode that can hit GitHub's
+  secondary rate limit on `workflow_dispatch`. The env may raise the gap past
+  either floor, never below it.
 
 Cross-invocation duplicates (a red re-dispatched before its verdict lands) are
 accepted: safe (prefixed → no reverts) and bounded by the throttle. Intra-run
@@ -155,7 +159,7 @@ dispatched once. This is bounded, safe (non-reverting), and matches
 | `AS_OF_START` / `AS_OF_END` | — | backfill range (UTC) |
 | `AS_OF_STEP_HOURS` | `24` | backfill chunk size |
 | `MAX_DISPATCHES_PER_RUN` | `10` | see Throttle |
-| `DISPATCH_GAP_SECONDS` | `3` | see Throttle |
+| `DISPATCH_GAP_SECONDS` | `3` | floored to 5s in backfill mode; see Throttle |
 | `DRY_RUN` | `true` | `false` arms real dispatch |
 | `LOG_LEVEL` | `INFO` | secret-leaking loggers pinned to WARNING regardless |
 

aws/lambda/pytorch-advisor-coverage/advisor_coverage/bootstrap.py

@@ -18,6 +18,8 @@
 import logging
 import sys
 from dataclasses import dataclass
+from datetime import datetime, timedelta, timezone
+from typing import Optional
 
 import boto3
 import github
@@ -43,6 +45,27 @@
 # excludes contents:write, so a leaked coverage token cannot push/revert.
 _DISPATCH_TOKEN_PERMISSIONS = {"actions": "write"}
 
+# GitHub installation tokens expire 60 minutes after minting, and the token is a
+# plain string handed to GHClientFactory — nothing refreshes it. A backfill
+# dispatches for hours off one startup mint, so the dispatch path re-mints ahead
+# of expiry. Freshness is judged against GitHub's own `expires_at` rather than
+# elapsed local time: a laptop suspended mid-backfill wakes with the token
+# already dead while a monotonic counter still reads it as fresh.
+_TOKEN_REFRESH_MARGIN = timedelta(minutes=10)
+
+
+@dataclass
+class _DispatchAuth:
+    """App credentials retained so the dispatch token can be re-minted."""
+
+    app_id: str
+    pem: str
+    installation_id: int
+    expires_at: datetime
+
+
+_dispatch_auth: Optional[_DispatchAuth] = None
+
 
 def configure_logging(log_level: str) -> None:
     """Configure logging and pin secret-leaking third-party loggers to WARNING."""
@@ -91,20 +114,35 @@ def _get_secret_from_aws(secret_store_name: str) -> _AWSSecrets:
         sys.exit(1)
 
 
-def _mint_scoped_installation_token(app_id: str, pem: str, installation_id: int) -> str:
-    """Mint an installation token scoped to `actions:write` only.
+def _mint_scoped_installation_auth(app_id: str, pem: str, installation_id: int):
+    """Mint an installation authorization scoped to `actions:write` only.
+
+    Returns the whole authorization, not just the token string, because its
+    `expires_at` is the only trustworthy basis for deciding when to re-mint.
 
-    Without token_permissions the mint inherits the App's full permission set
+    Without explicit permissions the mint inherits the App's full permission set
     (incl. contents:write → revert-capable). Scoping to actions:write is the
     minimum for workflow_dispatch and removes revert capability entirely.
+
+    Minted through GithubIntegration rather than Auth.AppInstallationAuth:
+    PyGithub (2.6.1) only builds that auth object's internal integration inside
+    `withRequester`, which nothing calls until the auth is handed to a
+    `github.Github(...)`, so reading `.token` off a standalone instance always
+    asserts.
     """
-    app_auth = github.Auth.AppAuth(app_id, pem)
-    inst_auth = github.Auth.AppInstallationAuth(
-        app_auth,
-        installation_id=installation_id,
-        token_permissions=_DISPATCH_TOKEN_PERMISSIONS,
-    )
-    return inst_auth.token
+    integration = github.GithubIntegration(auth=github.Auth.AppAuth(app_id, pem))
+    try:
+        return integration.get_access_token(
+            installation_id, permissions=_DISPATCH_TOKEN_PERMISSIONS
+        )
+    except github.GithubException as e:
+        # GitHub answers a key that is validly formed but registered to a
+        # different App with "A JSON web token could not be decoded" — naming the
+        # identifiers is what distinguishes that from a genuine outage.
+        raise RuntimeError(
+            f"Failed to mint an installation token for GITHUB_APP_ID={app_id}, "
+            f"GITHUB_INSTALLATION_ID={installation_id}: {e}"
+        ) from e
 
 
 def setup_clients(config: CoverageConfig) -> None:
@@ -128,10 +166,17 @@ def setup_clients(config: CoverageConfig) -> None:
     )
 
     if config.github_app_id and config.github_installation_id and gh_app_secret:
-        scoped_token = _mint_scoped_installation_token(
+        global _dispatch_auth
+        scoped = _mint_scoped_installation_auth(
             config.github_app_id, gh_app_secret, config.github_installation_id
         )
-        GHClientFactory.setup_client(token=scoped_token)
+        GHClientFactory.setup_client(token=scoped.token)
+        _dispatch_auth = _DispatchAuth(
+            app_id=config.github_app_id,
+            pem=gh_app_secret,
+            installation_id=config.github_installation_id,
+            expires_at=scoped.expires_at,
+        )
     elif config.github_access_token:
         GHClientFactory.setup_client(token=config.github_access_token)
     else:
@@ -145,3 +190,27 @@ def setup_clients(config: CoverageConfig) -> None:
         raise RuntimeError(
             "ClickHouse connection test failed. Please check your configuration."
         )
+
+
+def refresh_dispatch_token_if_stale() -> bool:
+    """Re-mint the installation token when it is close to GitHub's stated expiry.
+
+    Returns True when a new token was installed. A no-op when the client was
+    configured from a raw GITHUB_TOKEN — there are no App credentials to re-mint
+    from, and a PAT does not expire on this timescale.
+    """
+    if _dispatch_auth is None:
+        return False
+    if datetime.now(timezone.utc) + _TOKEN_REFRESH_MARGIN < _dispatch_auth.expires_at:
+        return False
+
+    scoped = _mint_scoped_installation_auth(
+        _dispatch_auth.app_id, _dispatch_auth.pem, _dispatch_auth.installation_id
+    )
+    GHClientFactory.setup_client(token=scoped.token)
+    _dispatch_auth.expires_at = scoped.expires_at
+    logging.info(
+        "[coverage] re-minted the installation token, now valid until %s",
+        scoped.expires_at.isoformat(timespec="seconds"),
+    )
+    return True

aws/lambda/pytorch-advisor-coverage/advisor_coverage/config.py

@@ -33,6 +33,10 @@
 # Compiled-in safety ceilings. Env/event may only make the throttle SMALLER.
 HARD_CAP_DISPATCHES = 100
 MIN_DISPATCH_GAP_SECONDS = 1
+# Backfill walks months of history in one pass, so it sustains a dispatch rate
+# the ongoing cron never approaches and is the only mode that can reach GitHub's
+# secondary rate limit on workflow_dispatch. It floors far higher as a result.
+BACKFILL_MIN_DISPATCH_GAP_SECONDS = 5
 # The Lambda's configured timeout (tf). The throttle is clamped so a run's
 # inter-dispatch sleeps can never approach it.
 LAMBDA_TIMEOUT_SECONDS = 260
@@ -146,8 +150,17 @@ class CoverageConfig:
     log_level: str = "INFO"
 
     def effective_gap_seconds(self) -> int:
-        """Gap floored to a positive minimum (never 0 → no dispatch storm)."""
-        return max(MIN_DISPATCH_GAP_SECONDS, int(self.dispatch_gap_seconds))
+        """Gap floored to a positive minimum (never 0 → no dispatch storm).
+
+        Backfill floors at its own, much higher minimum; env/event can raise the
+        gap beyond either floor but never below it.
+        """
+        floor = (
+            BACKFILL_MIN_DISPATCH_GAP_SECONDS
+            if self.mode == "backfill"
+            else MIN_DISPATCH_GAP_SECONDS
+        )
+        return max(floor, int(self.dispatch_gap_seconds))
 
     def effective_max_dispatches(self) -> int:
         """Configured cap, clamped by HARD_CAP and the Lambda timeout budget.

aws/lambda/pytorch-advisor-coverage/advisor_coverage/dispatcher.py

@@ -23,6 +23,7 @@
 from pytorch_auto_revert.github_client_helper import GHClientFactory
 from pytorch_auto_revert.utils import proper_workflow_create_dispatch, RetryWithBackoff
 
+from .bootstrap import refresh_dispatch_token_if_stale
 from .config import ADVISOR_WORKFLOW_FILE, COVERAGE_SIGNAL_KEY_PREFIX, CoverageConfig
 from .enumeration import _naive_utc, RedSignal, UnclassifiedRedEnumerator
 from .logfilter import has_readable_log
@@ -287,6 +288,7 @@ def _dispatch_one(
             )
             return
 
+        refresh_dispatch_token_if_stale()
         workflow = self._advisor_workflow()
         factory = GHClientFactory()
         # /dispatches is non-idempotent — single attempt via the retry=0

aws/lambda/pytorch-advisor-coverage/advisor_coverage/tests/test_advisor_coverage.py

@@ -8,13 +8,14 @@
 import logging
 import time
 import unittest
-from datetime import datetime, timedelta
+from datetime import datetime, timedelta, timezone
 from unittest.mock import MagicMock, patch
 
 from advisor_coverage import config as config_mod
 from advisor_coverage.backfill import run_backfill
 from advisor_coverage.config import (
     _parse_workflows,
+    BACKFILL_MIN_DISPATCH_GAP_SECONDS,
     COVERAGE_SIGNAL_KEY_PREFIX,
     CoverageConfig,
     HARD_CAP_DISPATCHES,
@@ -97,6 +98,25 @@ def make_red(
     )
 
 
+def _minted(token: str, valid_for: timedelta) -> MagicMock:
+    """A stand-in for PyGithub's InstallationAuthorization."""
+    auth = MagicMock()
+    auth.token = token
+    auth.expires_at = datetime.now(timezone.utc) + valid_for
+    return auth
+
+
+def _auth_expiring_in(remaining: timedelta):
+    from advisor_coverage.bootstrap import _DispatchAuth
+
+    return _DispatchAuth(
+        app_id="a",
+        pem="PEM",
+        installation_id=1,
+        expires_at=datetime.now(timezone.utc) + remaining,
+    )
+
+
 def _urlopen_cm(status, content_length):
     """A fake urlopen() context manager for logfilter HEAD tests."""
     resp = MagicMock()
@@ -557,6 +577,13 @@ def test_timeout_clamp(self):
     def test_gap_floor(self):
         self.assertEqual(make_config(dispatch_gap_seconds=0).effective_gap_seconds(), 1)
 
+    def test_backfill_gap_floor(self):
+        cfg = make_config(mode="backfill", dispatch_gap_seconds=3)
+        self.assertEqual(cfg.effective_gap_seconds(), BACKFILL_MIN_DISPATCH_GAP_SECONDS)
+        raised = make_config(mode="backfill", dispatch_gap_seconds=60)
+        self.assertEqual(raised.effective_gap_seconds(), 60)
+        self.assertEqual(make_config(dispatch_gap_seconds=3).effective_gap_seconds(), 3)
+
     def test_repo_pin(self):
         with self.assertRaises(ValueError):
             CoverageConfig.from_env_and_event({"repo_full_name": "evil/repo"})
@@ -835,18 +862,96 @@ def test_configure_logging_pins_secret_loggers(self):
         for name in ("github", "github.Requester", "botocore", "boto3", "urllib3"):
             self.assertEqual(logging.getLogger(name).level, logging.WARNING, name)
 
-    @patch("advisor_coverage.bootstrap.github")
-    def test_mint_scopes_token_to_actions_write(self, mock_github):
-        from advisor_coverage.bootstrap import _mint_scoped_installation_token
+    # Patches only GithubIntegration, so the surrounding call path runs against
+    # real PyGithub: a mint that cannot produce a token without a Requester
+    # fails here instead of passing against an all-mocked github module.
+    @patch("advisor_coverage.bootstrap.github.GithubIntegration")
+    def test_mint_scopes_token_to_actions_write(self, mock_integration):
+        import github
+        from advisor_coverage.bootstrap import _mint_scoped_installation_auth
+
+        get_token = mock_integration.return_value.get_access_token
+        get_token.return_value.token = "ghs_scoped"
+        scoped = _mint_scoped_installation_auth("app-id", "PEM", 4242)
+        self.assertEqual(scoped.token, "ghs_scoped")
+        self.assertIsInstance(
+            mock_integration.call_args.kwargs["auth"], github.Auth.AppAuth
+        )
+        args, kwargs = get_token.call_args
+        self.assertEqual(args, (4242,))
+        self.assertEqual(kwargs["permissions"], {"actions": "write"})
+
+    @patch("advisor_coverage.bootstrap.github.GithubIntegration")
+    def test_mint_failure_names_the_identifiers(self, mock_integration):
+        import github
+        from advisor_coverage.bootstrap import _mint_scoped_installation_auth
 
-        inst = MagicMock()
-        inst.token = "ghs_scoped"
-        mock_github.Auth.AppInstallationAuth.return_value = inst
-        token = _mint_scoped_installation_token("app-id", "PEM", 4242)
-        self.assertEqual(token, "ghs_scoped")
-        kwargs = mock_github.Auth.AppInstallationAuth.call_args.kwargs
-        self.assertEqual(kwargs["token_permissions"], {"actions": "write"})
-        self.assertEqual(kwargs["installation_id"], 4242)
+        mock_integration.return_value.get_access_token.side_effect = (
+            github.GithubException(401, {"message": "could not be decoded"}, None)
+        )
+        with self.assertRaises(RuntimeError) as ctx:
+            _mint_scoped_installation_auth("app-id", "PEM", 4242)
+        self.assertIn("app-id", str(ctx.exception))
+        self.assertIn("4242", str(ctx.exception))
+
+    def test_token_refresh_noop_without_app_auth(self):
+        from advisor_coverage import bootstrap
+
+        bootstrap._dispatch_auth = None
+        self.assertFalse(bootstrap.refresh_dispatch_token_if_stale())
+
+    def test_token_refresh_noop_while_fresh(self):
+        from advisor_coverage import bootstrap
+
+        bootstrap._dispatch_auth = _auth_expiring_in(timedelta(minutes=59))
+        try:
+            with patch(
+                "advisor_coverage.bootstrap._mint_scoped_installation_auth"
+            ) as mint:
+                self.assertFalse(bootstrap.refresh_dispatch_token_if_stale())
+            mint.assert_not_called()
+        finally:
+            bootstrap._dispatch_auth = None
+
+    def test_token_reminted_inside_the_expiry_margin(self):
+        from advisor_coverage import bootstrap
+
+        # Still valid, but inside the margin — a dispatch now could outlive it.
+        bootstrap._dispatch_auth = _auth_expiring_in(timedelta(minutes=5))
+        try:
+            with patch(
+                "advisor_coverage.bootstrap._mint_scoped_installation_auth",
+                return_value=_minted("ghs_new", timedelta(minutes=60)),
+            ) as mint, patch("advisor_coverage.bootstrap.GHClientFactory") as ghf:
+                self.assertTrue(bootstrap.refresh_dispatch_token_if_stale())
+                mint.assert_called_once_with("a", "PEM", 1)
+                ghf.setup_client.assert_called_once_with(token="ghs_new")
+                # Expiry advanced, so the next dispatch does not re-mint again.
+                self.assertFalse(bootstrap.refresh_dispatch_token_if_stale())
+        finally:
+            bootstrap._dispatch_auth = None
+
+    def test_token_reminted_after_a_suspend(self):
+        """An already-expired token re-mints even with no elapsed process time."""
+        from advisor_coverage import bootstrap
+
+        bootstrap._dispatch_auth = _auth_expiring_in(timedelta(minutes=-30))
+        try:
+            with patch(
+                "advisor_coverage.bootstrap._mint_scoped_installation_auth",
+                return_value=_minted("ghs_new", timedelta(minutes=60)),
+            ), patch("advisor_coverage.bootstrap.GHClientFactory"):
+                self.assertTrue(bootstrap.refresh_dispatch_token_if_stale())
+        finally:
+            bootstrap._dispatch_auth = None
+
+    def test_dispatch_checks_token_freshness(self):
+        reds = [make_red(job_name="j / t", observed="o1")]
+        with patch(
+            "advisor_coverage.dispatcher.refresh_dispatch_token_if_stale"
+        ) as refresh, _DispatchHarness(reds, config=make_config(dry_run=False)) as h:
+            h.dispatcher.dispatch_for_window(T(0), T(59))
+        refresh.assert_called_once_with()
 
     def test_setup_clients_uses_scoped_token(self):
         cfg = make_config(
@@ -855,8 +960,8 @@ def test_setup_clients_uses_scoped_token(self):
             github_app_secret=base64.b64encode(b"PEM").decode(),
         )
         with patch(
-            "advisor_coverage.bootstrap._mint_scoped_installation_token",
-            return_value="ghs_scoped",
+            "advisor_coverage.bootstrap._mint_scoped_installation_auth",
+            return_value=_minted("ghs_scoped", timedelta(minutes=60)),
         ) as mint, patch("advisor_coverage.bootstrap.GHClientFactory") as ghf, patch(
             "advisor_coverage.bootstrap.CHCliFactory"
         ) as ch: