Fix GPU validation authorization and accounting retries (#7732)

* ci: honor Triage role for GPU requests

* ci: retry transient Slurm accounting failures

Author: Stardust0831

.ci/slurm/runner.py

@@ -1220,7 +1220,7 @@ def github_admit() -> int:
         event_data = json.loads(Path(os.environ["GITHUB_EVENT_PATH"]).read_text(encoding="utf-8"))
         user = event_data["comment"]["user"]["login"]
         permission = _gh("repos/{}/collaborators/{}/permission".format(repository, user))
-        if permission.get("permission") not in ("admin", "maintain", "write", "triage"):
+        if permission.get("role_name") not in ("admin", "maintain", "write", "triage"):
             raise ValueError("commenter needs Triage permission")
         number = str(event_data["issue"]["number"])
         pull = _gh("repos/{}/pulls/{}".format(repository, number))

.ci/slurm/slurm.py

@@ -71,8 +71,17 @@ def wait(
                     raise
             time.sleep(self.poll_seconds)
 
+        failures = 0
         for _ in range(30):
-            rows = self._accounting(ids)
+            try:
+                rows = self._accounting(ids)
+                failures = 0
+            except SlurmError:
+                failures += 1
+                if failures == 6:
+                    raise
+                time.sleep(self.poll_seconds)
+                continue
             required = []
             for job in jobs:
                 count = self.jobs[job]

.ci/slurm/test_runner.py

@@ -259,6 +259,18 @@ def test_submit_and_accounting_require_each_array_task(self):
         self.assertEqual(states["101_0"], ("COMPLETED", "0:0"))
         self.assertEqual(states["101_1"], ("FAILED", "1:0"))
 
+    def test_accounting_retries_transient_failure(self):
+        responses = [
+            mock.Mock(returncode=0, stdout="", stderr=""),
+            mock.Mock(returncode=1, stdout="", stderr="Socket timed out"),
+            mock.Mock(returncode=0, stdout="101|COMPLETED|0:0\n", stderr=""),
+        ]
+        with mock.patch("slurm.subprocess.run", side_effect=responses):
+            client = slurm.Slurm(poll_seconds=0)
+            client.jobs["101"] = None
+            states = client.wait(("101",))
+        self.assertEqual(states["101"], ("COMPLETED", "0:0"))
+
     def test_pass_requires_successful_slurm_accounting(self):
         config = runner.Config(
             runner.Site("Example cluster", "https://cluster.example/", "Computing resources were provided by"),
@@ -808,6 +820,26 @@ def test_mpi_startup_failure_requires_complete_signature(self):
 
 
 class GitHubTests(unittest.TestCase):
+    def test_read_user_cannot_trigger_pr_validation(self):
+        with tempfile.TemporaryDirectory() as directory:
+            root = Path(directory)
+            event = root / "event.json"
+            event.write_text(json.dumps({
+                "comment": {"user": {"login": "reader"}},
+                "issue": {"number": 23},
+            }), encoding="utf-8")
+            output = root / "output"
+            environment = {
+                "GITHUB_EVENT_NAME": "issue_comment", "GITHUB_REPOSITORY": "owner/repo",
+                "GITHUB_EVENT_PATH": str(event), "GITHUB_OUTPUT": str(output),
+            }
+            with mock.patch.dict(os.environ, environment, clear=True), \
+                    mock.patch("runner._gh", return_value={"permission": "read", "role_name": "read"}) as api:
+                with self.assertRaisesRegex(ValueError, "commenter needs Triage permission"):
+                    runner.github_admit()
+            api.assert_called_once_with("repos/owner/repo/collaborators/reader/permission")
+            self.assertFalse(output.exists())
+
     def test_pr_comment_is_created_queued_and_updated_in_place(self):
         source_sha = "a" * 40
         with tempfile.TemporaryDirectory() as directory:
@@ -819,7 +851,7 @@ def test_pr_comment_is_created_queued_and_updated_in_place(self):
             }), encoding="utf-8")
             output = root / "output"
             admitted = [
-                {"permission": "triage"},
+                {"permission": "read", "role_name": "triage"},
                 {"state": "open", "head": {"repo": {"full_name": "owner/fork"}, "sha": source_sha}},
                 {"id": 456}, {"id": 123},
             ]