disable issues: throw error if issue retrieval goes wrong (#7141)

Reason: I've noticed that sometimes a lot of stuff in the disable json
gets removed even though the issues weren't closed -> CI runs the tests
which should be disabled -> fails -> HUD is red. Looking at the vercel
logs, I can see `Expected 577 issues with prefix "DISABLED", but found
0.`, so it's probably that we are somehow failing to fetch the issues.

My solution is to thrown an error in the API call, which then gets
caught by the python script and raises an runtime error, so in the case
of an error, the json won't get updated (and the old one will be left)

Additional things that could be done: don't update json if dramatic
changes happen (ex count goes from 100 -> 0)

Testing:
Change hud url to localhost on python script, run the python script

Author: clee2000

.github/scripts/update_disabled_issues.py

@@ -9,6 +9,7 @@
 import time
 from datetime import datetime, timezone
 from typing import Any, Dict
+from urllib.error import HTTPError
 from urllib.request import Request, urlopen
 
 from gen_historical_disabled_issue_data import format_info
@@ -23,16 +24,22 @@ def dump_json(data: Dict[str, Any], filename: str):
 
 
 def main() -> None:
-    with urlopen(
-        Request(
-            f"{HUD_URL}/api/flaky-tests/getDisabledTestsAndJobs",
-            headers={"Authorization": os.environ["FLAKY_TEST_BOT_KEY"]},
-        )
-    ) as result:
-        if result.status != 200:
-            raise RuntimeError(f"Failed to fetch data: {result.status} {result.reason}")
-
-        json_data = json.loads(result.read().decode("utf-8"))
+    try:
+        with urlopen(
+            Request(
+                f"{HUD_URL}/api/flaky-tests/getDisabledTestsAndJobs",
+                headers={"Authorization": os.environ["FLAKY_TEST_BOT_KEY"]},
+            )
+        ) as result:
+            if result.status != 200:
+                # Not sure if this is necessary but just in case
+                raise RuntimeError(
+                    f"Failed to fetch data: {result.status} {result.reason}"
+                )
+            json_data = json.loads(result.read().decode("utf-8"))
+    except HTTPError as e:
+        error_body = e.read().decode("utf-8")
+        raise RuntimeError(f"HTTPError: {e.code} {e.reason} - {error_body}")
 
     dump_json(json_data["disabledTests"], "disabled-tests-condensed.json")
     dump_json(json_data["disabledJobs"], "disabled-jobs.json")

torchci/pages/api/flaky-tests/getDisabledTestsAndJobs.ts

@@ -55,8 +55,14 @@ export default async function handler(
 ) {
   const authorization = req.headers.authorization;
   if (authorization === process.env.FLAKY_TEST_BOT_KEY) {
-    const octokit = await getOctokit(PYTORCH, PYTORCH);
-    res.status(200).json(await getDisabledTestsAndJobs(octokit));
+    try {
+      const octokit = await getOctokit(PYTORCH, PYTORCH);
+      res.status(200).json(await getDisabledTestsAndJobs(octokit));
+    } catch (err: any) {
+      res
+        .status(500)
+        .json({ error: err instanceof Error ? err.message : String(err) });
+    }
   } else {
     res.status(403).end();
   }
@@ -97,9 +103,9 @@ async function getIssues(octokit: Octokit, prefix: string) {
   } while (cursor);
 
   if (issues.length !== totalCount) {
-    console.warn(
-      `Expected ${totalCount} issues with prefix "${prefix}", but found ${issues.length}.`
-    );
+    const errString = `Expected ${totalCount} issues with prefix "${prefix}", but found ${issues.length}.`;
+    console.error(errString);
+    throw new Error(errString);
   }
 
   return issues.sort((a, b) => a.url.localeCompare(b.url));

torchci/test/flakyBotTests/getDisabledTestsAndJobs.test.ts

@@ -64,6 +64,29 @@ describe("Get disable/unstable job/test jsons", () => {
     handleScope(scope);
   });
 
+  test("Throw error if result from graphql is inconsistent", async () => {
+    // Number of tests != node list => throw error
+    const scope = nock("https://api.github.com")
+      .post("/graphql", (body) => {
+        return body.query.includes("search");
+      })
+      .reply(200, {
+        data: {
+          search: {
+            issueCount: 15,
+            pageInfo: { hasNextPage: false, endCursor: "" },
+            nodes: [],
+          },
+        },
+      });
+
+    await expect(
+      getDisabledTestsAndJobs.getDisabledTestsAndJobs(octokit)
+    ).rejects.toThrow();
+
+    handleScope(scope);
+  });
+
   test("One test", async () => {
     const issue = genSingleIssueFor(flakyTestA, {});
     const scope = [