Auto-Recover Stuck Queue Builds And Fix Queue Status API

firestore.indexes.json

@@ -121,6 +121,20 @@
           "order": "DESCENDING"
         }
       ]
+    },
+    {
+      "collectionGroup": "ciBuilds",
+      "queryScope": "COLLECTION",
+      "fields": [
+        {
+          "fieldPath": "status",
+          "order": "ASCENDING"
+        },
+        {
+          "fieldPath": "buildInfo.repoVersion",
+          "order": "ASCENDING"
+        }
+      ]
     }
   ],
   "fieldOverrides": []

functions/src/api/queueStatus.ts

@@ -4,8 +4,20 @@ import { CiJobs } from '../model/ciJobs';
 import { CiBuilds } from '../model/ciBuilds';
 
 export const queueStatus = onRequest(async (req: Request, res: Response) => {
+  res.set('Access-Control-Allow-Origin', '*');
+  res.set('Access-Control-Allow-Methods', 'GET, OPTIONS');
+  res.set('Access-Control-Allow-Headers', 'Content-Type, Authorization');
+
+  if (req.method === 'OPTIONS') {
+    res.status(204).send();
+    return;
+  }
+
+  const repoVersion = typeof req.query.repoVersion === 'string' ? req.query.repoVersion : '';
   const jobs = await CiJobs.getAll();
-  const builds = await CiBuilds.getAll();
+  const builds = repoVersion
+    ? await CiBuilds.getAllForRepoVersion(repoVersion)
+    : await CiBuilds.getAll();
 
-  res.status(200).send({ jobs, builds });
+  res.status(200).send({ jobs, builds, repoVersion: repoVersion || null });
 });

functions/src/api/retryBuild.ts

@@ -5,7 +5,6 @@ import { CiBuilds } from '../model/ciBuilds';
 import { CiJobs } from '../model/ciJobs';
 import { Ingeminator } from '../logic/buildQueue/ingeminator';
 import { GitHub } from '../service/github';
-import { RepoVersionInfo } from '../model/repoVersionInfo';
 import { Discord } from '../service/discord';
 import { defineSecret } from 'firebase-functions/params';
 
@@ -79,8 +78,7 @@ export const retryBuild = onRequest(
 
       // Schedule new build
       const gitHubClient = await GitHub.init(githubPrivateKey.value(), githubClientSecret.value());
-      const repoVersionInfo = await RepoVersionInfo.getLatest();
-      const scheduler = new Ingeminator(1, gitHubClient, repoVersionInfo);
+      const scheduler = new Ingeminator(1, gitHubClient);
       const scheduledSuccessfully = await scheduler.rescheduleBuild(jobId, job, buildId, build);
 
       // Report result

functions/src/logic/buildQueue/cleaner.ts

@@ -14,6 +14,81 @@ export class Cleaner {
     await this.healFailedBuildsAlreadyOnDockerHub();
   }
 
+  /**
+   * A build that has been auto-recovered this many times will not be reset
+   * again. Beyond this point the build is almost certainly fundamentally
+   * broken (e.g. an unbuildable editor version on a given base OS); further
+   * resets only burn GitHub Actions minutes and DockerHub API quota.
+   */
+  static readonly maxRecoveryAttempts: number = 2;
+
+  /**
+   * Automatically recover maxed-out failed builds for the latest repo version.
+   * If the image is already on DockerHub, mark it as published.
+   * Otherwise reset its failure count so the Ingeminator can retry it again.
+   *
+   * Per run we process at most `maxBuildsProcessedPerRun` builds and we cap
+   * the number of times any single build can be auto-reset. Builds that
+   * exceed `maxRecoveryAttempts` are left alone and an alert is sent so a
+   * maintainer can investigate.
+   */
+  public static async recoverMaxedOutFailedBuilds(repoVersion: string): Promise<string[]> {
+    const results: string[] = [];
+    const maxedBuilds = await CiBuilds.getMaxedOutFailedBuildsForRepoVersion(repoVersion);
+
+    let processed = 0;
+    for (const { id: buildId, data: build } of maxedBuilds) {
+      if (processed >= this.maxBuildsProcessedPerRun) {
+        results.push(`deferred:${buildId}`);
+        continue;
+      }
+
+      const { relatedJobId: jobId, imageType, buildInfo, meta } = build;
+      const { baseOs } = buildInfo;
+      const tag = buildId.replace(new RegExp(`^${imageType}-`), '');
+      const recoveryCount = meta?.recoveryCount ?? 0;
+
+      processed += 1;
+
+      const response = await Dockerhub.fetchImageData(imageType, tag);
+      if (response) {
+        const digest = response.digest || '';
+        await Discord.sendDebug(
+          `[Cleaner] Maxed-out build "${tag}" already exists on DockerHub. Marking as published.`,
+        );
+        await CiBuilds.markBuildAsPublished(buildId, jobId, {
+          digest,
+          specificTag: `${baseOs}-${repoVersion}`,
+          friendlyTag: repoVersion.replace(/\.\d+$/, ''),
+          imageName: Dockerhub.getImageName(imageType),
+          imageRepo: Dockerhub.getRepositoryBaseName(),
+        });
+        results.push(`published:${buildId}`);
+        continue;
+      }
+
+      if (recoveryCount >= this.maxRecoveryAttempts) {
+        await Discord.sendAlert(
+          `[Cleaner] Build "${tag}" has been auto-recovered ${recoveryCount} time(s) and is still failing. ` +
+            `Leaving it at max retries - manual investigation required.`,
+        );
+        results.push(`exhausted:${buildId}`);
+        continue;
+      }
+
+      await CiBuilds.resetFailureCount(buildId);
+      await CiBuilds.incrementRecoveryCount(buildId);
+      await Discord.sendAlert(
+        `[Cleaner] Reset failure count for maxed-out build "${tag}" (recovery attempt ${
+          recoveryCount + 1
+        }/${this.maxRecoveryAttempts}).`,
+      );
+      results.push(`reset:${buildId}`);
+    }
+
+    return results;
+  }
+
   /**
    * Manual cleanup for maintainers. Processes all stuck builds without the
    * 6-hour wait and without the per-run build limit.

functions/src/logic/buildQueue/ingeminator.ts

@@ -3,7 +3,6 @@ import { CiBuild, CiBuilds } from '../../model/ciBuilds';
 import { EditorVersionInfo } from '../../model/editorVersionInfo';
 import { Discord } from '../../service/discord';
 import { Octokit } from '@octokit/rest';
-import { RepoVersionInfo } from '../../model/repoVersionInfo';
 import { Scheduler } from './scheduler';
 import admin from 'firebase-admin';
 import Timestamp = admin.firestore.Timestamp;
@@ -15,12 +14,10 @@ import { logger } from 'firebase-functions/v2';
 export class Ingeminator {
   numberToSchedule: number;
   gitHubClient: Octokit;
-  repoVersionInfo: RepoVersionInfo;
 
-  constructor(numberToSchedule: number, gitHubClient: Octokit, repoVersionInfo: RepoVersionInfo) {
+  constructor(numberToSchedule: number, gitHubClient: Octokit) {
     this.numberToSchedule = numberToSchedule;
     this.gitHubClient = gitHubClient;
-    this.repoVersionInfo = repoVersionInfo;
   }
 
   async rescheduleFailedJobs(jobs: CiJobQueue) {
@@ -65,7 +62,10 @@ export class Ingeminator {
       // Max retries check
       const { maxFailuresPerBuild } = settings;
       const { lastBuildFailure, failureCount } = build.data.meta;
-      const lastFailure = lastBuildFailure as Timestamp;
+      // `lastBuildFailure` can be null after Cleaner.recoverMaxedOutFailedBuilds
+      // or the admin reset endpoint clears it. Fall back to the epoch so the
+      // backoff window is always considered elapsed.
+      const lastFailure = lastBuildFailure ?? Timestamp.fromMillis(0);
       if (failureCount >= maxFailuresPerBuild) {
         // Log warning
         const retries: number = maxFailuresPerBuild - 1;
@@ -117,7 +117,7 @@ export class Ingeminator {
     const { baseOs, targetPlatform } = buildInfo;
 
     // Info from repo
-    const repoVersions = Scheduler.parseRepoVersions(this.repoVersionInfo);
+    const repoVersions = Scheduler.parseRepoVersions(jobData.repoVersionInfo);
     const { repoVersionFull, repoVersionMinor, repoVersionMajor } = repoVersions;
 
     // Send the retry request

functions/src/logic/buildQueue/scheduleBuildsFromTheQueue.ts

@@ -2,6 +2,7 @@ import { RepoVersionInfo } from '../../model/repoVersionInfo';
 import { Scheduler } from './scheduler';
 import { Discord } from '../../service/discord';
 import { CiJobs } from '../../model/ciJobs';
+import { Cleaner } from './cleaner';
 
 /**
  * When a new Unity version gets ingested:
@@ -30,6 +31,13 @@ export const scheduleBuildsFromTheQueue = async (
     );
   }
 
+  const recoveredBuilds = await Cleaner.recoverMaxedOutFailedBuilds(repoVersionInfo.version);
+  if (recoveredBuilds.length >= 1) {
+    await Discord.sendDebug(
+      `[Build queue] Recovered ${recoveredBuilds.length} maxed-out failed build(s) for repo version ${repoVersionInfo.version}.`,
+    );
+  }
+
   const scheduler = await new Scheduler(repoVersionInfo).init(githubPrivateKey, githubClientSecret);
 
   const testVersion = '0.1.0';

functions/src/logic/buildQueue/scheduler.ts

@@ -176,7 +176,7 @@ export class Scheduler {
         return false;
       }
 
-      const ingeminator = new Ingeminator(numberToReschedule, this.gitHub, this.repoVersionInfo);
+      const ingeminator = new Ingeminator(numberToReschedule, this.gitHub);
       await ingeminator.rescheduleFailedJobs(failingJobs);
     }
 

functions/src/model/ciBuilds.ts

@@ -36,6 +36,7 @@ interface MetaData {
   failureCount: number;
   lastBuildFailure: Timestamp | null;
   publishedDate: Timestamp | null;
+  recoveryCount?: number;
 }
 
 export interface CiBuild {
@@ -70,6 +71,15 @@ export class CiBuilds {
     return snapshot.docs.map((doc) => doc.data()) as CiBuild[];
   };
 
+  public static getAllForRepoVersion = async (repoVersion: string): Promise<CiBuild[]> => {
+    const snapshot = await db
+      .collection(CiBuilds.collection)
+      .where('buildInfo.repoVersion', '==', repoVersion)
+      .get();
+
+    return snapshot.docs.map((doc) => doc.data()) as CiBuild[];
+  };
+
   public static get = async (buildId: string): Promise<CiBuild | null> => {
     const snapshot = await db.doc(`${CiBuilds.collection}/${buildId}`).get();
 
@@ -223,9 +233,19 @@ export class CiBuilds {
 
   public static resetFailureCount = async (buildId: string): Promise<void> => {
     const build = db.collection(CiBuilds.collection).doc(buildId);
+    // Use an epoch sentinel rather than null so the Ingeminator's backoff math
+    // (lastBuildFailure.toMillis() + backoffMs) never reads null at runtime.
     await build.update({
       'meta.failureCount': 0,
-      'meta.lastBuildFailure': null,
+      'meta.lastBuildFailure': Timestamp.fromMillis(0),
+      modifiedDate: Timestamp.now(),
+    });
+  };
+
+  public static incrementRecoveryCount = async (buildId: string): Promise<void> => {
+    const build = db.collection(CiBuilds.collection).doc(buildId);
+    await build.update({
+      'meta.recoveryCount': FieldValue.increment(1),
       modifiedDate: Timestamp.now(),
     });
   };
@@ -254,6 +274,26 @@ export class CiBuilds {
       }));
   };
 
+  public static getMaxedOutFailedBuildsForRepoVersion = async (
+    repoVersion: string,
+  ): Promise<CiBuildQueue> => {
+    const snapshot = await db
+      .collection(CiBuilds.collection)
+      .where('status', '==', 'failed')
+      .where('buildInfo.repoVersion', '==', repoVersion)
+      .get();
+
+    return snapshot.docs
+      .filter((doc) => {
+        const data = doc.data() as CiBuild;
+        return (data.meta?.failureCount ?? 0) >= settings.maxFailuresPerBuild;
+      })
+      .map((doc) => ({
+        id: doc.id,
+        data: doc.data() as CiBuild,
+      }));
+  };
+
   public static haveAllBuildsForJobBeenPublished = async (jobId: string): Promise<boolean> => {
     const snapshot = await db
       .collection(CiBuilds.collection)