fix: mark parent job failed when build times out or is manually cleared (#98)

When cleanUpBuildsThatDidntReportBack marks a build as failed after the
6-hour GitHub Actions timeout, or manualCleanUp marks a build as failed,
the parent job was left in its current state (typically inProgress).

The Ingeminator only processes jobs in "failed" status, so these builds
were never retried. Worse, with maxConcurrentJobs=9 the ghost-inProgress
jobs consumed all queue capacity, blocking all 53 created jobs from
being scheduled.

Three-part fix:
- cleanUpBuildsThatDidntReportBack: call markFailureForJob after marking
  build as failed so the Ingeminator picks it up next cycle
- manualCleanUp: same fix for the manual path
- transitionStaleInProgressJobsToFailed: new cleaner step that detects
  active jobs where all builds have settled (none are started) and
  transitions them to failed, recovering capacity for builds already stuck

Adds CiBuilds.hasAnyStartedBuildsForJob to support the new step.

Co-authored-by: Claude Sonnet 4.6 <noreply@anthropic.com>

Author: frostebite

functions/src/logic/buildQueue/cleaner.ts

@@ -16,6 +16,7 @@ export class Cleaner {
   public static async cleanUp(latestRepoVersion: string) {
     this.buildsProcessed = 0;
     await this.requeueActiveJobsWithoutBuilds();
+    await this.transitionStaleInProgressJobsToFailed();
     await this.recoverMaxedOutFailedBuilds(latestRepoVersion);
     await this.reconcileStartedBuildsThatMayHavePublished();
     await this.cleanUpBuildsThatDidntReportBack();
@@ -60,6 +61,40 @@ export class Cleaner {
     }
   }
 
+  /**
+   * An inProgress job whose builds have all finished (none are "started") is
+   * stuck — the job will never self-transition because only the build-report
+   * path calls markFailureForJob. Move these to "failed" so the Ingeminator
+   * can schedule retries and free up queue capacity for fresh jobs.
+   */
+  private static async transitionStaleInProgressJobsToFailed() {
+    const activeJobs = await CiJobs.getActiveJobs();
+    const staleThresholdMs = this.activeJobWithoutBuildsAfterMinutes * 60 * 1000;
+
+    for (const activeJob of activeJobs) {
+      if (this.buildsProcessed >= this.maxBuildsProcessedPerRun) return;
+
+      const { id: jobId, data: job } = activeJob;
+      const lastTouchedSeconds = job.modifiedDate?.seconds || job.addedDate?.seconds;
+      if (!lastTouchedSeconds) continue;
+
+      const ageMs = Date.now() - lastTouchedSeconds * 1000;
+      if (ageMs < staleThresholdMs) continue;
+
+      const hasBuilds = await CiBuilds.hasAnyBuildsForJob(jobId);
+      if (!hasBuilds) continue; // no builds at all — handled by requeueActiveJobsWithoutBuilds
+
+      const hasStartedBuilds = await CiBuilds.hasAnyStartedBuildsForJob(jobId);
+      if (hasStartedBuilds) continue; // at least one build is still running
+
+      this.buildsProcessed += 1;
+      await CiJobs.markFailureForJob(jobId);
+      await Discord.sendAlert(
+        `[Cleaner] Transitioned ${job.status} job "${jobId}" to failed: all its builds have settled with no started builds remaining.`,
+      );
+    }
+  }
+
   /**
    * Automatically recover maxed-out failed builds for the latest repo version.
    * If the image is already on DockerHub, mark it as published.
@@ -161,6 +196,7 @@ export class Cleaner {
         await CiBuilds.markBuildAsFailed(buildId, {
           reason: `[ManualCleanup] Build never reported back and image not found on DockerHub.`,
         });
+        await CiJobs.markFailureForJob(jobId);
         continue;
       }
 
@@ -315,6 +351,7 @@ export class Cleaner {
           await CiBuilds.markBuildAsFailed(buildId, {
             reason: markAsFailedMessage,
           });
+          await CiJobs.markFailureForJob(jobId);
 
           continue;
         }

functions/src/model/ciBuilds.ts

@@ -126,6 +126,17 @@ export class CiBuilds {
     return snapshot.docs.length > 0;
   };
 
+  public static hasAnyStartedBuildsForJob = async (jobId: string): Promise<boolean> => {
+    const snapshot = await db
+      .collection(CiBuilds.collection)
+      .where('relatedJobId', '==', jobId)
+      .where('status', '==', 'started')
+      .limit(1)
+      .get();
+
+    return snapshot.docs.length > 0;
+  };
+
   /**
    * Registers a new build or handles duplicate dispatches gracefully.
    * Returns the existing status if the build is already in progress or published,