fix: avoid queue starvation during retries

Author: frostebite

functions/src/api/retryBuild.ts

@@ -5,7 +5,6 @@ import { CiBuilds } from '../model/ciBuilds';
 import { CiJobs } from '../model/ciJobs';
 import { Ingeminator } from '../logic/buildQueue/ingeminator';
 import { GitHub } from '../service/github';
-import { RepoVersionInfo } from '../model/repoVersionInfo';
 import { Discord } from '../service/discord';
 import { defineSecret } from 'firebase-functions/params';
 
@@ -79,8 +78,7 @@ export const retryBuild = onRequest(
 
       // Schedule new build
       const gitHubClient = await GitHub.init(githubPrivateKey.value(), githubClientSecret.value());
-      const repoVersionInfo = await RepoVersionInfo.getLatest();
-      const scheduler = new Ingeminator(1, gitHubClient, repoVersionInfo);
+      const scheduler = new Ingeminator(1, gitHubClient);
       const scheduledSuccessfully = await scheduler.rescheduleBuild(jobId, job, buildId, build);
 
       // Report result

functions/src/logic/buildQueue/ingeminator.ts

@@ -3,7 +3,6 @@ import { CiBuild, CiBuilds } from '../../model/ciBuilds';
 import { EditorVersionInfo } from '../../model/editorVersionInfo';
 import { Discord } from '../../service/discord';
 import { Octokit } from '@octokit/rest';
-import { RepoVersionInfo } from '../../model/repoVersionInfo';
 import { Scheduler } from './scheduler';
 import admin from 'firebase-admin';
 import Timestamp = admin.firestore.Timestamp;
@@ -15,15 +14,14 @@ import { logger } from 'firebase-functions/v2';
 export class Ingeminator {
   numberToSchedule: number;
   gitHubClient: Octokit;
-  repoVersionInfo: RepoVersionInfo;
+  private scheduledBuilds = 0;
 
-  constructor(numberToSchedule: number, gitHubClient: Octokit, repoVersionInfo: RepoVersionInfo) {
+  constructor(numberToSchedule: number, gitHubClient: Octokit) {
     this.numberToSchedule = numberToSchedule;
     this.gitHubClient = gitHubClient;
-    this.repoVersionInfo = repoVersionInfo;
   }
 
-  async rescheduleFailedJobs(jobs: CiJobQueue) {
+  async rescheduleFailedJobs(jobs: CiJobQueue): Promise<number> {
     if (jobs.length <= 0) {
       throw new Error(
         '[Ingeminator] Expected ingeminator to be called with jobs to retry, none were given.',
@@ -39,6 +37,8 @@ export class Ingeminator {
 
       await this.rescheduleFailedBuildsForJob(job);
     }
+
+    return this.scheduledBuilds;
   }
 
   private async rescheduleFailedBuildsForJob(job: CiJobQueueItem) {
@@ -101,6 +101,7 @@ export class Ingeminator {
       if (!(await this.rescheduleBuild(jobId, jobData, buildId, BuildData))) {
         return;
       }
+      this.scheduledBuilds += 1;
     }
 
     await CiJobs.markJobAsScheduled(jobId);
@@ -117,7 +118,7 @@ export class Ingeminator {
     const { baseOs, targetPlatform } = buildInfo;
 
     // Info from repo
-    const repoVersions = Scheduler.parseRepoVersions(this.repoVersionInfo);
+    const repoVersions = Scheduler.parseRepoVersions(jobData.repoVersionInfo);
     const { repoVersionFull, repoVersionMinor, repoVersionMajor } = repoVersions;
 
     // Send the retry request

functions/src/logic/buildQueue/scheduler.ts

@@ -19,6 +19,7 @@ export class Scheduler {
   private _gitHub: Octokit | undefined;
   private maxConcurrentJobs: number;
   private repoVersionInfo: RepoVersionInfo;
+  private reservedRetrySlots = 0;
 
   private get gitHub(): Octokit {
     // @ts-ignore
@@ -166,6 +167,7 @@ export class Scheduler {
   async ensureThereAreNoFailedJobs(): Promise<boolean> {
     const { maxToleratedFailures, maxExtraJobsForRescheduling } = settings;
     const failingJobs = await CiJobs.getFailingJobsQueue();
+    this.reservedRetrySlots = 0;
 
     if (failingJobs.length >= 1) {
       const openSpots = await this.determineOpenSpots();
@@ -176,15 +178,24 @@ export class Scheduler {
         return false;
       }
 
-      const ingeminator = new Ingeminator(numberToReschedule, this.gitHub, this.repoVersionInfo);
-      await ingeminator.rescheduleFailedJobs(failingJobs);
+      const ingeminator = new Ingeminator(numberToReschedule, this.gitHub);
+      const scheduledRetries = await ingeminator.rescheduleFailedJobs(failingJobs);
+      this.reservedRetrySlots = Math.min(scheduledRetries, openSpots);
+
+      const remainingFreshSlots = openSpots - this.reservedRetrySlots;
+      if (remainingFreshSlots > 0) {
+        await Discord.sendDebug(
+          `[Scheduler] Reserved ${this.reservedRetrySlots} slot(s) for retries and kept ${remainingFreshSlots} slot(s) available for fresh jobs.`,
+        );
+      }
     }
 
-    return failingJobs.length <= maxToleratedFailures;
+    const openSpotsAfterRetries = await this.determineOpenSpotsForFreshJobs();
+    return failingJobs.length <= maxToleratedFailures || openSpotsAfterRetries > 0;
   }
 
   async buildLatestEditorImages(): Promise<boolean> {
-    const openSpots = await this.determineOpenSpots();
+    const openSpots = await this.determineOpenSpotsForFreshJobs();
     if (openSpots <= 0) {
       await Discord.sendDebug('[Scheduler] Not scheduling any new jobs, as the queue is full');
       return false;
@@ -246,4 +257,10 @@ export class Scheduler {
     const openSpots = this.maxConcurrentJobs - currentlyRunningJobs;
     return openSpots <= 0 ? 0 : openSpots;
   }
+
+  private async determineOpenSpotsForFreshJobs(): Promise<number> {
+    const openSpots = await this.determineOpenSpots();
+    const availableSpots = openSpots - this.reservedRetrySlots;
+    return availableSpots <= 0 ? 0 : availableSpots;
+  }
 }