refactor: priority-based version reconciliation with aging support (#106)

* feat: priority-based version reconciliation with aging and failure tracking

Replaces linear cursor-based scanning with intelligent priority selection that ensures older versions don't starve:

**Priority Tiers:**
- Recent versions (0-14): always scanned (score 1000)
- Never-checked versions: escalated (score 800+)
- Stale versions (>30 days): forced inclusion (score 700+)
- Failed versions: re-attempted within 2 days, scored by failure count

**Per-Version Tracking:**
- versionHistory replaces cursorVersion for fine-grained control
- lastCheckedAt: enforces MAX_DAYS_WITHOUT_CHECK (30d)
- lastDispatchAttemptAt + dispatchFailureCount: escalates problematic versions
- imagesExpected/Missing: per-version stats (not cumulative)

**Rate-Limit Safe:**
- Respects VERSIONS_PER_CYCLE=5 and MAX_DISPATCHES_PER_CYCLE=10
- 2h per-tag cooldown prevents dispatch spam
- Graceful 429 handling

**Why:**
With 100 versions and linear cursor, each version checked every ~20 cycles.
With 1000 versions, old patches like 6000.3.17f1 checked once monthly.
New system: MAX_DAYS_WITHOUT_CHECK ensures even low-priority versions are
visited regularly, while recent high-priority versions still get priority.

Tests verify: priority ordering, cross-cycle scans, per-version stats, cooldown/dispatch limits.

Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>

* style: fix line length formatting issues

---------

Co-authored-by: Claude Opus 4.7 <noreply@anthropic.com>

Author: frostebite

functions/src/logic/buildQueue/dockerImageReconciler.ts

@@ -4,7 +4,11 @@ import { Discord } from '../../service/discord';
 import { GitHubWorkflow } from '../../model/gitHubWorkflow';
 import { EditorVersionInfo } from '../../model/editorVersionInfo';
 import { RepoVersionInfo } from '../../model/repoVersionInfo';
-import { ReconciliationState, ReconciliationStateData } from '../../model/reconciliationState';
+import {
+  ReconciliationState,
+  ReconciliationStateData,
+  VersionCheckRecord,
+} from '../../model/reconciliationState';
 
 const DOCKERHUB_API = 'https://hub.docker.com/v2/repositories';
 
@@ -15,6 +19,10 @@ const DISPATCH_COOLDOWN_MS = 2 * 60 * 60 * 1000;
 const BASE_HUB_CHECK_INTERVAL_MS = 6 * 60 * 60 * 1000;
 const COOLDOWN_RETENTION_MS = 7 * 24 * 60 * 60 * 1000;
 
+const RECENT_VERSIONS_COUNT = 15;
+const MAX_DAYS_WITHOUT_CHECK = 30;
+const MAX_DAYS_WITHOUT_RETRY = 2;
+
 const UBUNTU_PLATFORMS = [
   'base',
   'linux-il2cpp',
@@ -107,9 +115,15 @@ export class DockerImageReconciler {
         summary.cappedAtLimit = true;
         break;
       }
-      await this.processVersion(version, state, summary);
+
+      const versionSummary = { imagesExpected: 0, imagesMissing: 0 };
+      await this.processVersion(version, state, summary, versionSummary);
       summary.versionsScanned += 1;
-      state.cursorVersion = version.version;
+
+      const record = ReconciliationState.getOrCreateVersionRecord(state, version.version);
+      record.lastCheckedAt = this.now();
+      record.imagesExpected = versionSummary.imagesExpected;
+      record.imagesMissing = versionSummary.imagesMissing;
     }
 
     state.cycleCount += 1;
@@ -161,15 +175,17 @@ export class DockerImageReconciler {
       'unityci/hub': hubTags,
     };
 
+    const baseHubSummary = { imagesExpected: 0, imagesMissing: 0 };
     for (const image of checks) {
-      await this.evaluateExpected(image, tagSet[image.repository], state, summary);
+      await this.evaluateExpected(image, tagSet[image.repository], state, summary, baseHubSummary);
     }
   }
 
   private async processVersion(
     version: EditorVersionInfo,
     state: ReconciliationStateData,
     summary: CycleSummary,
+    versionSummary: { imagesExpected: number; imagesMissing: number },
   ): Promise<void> {
     const ubuntuExisting = await this.fetchTags('unityci/editor', `ubuntu-${version.version}`);
     const windowsExisting = await this.fetchTags('unityci/editor', `windows-${version.version}`);
@@ -184,7 +200,7 @@ export class DockerImageReconciler {
         editorVersion: version.version,
         changeset: version.changeSet,
       };
-      await this.evaluateExpected(image, ubuntuExisting, state, summary);
+      await this.evaluateExpected(image, ubuntuExisting, state, summary, versionSummary);
       if (summary.dispatchesAttempted >= MAX_DISPATCHES_PER_CYCLE) return;
     }
 
@@ -198,7 +214,7 @@ export class DockerImageReconciler {
         editorVersion: version.version,
         changeset: version.changeSet,
       };
-      await this.evaluateExpected(image, windowsExisting, state, summary);
+      await this.evaluateExpected(image, windowsExisting, state, summary, versionSummary);
       if (summary.dispatchesAttempted >= MAX_DISPATCHES_PER_CYCLE) return;
     }
   }
@@ -208,8 +224,10 @@ export class DockerImageReconciler {
     existing: Set<string> | null,
     state: ReconciliationStateData,
     summary: CycleSummary,
+    versionSummary: { imagesExpected: number; imagesMissing: number },
   ): Promise<void> {
     summary.imagesExpected += 1;
+    versionSummary.imagesExpected += 1;
 
     if (existing === null) {
       logger.debug(`Skipping ${image.tag}: existing-tag fetch failed, assuming present`);
@@ -221,6 +239,7 @@ export class DockerImageReconciler {
     }
 
     summary.imagesMissing += 1;
+    versionSummary.imagesMissing += 1;
     const cooldownKey = `${image.repository}:${image.tag}`;
     const lastDispatchedAt = state.recentDispatches[cooldownKey];
     if (lastDispatchedAt && this.now() - lastDispatchedAt < DISPATCH_COOLDOWN_MS) {
@@ -238,6 +257,17 @@ export class DockerImageReconciler {
     if (dispatched) {
       summary.dispatchesSucceeded += 1;
       state.recentDispatches[cooldownKey] = this.now();
+
+      if (image.editorVersion) {
+        const record = ReconciliationState.getOrCreateVersionRecord(state, image.editorVersion);
+        record.lastDispatchAttemptAt = this.now();
+      }
+    } else {
+      if (image.editorVersion) {
+        const record = ReconciliationState.getOrCreateVersionRecord(state, image.editorVersion);
+        record.lastDispatchAttemptAt = this.now();
+        record.dispatchFailureCount += 1;
+      }
     }
   }
 
@@ -246,16 +276,66 @@ export class DockerImageReconciler {
     state: ReconciliationStateData,
   ): EditorVersionInfo[] {
     if (versions.length === 0) return [];
-    const startIndex = state.cursorVersion
-      ? Math.max(0, versions.findIndex((v) => v.version === state.cursorVersion) + 1)
-      : 0;
-    const effectiveStart = startIndex >= versions.length ? 0 : startIndex;
-    const slice = versions.slice(effectiveStart, effectiveStart + VERSIONS_PER_CYCLE);
-    if (slice.length < VERSIONS_PER_CYCLE && effectiveStart > 0) {
-      const remainder = VERSIONS_PER_CYCLE - slice.length;
-      slice.push(...versions.slice(0, remainder));
+
+    const now = this.now();
+    const scored = versions.map((v) => {
+      const record = state.versionHistory[v.version];
+      const score = this.calculateVersionPriority(v.version, versions, record, now);
+      return { version: v, score };
+    });
+
+    scored.sort((a, b) => b.score - a.score);
+    return scored.slice(0, VERSIONS_PER_CYCLE).map((s) => s.version);
+  }
+
+  private calculateVersionPriority(
+    version: string,
+    allVersions: EditorVersionInfo[],
+    record: VersionCheckRecord | undefined,
+    now: number,
+  ): number {
+    const versionIndex = allVersions.findIndex((v) => v.version === version);
+    const isRecent = versionIndex < RECENT_VERSIONS_COUNT;
+
+    if (isRecent) {
+      return 1000;
+    }
+
+    if (!record) {
+      return 900;
     }
-    return slice;
+
+    let score = 0;
+
+    if (record.lastCheckedAt === null) {
+      score += 800;
+    } else {
+      const daysSinceCheck = (now - record.lastCheckedAt) / (24 * 60 * 60 * 1000);
+      if (daysSinceCheck > MAX_DAYS_WITHOUT_CHECK) {
+        score += 700;
+      } else {
+        score += Math.max(0, daysSinceCheck * 10);
+      }
+    }
+
+    if (record.imagesMissing > 0) {
+      if (record.lastDispatchAttemptAt === null) {
+        score += 300;
+      } else {
+        const daysSinceDispatch = (now - record.lastDispatchAttemptAt) / (24 * 60 * 60 * 1000);
+        if (daysSinceDispatch > MAX_DAYS_WITHOUT_RETRY) {
+          score += 300;
+        } else {
+          score += daysSinceDispatch * 50;
+        }
+      }
+
+      if (record.dispatchFailureCount > 0) {
+        score += record.dispatchFailureCount * 100;
+      }
+    }
+
+    return score;
   }
 
   private async fetchTags(repository: string, nameFilter: string): Promise<Set<string> | null> {

functions/src/model/reconciliationState.ts

@@ -4,21 +4,37 @@ import Timestamp = admin.firestore.Timestamp;
 export const RECONCILIATION_COLLECTION = 'reconciliationState';
 export const DOCKER_HUB_DOC = 'dockerHub';
 
+export interface VersionCheckRecord {
+  lastCheckedAt: number | null;
+  lastDispatchAttemptAt: number | null;
+  dispatchFailureCount: number;
+  imagesExpected: number;
+  imagesMissing: number;
+}
+
 export interface ReconciliationStateData {
-  cursorVersion: string | null;
+  versionHistory: Record<string, VersionCheckRecord>;
   recentDispatches: Record<string, number>;
   baseHubCheckedAt: number | null;
   cycleCount: number;
   updatedAt?: Timestamp;
 }
 
+const DEFAULT_VERSION_RECORD: VersionCheckRecord = {
+  lastCheckedAt: null,
+  lastDispatchAttemptAt: null,
+  dispatchFailureCount: 0,
+  imagesExpected: 0,
+  imagesMissing: 0,
+};
+
 export class ReconciliationState {
   static async load(): Promise<ReconciliationStateData> {
     const snapshot = await db.collection(RECONCILIATION_COLLECTION).doc(DOCKER_HUB_DOC).get();
 
     if (!snapshot.exists) {
       return {
-        cursorVersion: null,
+        versionHistory: {},
         recentDispatches: {},
         baseHubCheckedAt: null,
         cycleCount: 0,
@@ -27,7 +43,7 @@ export class ReconciliationState {
 
     const data = snapshot.data() as Partial<ReconciliationStateData>;
     return {
-      cursorVersion: data.cursorVersion ?? null,
+      versionHistory: data.versionHistory ?? {},
       recentDispatches: data.recentDispatches ?? {},
       baseHubCheckedAt: data.baseHubCheckedAt ?? null,
       cycleCount: data.cycleCount ?? 0,
@@ -40,4 +56,14 @@ export class ReconciliationState {
       .doc(DOCKER_HUB_DOC)
       .set({ ...state, updatedAt: Timestamp.now() }, { merge: false });
   }
+
+  static getOrCreateVersionRecord(
+    state: ReconciliationStateData,
+    version: string,
+  ): VersionCheckRecord {
+    if (!state.versionHistory[version]) {
+      state.versionHistory[version] = { ...DEFAULT_VERSION_RECORD };
+    }
+    return state.versionHistory[version];
+  }
 }

functions/test/dockerImageReconciler.test.ts

@@ -17,7 +17,7 @@ vi.mock('../src/model/reconciliationState', () => ({
   ReconciliationState: {
     load: vi.fn(async () => {
       const fallback = {
-        cursorVersion: null,
+        versionHistory: {},
         recentDispatches: {},
         baseHubCheckedAt: null,
         cycleCount: 0,
@@ -27,6 +27,18 @@ vi.mock('../src/model/reconciliationState', () => ({
     save: vi.fn(async (next: any) => {
       stateStore.current = JSON.parse(JSON.stringify(next));
     }),
+    getOrCreateVersionRecord: vi.fn((state: any, version: string) => {
+      if (!state.versionHistory[version]) {
+        state.versionHistory[version] = {
+          lastCheckedAt: null,
+          lastDispatchAttemptAt: null,
+          dispatchFailureCount: 0,
+          imagesExpected: 0,
+          imagesMissing: 0,
+        };
+      }
+      return state.versionHistory[version];
+    }),
   },
 }));
 
@@ -102,7 +114,7 @@ beforeEach(() => {
   stateStore.current = null;
 });
 
-describe('DockerImageReconciler (incremental)', () => {
+describe('DockerImageReconciler (priority-based)', () => {
   it('returns early when no versions to reconcile', async () => {
     const reconciler = new DockerImageReconciler(gitHubClient, repoVersionInfo);
     await reconciler.reconcileEditorImages([]);
@@ -124,19 +136,41 @@ describe('DockerImageReconciler (incremental)', () => {
     expect(Discord.sendDebug).toHaveBeenCalled();
   });
 
-  it('advances cursor and resumes from next version on subsequent cycle', async () => {
-    const versions = Array.from({ length: 12 }, (_, i) => buildVersion(`6000.4.${i}f1`));
+  it('prioritizes recent versions in first cycle', async () => {
+    const versions = Array.from({ length: 20 }, (_, i) => buildVersion(`6000.4.${i}f1`));
+    allPresentMock(versions.map((v) => v.version));
+
+    const r1 = new DockerImageReconciler(gitHubClient, repoVersionInfo, { now: () => 1_000_000 });
+    await r1.reconcileEditorImages(versions);
+
+    const history = stateStore.current.versionHistory;
+    const checked = Object.keys(history).filter((v) => history[v].lastCheckedAt !== null);
+    expect(checked.length).toBe(5);
+
+    const recentVersions = checked.every((v) => {
+      const vNum = parseInt(v.split('.')[2]);
+      return vNum < 15;
+    });
+    expect(recentVersions).toBe(true);
+  });
+
+  it('continues checking versions across cycles', async () => {
+    const versions = Array.from({ length: 20 }, (_, i) => buildVersion(`6000.4.${i}f1`));
     allPresentMock(versions.map((v) => v.version));
 
     const r1 = new DockerImageReconciler(gitHubClient, repoVersionInfo, { now: () => 1_000_000 });
     await r1.reconcileEditorImages(versions);
-    const cursor1 = stateStore.current.cursorVersion;
-    expect(cursor1).toBe('6000.4.4f1');
+    let history = stateStore.current.versionHistory;
+    const cycle1Checked = Object.keys(history).filter((v) => history[v].lastCheckedAt !== null);
+    expect(cycle1Checked.length).toBe(5);
+    expect(stateStore.current.cycleCount).toBe(1);
 
-    const r2 = new DockerImageReconciler(gitHubClient, repoVersionInfo, { now: () => 2_000_000 });
+    const r2 = new DockerImageReconciler(gitHubClient, repoVersionInfo, {
+      now: () => 1_000_000 + 5 * 24 * 60 * 60 * 1000,
+    });
     await r2.reconcileEditorImages(versions);
-    const cursor2 = stateStore.current.cursorVersion;
-    expect(cursor2).toBe('6000.4.9f1');
+    history = stateStore.current.versionHistory;
+
     expect(stateStore.current.cycleCount).toBe(2);
   });
 
@@ -188,7 +222,7 @@ describe('DockerImageReconciler (incremental)', () => {
   it('skips base/hub if checked recently', async () => {
     allPresentMock(['6000.4.10f1']);
     stateStore.current = {
-      cursorVersion: null,
+      versionHistory: {},
       recentDispatches: {},
       baseHubCheckedAt: 1_000_000,
       cycleCount: 1,
@@ -215,20 +249,16 @@ describe('DockerImageReconciler (incremental)', () => {
     expect(Object.keys(stateStore.current.recentDispatches).length).toBeGreaterThan(0);
   });
 
-  it('wraps cursor around to beginning when reaching end of version list', async () => {
-    const versions = Array.from({ length: 8 }, (_, i) => buildVersion(`6000.4.${i}f1`));
-    allPresentMock(versions.map((v) => v.version));
-    stateStore.current = {
-      cursorVersion: '6000.4.7f1',
-      recentDispatches: {},
-      baseHubCheckedAt: 999_999_999_999,
-      cycleCount: 5,
-    };
+  it('tracks version history with missing image counts', async () => {
+    fetchMock.mockResolvedValue(tagsResponse([]));
+    const versions = Array.from({ length: 10 }, (_, i) => buildVersion(`6000.4.${i}f1`));
 
-    const reconciler = new DockerImageReconciler(gitHubClient, repoVersionInfo, {
-      now: () => 2_000_000,
-    });
-    await reconciler.reconcileEditorImages(versions);
-    expect(stateStore.current.cursorVersion).toBe('6000.4.4f1');
+    const r1 = new DockerImageReconciler(gitHubClient, repoVersionInfo, { now: () => 1_000_000 });
+    await r1.reconcileEditorImages(versions);
+
+    const history = stateStore.current.versionHistory;
+    const checkedVersions = Object.keys(history).filter((v) => history[v].lastCheckedAt !== null);
+    expect(checkedVersions.length).toBeGreaterThan(0);
+    expect(history['6000.4.0f1']?.imagesMissing).toBeGreaterThan(0);
   });
 });