game-ci
diff --git a/‎functions/src/logic/ingestUnityVersions/scrapeVersions.ts‎
Lines changed: 59 additions & 20 deletions b/‎functions/src/logic/ingestUnityVersions/scrapeVersions.ts‎
Lines changed: 59 additions & 20 deletions
diff --git a/‎functions/test/scrapeVersions.test.ts‎
Lines changed: 174 additions & 15 deletions b/‎functions/test/scrapeVersions.test.ts‎
Lines changed: 174 additions & 15 deletions
@@ -31,7 +31,7 @@ const toEditorVersionInfo = (unityVersion: UnityChangesetVersion): EditorVersion
   } as EditorVersionInfo;
 };
 
-export const scrapeLatestOfficialUnityVersion = async (): Promise<EditorVersionInfo | null> => {
+export const scrapeRecentOfficialUnityVersions = async (): Promise<EditorVersionInfo[]> => {
   const response = await fetch(unity_whats_new_url, {
     redirect: 'follow',
     headers: {
@@ -44,21 +44,56 @@ export const scrapeLatestOfficialUnityVersion = async (): Promise<EditorVersionI
   }
 
   const html = await response.text();
-  const versionMatch = /Unity\s+(\d+\.\d+\.\d+f\d+)/.exec(html);
-  const escapedVersion = versionMatch?.[1].replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
-  const changesetMatch = versionMatch
-    ? new RegExp(`unityhub://${escapedVersion}/([a-f0-9]{12})`, 'i').exec(html) ||
-      /Changeset:\s*([a-f0-9]{12})/i.exec(html)
-    : null;
-
-  if (!versionMatch || !changesetMatch) {
-    return null;
+  const versions: UnityChangesetVersion[] = [];
+  const processedVersions = new Set<string>();
+
+  const versionRegex = /(\d+\.\d+\.\d+f\d+)/g;
+  let match;
+  while ((match = versionRegex.exec(html)) !== null) {
+    const version = match[1];
+    if (processedVersions.has(version)) continue;
+    processedVersions.add(version);
+
+    const escapedVersion = version.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
+    let changeset: string | undefined;
+
+    const changesetMatch =
+      new RegExp(`unityhub://${escapedVersion}/([a-f0-9]{12})`, 'i').exec(html) ||
+      new RegExp(`${escapedVersion}[^a-f0-9]*([a-f0-9]{12})`, 'i').exec(html);
+
+    if (changesetMatch?.[1]) {
+      changeset = changesetMatch[1];
+    } else {
+      // As a last resort, search for changeset within a context window near the version
+      // Extract ~500 chars around the version match for local context search
+      const versionIndex = html.indexOf(version);
+      if (versionIndex !== -1) {
+        const contextStart = Math.max(0, versionIndex - 200);
+        const contextEnd = Math.min(html.length, versionIndex + 300);
+        const context = html.substring(contextStart, contextEnd);
+        const contextChangesetMatch = /[Cc]hangeset:\s*([a-f0-9]{12})/i.exec(context);
+        if (contextChangesetMatch?.[1]) {
+          changeset = contextChangesetMatch[1];
+        }
+      }
+    }
+
+    if (changeset) {
+      versions.push({
+        version,
+        changeset,
+      });
+    }
   }
 
-  return toEditorVersionInfo({
-    version: versionMatch[1],
-    changeset: changesetMatch[1],
-  });
+  return versions
+    .map(toEditorVersionInfo)
+    .filter((versionInfo): versionInfo is EditorVersionInfo => versionInfo !== null);
+};
+
+export const scrapeLatestOfficialUnityVersion = async (): Promise<EditorVersionInfo | null> => {
+  const recentVersions = await scrapeRecentOfficialUnityVersions();
+  return recentVersions.length > 0 ? recentVersions[0] : null;
 };
 
 export const scrapeVersions = async (): Promise<EditorVersionInfo[]> => {
@@ -74,7 +109,7 @@ export const scrapeVersions = async (): Promise<EditorVersionInfo[]> => {
       changeset,
     }),
   );
-  const latestOfficialVersion = await scrapeLatestOfficialUnityVersion();
+  const recentOfficialVersions = await scrapeRecentOfficialUnityVersions();
 
   // Merge XLTS versions into main list, avoiding duplicates
   const existingVersions = new Set(unityVersions.map((v) => v.version));
@@ -85,11 +120,15 @@ export const scrapeVersions = async (): Promise<EditorVersionInfo[]> => {
     }
   }
 
-  if (latestOfficialVersion && !existingVersions.has(latestOfficialVersion.version)) {
-    unityVersions.push({
-      version: latestOfficialVersion.version,
-      changeset: latestOfficialVersion.changeSet,
-    });
+  // Merge recent official versions discovered from Unity releases page
+  for (const officialVersion of recentOfficialVersions) {
+    if (!existingVersions.has(officialVersion.version)) {
+      unityVersions.push({
+        version: officialVersion.version,
+        changeset: officialVersion.changeSet,
+      });
+      existingVersions.add(officialVersion.version);
+    }
   }
 
   if (unityVersions?.length > 0) {
 
@@ -2,6 +2,7 @@ import { describe, it, expect, vi, beforeEach } from 'vitest';
 import {
   scrapeLatestOfficialUnityVersion,
   scrapeVersions,
+  scrapeRecentOfficialUnityVersions,
 } from '../src/logic/ingestUnityVersions/scrapeVersions';
 import { SearchMode } from 'unity-changeset';
 import fetch from 'node-fetch';
@@ -47,18 +48,18 @@ describe('scrapeVersions', () => {
       },
       {
         version: '2023.2.10f1',
-        changeset: 'def456ghi789',
+        changeset: '234567ab8cd9',
       },
     ];
 
     const mockXltsVersions = [
       {
         version: '2022.3.21f1', // XLTS versions might have same format as regular versions
-        changeset: 'xyz789uvw123',
+        changeset: '789abcdef012',
       },
       {
         version: '2021.3.25f1',
-        changeset: 'uvw123rst456',
+        changeset: '345cdef67890',
       },
     ];
 
@@ -94,7 +95,7 @@ describe('scrapeVersions', () => {
     expect(result).toContainEqual(
       expect.objectContaining({
         version: '2023.2.10f1',
-        changeSet: 'def456ghi789',
+        changeSet: '234567ab8cd9',
         major: 2023,
         minor: 2,
         patch: '10',
@@ -105,7 +106,7 @@ describe('scrapeVersions', () => {
     expect(result).toContainEqual(
       expect.objectContaining({
         version: '2022.3.21f1',
-        changeSet: 'xyz789uvw123',
+        changeSet: '789abcdef012',
         major: 2022,
         minor: 3,
         patch: '21',
@@ -115,7 +116,7 @@ describe('scrapeVersions', () => {
     expect(result).toContainEqual(
       expect.objectContaining({
         version: '2021.3.25f1',
-        changeSet: 'uvw123rst456',
+        changeSet: '345cdef67890',
         major: 2021,
         minor: 3,
         patch: '25',
@@ -191,11 +192,11 @@ describe('scrapeVersions', () => {
     const mockXltsVersions = [
       {
         version: '2022.3.20f1', // Duplicate version
-        changeset: 'duplicate456',
+        changeset: 'abc123def456',
       },
       {
         version: '2022.3.21f1',
-        changeset: 'xyz789uvw123',
+        changeset: '789abcdef012',
       },
     ];
 
@@ -224,18 +225,18 @@ describe('scrapeVersions', () => {
       },
       {
         version: '2022.3.20a1', // Alpha version - should be excluded
-        changeset: 'def456ghi789',
+        changeset: '234567ab8cd9',
       },
     ];
 
     const mockXltsVersions = [
       {
         version: '2021.3.25f1', // Final version - should be included
-        changeset: 'xyz789uvw123',
+        changeset: '789abcdef012',
       },
       {
         version: '2020.3.15a2', // Alpha version - should be excluded
-        changeset: 'uvw123rst456',
+        changeset: '345cdef67890',
       },
     ];
 
@@ -264,7 +265,7 @@ describe('scrapeVersions', () => {
     expect(result).toContainEqual(
       expect.objectContaining({
         version: '2021.3.25f1',
-        changeSet: 'xyz789uvw123',
+        changeSet: '789abcdef012',
         major: 2021,
         minor: 3,
         patch: '25',
@@ -283,14 +284,14 @@ describe('scrapeVersions', () => {
       },
       {
         version: '5.6.7f1', // Should be excluded (major < 2017)
-        changeset: 'def456ghi789',
+        changeset: '234567ab8cd9',
       },
     ];
 
     const mockXltsVersions = [
       {
         version: '2021.3.25f1', // Should be included
-        changeset: 'xyz789uvw123',
+        changeset: '789abcdef012',
       },
     ];
 
@@ -319,7 +320,7 @@ describe('scrapeVersions', () => {
     expect(result).toContainEqual(
       expect.objectContaining({
         version: '2021.3.25f1',
-        changeSet: 'xyz789uvw123',
+        changeSet: '789abcdef012',
         major: 2021,
         minor: 3,
         patch: '25',
@@ -337,3 +338,161 @@ describe('scrapeVersions', () => {
     await expect(scrapeVersions()).rejects.toThrow('No Unity versions found!');
   });
 });
+
+describe('scrapeRecentOfficialUnityVersions', () => {
+  beforeEach(() => {
+    vi.clearAllMocks();
+  });
+
+  it('should discover multiple recent versions from the releases page', async () => {
+    const html = `
+      <h1>Unity 6000.4.10f1</h1>
+      <a href="unityhub://6000.4.10f1/feeafc12a938">Install</a>
+
+      <h2>Unity 6000.3.17f1</h2>
+      <p>Changeset: abc123def456</p>
+
+      <h2>Unity 6000.2.5f1</h2>
+      <a href="unityhub://6000.2.5f1/deadbeef0123">Download</a>
+    `;
+    mockedFetch.mockResolvedValue({
+      ok: true,
+      status: 200,
+      text: async () => html,
+    } as any);
+
+    const result = await scrapeRecentOfficialUnityVersions();
+
+    expect(result).toHaveLength(3);
+    expect(result.map((v) => v.version)).toContain('6000.4.10f1');
+    expect(result.map((v) => v.version)).toContain('6000.3.17f1');
+    expect(result.map((v) => v.version)).toContain('6000.2.5f1');
+  });
+
+  it('should extract changesets from unityhub:// URLs', async () => {
+    const html = `
+      <a href="unityhub://6000.4.10f1/feeafc12a938">Install</a>
+      <a href="unityhub://6000.3.17f1/abc123def456">Install</a>
+    `;
+    mockedFetch.mockResolvedValue({
+      ok: true,
+      status: 200,
+      text: async () => html,
+    } as any);
+
+    const result = await scrapeRecentOfficialUnityVersions();
+
+    expect(result).toContainEqual(
+      expect.objectContaining({
+        version: '6000.4.10f1',
+        changeSet: 'feeafc12a938',
+      }),
+    );
+    expect(result).toContainEqual(
+      expect.objectContaining({
+        version: '6000.3.17f1',
+        changeSet: 'abc123def456',
+      }),
+    );
+  });
+
+  it('should extract changesets from context near the version', async () => {
+    const html = `
+      <h2>Unity 6000.4.10f1</h2>
+      <p>Changeset: feeafc12a938</p>
+
+      <h2>Unity 6000.3.17f1</h2>
+      <p>Changeset: abc123def456 is the commit hash</p>
+    `;
+    mockedFetch.mockResolvedValue({
+      ok: true,
+      status: 200,
+      text: async () => html,
+    } as any);
+
+    const result = await scrapeRecentOfficialUnityVersions();
+
+    // Both should be found - implementation uses context-window search
+    expect(result.length).toBeGreaterThanOrEqual(2);
+    expect(result.some((v) => v.version === '6000.4.10f1')).toBe(true);
+    expect(result.some((v) => v.version === '6000.3.17f1')).toBe(true);
+  });
+
+  it('should skip versions without valid changesets nearby', async () => {
+    const html = `
+      <h2>Unity 6000.4.10f1</h2>
+      <a href="unityhub://6000.4.10f1/feeafc12a938">Install</a>
+
+      <h2>Unity 6000.2.5f1</h2>
+      <p>This version has no changeset information</p>
+    `;
+    mockedFetch.mockResolvedValue({
+      ok: true,
+      status: 200,
+      text: async () => html,
+    } as any);
+
+    const result = await scrapeRecentOfficialUnityVersions();
+
+    // Only 6000.4.10f1 should be found with a valid changeset
+    expect(result.length).toBeGreaterThanOrEqual(1);
+    expect(result.map((v) => v.version)).toContain('6000.4.10f1');
+    expect(result.map((v) => v.version)).not.toContain('6000.2.5f1');
+  });
+
+  it('should deduplicate versions found multiple times on the page', async () => {
+    const html = `
+      <h2>Unity 6000.4.10f1</h2>
+      <a href="unityhub://6000.4.10f1/feeafc12a938">Install</a>
+
+      <p>Latest version: 6000.4.10f1</p>
+      <a href="unityhub://6000.4.10f1/feeafc12a938">Download</a>
+    `;
+    mockedFetch.mockResolvedValue({
+      ok: true,
+      status: 200,
+      text: async () => html,
+    } as any);
+
+    const result = await scrapeRecentOfficialUnityVersions();
+
+    expect(result).toHaveLength(1);
+    expect(result[0].version).toBe('6000.4.10f1');
+  });
+
+  it('should return empty array if page returns error', async () => {
+    mockedFetch.mockResolvedValue({
+      ok: false,
+      status: 404,
+    } as any);
+
+    await expect(scrapeRecentOfficialUnityVersions()).rejects.toThrow(
+      'Unity release page returned 404',
+    );
+  });
+
+  it('should filter out non-final versions', async () => {
+    const html = `
+      <h2>Unity 6000.4.10f1</h2>
+      <a href="unityhub://6000.4.10f1/feeafc12a938">Install</a>
+
+      <h2>Unity 6000.4.10a1</h2>
+      <a href="unityhub://6000.4.10a1/abc1234567ab">Install</a>
+
+      <h2>Unity 6000.4.9f1</h2>
+      <a href="unityhub://6000.4.9f1/def1234567cd">Install</a>
+    `;
+    mockedFetch.mockResolvedValue({
+      ok: true,
+      status: 200,
+      text: async () => html,
+    } as any);
+
+    const result = await scrapeRecentOfficialUnityVersions();
+
+    expect(result.length).toBeGreaterThanOrEqual(2);
+    expect(result.map((v) => v.version)).toContain('6000.4.10f1');
+    expect(result.map((v) => v.version)).toContain('6000.4.9f1');
+    expect(result.map((v) => v.version)).not.toContain('6000.4.10a1');
+  });
+});