diff --git a/functions/src/logic/ingestUnityVersions/scrapeVersions.ts b/functions/src/logic/ingestUnityVersions/scrapeVersions.ts index 6251ead..6b03dd0 100644 --- a/functions/src/logic/ingestUnityVersions/scrapeVersions.ts +++ b/functions/src/logic/ingestUnityVersions/scrapeVersions.ts @@ -31,7 +31,7 @@ const toEditorVersionInfo = (unityVersion: UnityChangesetVersion): EditorVersion } as EditorVersionInfo; }; -export const scrapeLatestOfficialUnityVersion = async (): Promise => { +export const scrapeRecentOfficialUnityVersions = async (): Promise => { const response = await fetch(unity_whats_new_url, { redirect: 'follow', headers: { @@ -44,21 +44,56 @@ export const scrapeLatestOfficialUnityVersion = async (): Promise(); + + const versionRegex = /(\d+\.\d+\.\d+f\d+)/g; + let match; + while ((match = versionRegex.exec(html)) !== null) { + const version = match[1]; + if (processedVersions.has(version)) continue; + processedVersions.add(version); + + const escapedVersion = version.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); + let changeset: string | undefined; + + const changesetMatch = + new RegExp(`unityhub://${escapedVersion}/([a-f0-9]{12})`, 'i').exec(html) || + new RegExp(`${escapedVersion}[^a-f0-9]*([a-f0-9]{12})`, 'i').exec(html); + + if (changesetMatch?.[1]) { + changeset = changesetMatch[1]; + } else { + // As a last resort, search for changeset within a context window near the version + // Extract ~500 chars around the version match for local context search + const versionIndex = html.indexOf(version); + if (versionIndex !== -1) { + const contextStart = Math.max(0, versionIndex - 200); + const contextEnd = Math.min(html.length, versionIndex + 300); + const context = html.substring(contextStart, contextEnd); + const contextChangesetMatch = /[Cc]hangeset:\s*([a-f0-9]{12})/i.exec(context); + if (contextChangesetMatch?.[1]) { + changeset = contextChangesetMatch[1]; + } + } + } + + if (changeset) { + versions.push({ + version, + changeset, + }); + } } - return toEditorVersionInfo({ - version: versionMatch[1], - changeset: changesetMatch[1], - }); + return versions + .map(toEditorVersionInfo) + .filter((versionInfo): versionInfo is EditorVersionInfo => versionInfo !== null); +}; + +export const scrapeLatestOfficialUnityVersion = async (): Promise => { + const recentVersions = await scrapeRecentOfficialUnityVersions(); + return recentVersions.length > 0 ? recentVersions[0] : null; }; export const scrapeVersions = async (): Promise => { @@ -74,7 +109,7 @@ export const scrapeVersions = async (): Promise => { changeset, }), ); - const latestOfficialVersion = await scrapeLatestOfficialUnityVersion(); + const recentOfficialVersions = await scrapeRecentOfficialUnityVersions(); // Merge XLTS versions into main list, avoiding duplicates const existingVersions = new Set(unityVersions.map((v) => v.version)); @@ -85,11 +120,15 @@ export const scrapeVersions = async (): Promise => { } } - if (latestOfficialVersion && !existingVersions.has(latestOfficialVersion.version)) { - unityVersions.push({ - version: latestOfficialVersion.version, - changeset: latestOfficialVersion.changeSet, - }); + // Merge recent official versions discovered from Unity releases page + for (const officialVersion of recentOfficialVersions) { + if (!existingVersions.has(officialVersion.version)) { + unityVersions.push({ + version: officialVersion.version, + changeset: officialVersion.changeSet, + }); + existingVersions.add(officialVersion.version); + } } if (unityVersions?.length > 0) { diff --git a/functions/test/scrapeVersions.test.ts b/functions/test/scrapeVersions.test.ts index 879a5e9..41d4659 100644 --- a/functions/test/scrapeVersions.test.ts +++ b/functions/test/scrapeVersions.test.ts @@ -2,6 +2,7 @@ import { describe, it, expect, vi, beforeEach } from 'vitest'; import { scrapeLatestOfficialUnityVersion, scrapeVersions, + scrapeRecentOfficialUnityVersions, } from '../src/logic/ingestUnityVersions/scrapeVersions'; import { SearchMode } from 'unity-changeset'; import fetch from 'node-fetch'; @@ -47,18 +48,18 @@ describe('scrapeVersions', () => { }, { version: '2023.2.10f1', - changeset: 'def456ghi789', + changeset: '234567ab8cd9', }, ]; const mockXltsVersions = [ { version: '2022.3.21f1', // XLTS versions might have same format as regular versions - changeset: 'xyz789uvw123', + changeset: '789abcdef012', }, { version: '2021.3.25f1', - changeset: 'uvw123rst456', + changeset: '345cdef67890', }, ]; @@ -94,7 +95,7 @@ describe('scrapeVersions', () => { expect(result).toContainEqual( expect.objectContaining({ version: '2023.2.10f1', - changeSet: 'def456ghi789', + changeSet: '234567ab8cd9', major: 2023, minor: 2, patch: '10', @@ -105,7 +106,7 @@ describe('scrapeVersions', () => { expect(result).toContainEqual( expect.objectContaining({ version: '2022.3.21f1', - changeSet: 'xyz789uvw123', + changeSet: '789abcdef012', major: 2022, minor: 3, patch: '21', @@ -115,7 +116,7 @@ describe('scrapeVersions', () => { expect(result).toContainEqual( expect.objectContaining({ version: '2021.3.25f1', - changeSet: 'uvw123rst456', + changeSet: '345cdef67890', major: 2021, minor: 3, patch: '25', @@ -191,11 +192,11 @@ describe('scrapeVersions', () => { const mockXltsVersions = [ { version: '2022.3.20f1', // Duplicate version - changeset: 'duplicate456', + changeset: 'abc123def456', }, { version: '2022.3.21f1', - changeset: 'xyz789uvw123', + changeset: '789abcdef012', }, ]; @@ -224,18 +225,18 @@ describe('scrapeVersions', () => { }, { version: '2022.3.20a1', // Alpha version - should be excluded - changeset: 'def456ghi789', + changeset: '234567ab8cd9', }, ]; const mockXltsVersions = [ { version: '2021.3.25f1', // Final version - should be included - changeset: 'xyz789uvw123', + changeset: '789abcdef012', }, { version: '2020.3.15a2', // Alpha version - should be excluded - changeset: 'uvw123rst456', + changeset: '345cdef67890', }, ]; @@ -264,7 +265,7 @@ describe('scrapeVersions', () => { expect(result).toContainEqual( expect.objectContaining({ version: '2021.3.25f1', - changeSet: 'xyz789uvw123', + changeSet: '789abcdef012', major: 2021, minor: 3, patch: '25', @@ -283,14 +284,14 @@ describe('scrapeVersions', () => { }, { version: '5.6.7f1', // Should be excluded (major < 2017) - changeset: 'def456ghi789', + changeset: '234567ab8cd9', }, ]; const mockXltsVersions = [ { version: '2021.3.25f1', // Should be included - changeset: 'xyz789uvw123', + changeset: '789abcdef012', }, ]; @@ -319,7 +320,7 @@ describe('scrapeVersions', () => { expect(result).toContainEqual( expect.objectContaining({ version: '2021.3.25f1', - changeSet: 'xyz789uvw123', + changeSet: '789abcdef012', major: 2021, minor: 3, patch: '25', @@ -337,3 +338,161 @@ describe('scrapeVersions', () => { await expect(scrapeVersions()).rejects.toThrow('No Unity versions found!'); }); }); + +describe('scrapeRecentOfficialUnityVersions', () => { + beforeEach(() => { + vi.clearAllMocks(); + }); + + it('should discover multiple recent versions from the releases page', async () => { + const html = ` +

Unity 6000.4.10f1

+ Install + +

Unity 6000.3.17f1

+

Changeset: abc123def456

+ +

Unity 6000.2.5f1

+ Download + `; + mockedFetch.mockResolvedValue({ + ok: true, + status: 200, + text: async () => html, + } as any); + + const result = await scrapeRecentOfficialUnityVersions(); + + expect(result).toHaveLength(3); + expect(result.map((v) => v.version)).toContain('6000.4.10f1'); + expect(result.map((v) => v.version)).toContain('6000.3.17f1'); + expect(result.map((v) => v.version)).toContain('6000.2.5f1'); + }); + + it('should extract changesets from unityhub:// URLs', async () => { + const html = ` + Install + Install + `; + mockedFetch.mockResolvedValue({ + ok: true, + status: 200, + text: async () => html, + } as any); + + const result = await scrapeRecentOfficialUnityVersions(); + + expect(result).toContainEqual( + expect.objectContaining({ + version: '6000.4.10f1', + changeSet: 'feeafc12a938', + }), + ); + expect(result).toContainEqual( + expect.objectContaining({ + version: '6000.3.17f1', + changeSet: 'abc123def456', + }), + ); + }); + + it('should extract changesets from context near the version', async () => { + const html = ` +

Unity 6000.4.10f1

+

Changeset: feeafc12a938

+ +

Unity 6000.3.17f1

+

Changeset: abc123def456 is the commit hash

+ `; + mockedFetch.mockResolvedValue({ + ok: true, + status: 200, + text: async () => html, + } as any); + + const result = await scrapeRecentOfficialUnityVersions(); + + // Both should be found - implementation uses context-window search + expect(result.length).toBeGreaterThanOrEqual(2); + expect(result.some((v) => v.version === '6000.4.10f1')).toBe(true); + expect(result.some((v) => v.version === '6000.3.17f1')).toBe(true); + }); + + it('should skip versions without valid changesets nearby', async () => { + const html = ` +

Unity 6000.4.10f1

+ Install + +

Unity 6000.2.5f1

+

This version has no changeset information

+ `; + mockedFetch.mockResolvedValue({ + ok: true, + status: 200, + text: async () => html, + } as any); + + const result = await scrapeRecentOfficialUnityVersions(); + + // Only 6000.4.10f1 should be found with a valid changeset + expect(result.length).toBeGreaterThanOrEqual(1); + expect(result.map((v) => v.version)).toContain('6000.4.10f1'); + expect(result.map((v) => v.version)).not.toContain('6000.2.5f1'); + }); + + it('should deduplicate versions found multiple times on the page', async () => { + const html = ` +

Unity 6000.4.10f1

+ Install + +

Latest version: 6000.4.10f1

+ Download + `; + mockedFetch.mockResolvedValue({ + ok: true, + status: 200, + text: async () => html, + } as any); + + const result = await scrapeRecentOfficialUnityVersions(); + + expect(result).toHaveLength(1); + expect(result[0].version).toBe('6000.4.10f1'); + }); + + it('should return empty array if page returns error', async () => { + mockedFetch.mockResolvedValue({ + ok: false, + status: 404, + } as any); + + await expect(scrapeRecentOfficialUnityVersions()).rejects.toThrow( + 'Unity release page returned 404', + ); + }); + + it('should filter out non-final versions', async () => { + const html = ` +

Unity 6000.4.10f1

+ Install + +

Unity 6000.4.10a1

+ Install + +

Unity 6000.4.9f1

+ Install + `; + mockedFetch.mockResolvedValue({ + ok: true, + status: 200, + text: async () => html, + } as any); + + const result = await scrapeRecentOfficialUnityVersions(); + + expect(result.length).toBeGreaterThanOrEqual(2); + expect(result.map((v) => v.version)).toContain('6000.4.10f1'); + expect(result.map((v) => v.version)).toContain('6000.4.9f1'); + expect(result.map((v) => v.version)).not.toContain('6000.4.10a1'); + }); +});