Merge pull request #284438 from microsoft/dev/dmitriv/fetch-tool-fixes

Allow redirects to trusted domains

Author: dmitrivMS

…nch/contrib/url/common/trustedDomains.ts …vs/platform/url/common/trustedDomains.tssrc/vs/workbench/contrib/url/common/trustedDomains.ts renamed to src/vs/platform/url/common/trustedDomains.ts src/vs/workbench/contrib/url/common/trustedDomains.ts renamed to src/vs/platform/url/common/trustedDomains.ts

@@ -2,9 +2,9 @@
  *  Copyright (c) Microsoft Corporation. All rights reserved.
  *  Licensed under the MIT License. See License.txt in the project root for license information.
  *--------------------------------------------------------------------------------------------*/
-import { URI } from '../../../../base/common/uri.js';
-import { testUrlMatchesGlob } from './urlGlob.js';
 
+import { URI } from '../../../base/common/uri.js';
+import { testUrlMatchesGlob } from './urlGlob.js';
 
 /**
  * Check whether a domain like https://www.microsoft.com matches
@@ -14,7 +14,6 @@ import { testUrlMatchesGlob } from './urlGlob.js';
  * - There's no subdomain matching. For example https://microsoft.com doesn't match https://www.microsoft.com
  * - Star matches all subdomains. For example https://*.microsoft.com matches https://www.microsoft.com and https://foo.bar.microsoft.com
  */
-
 export function isURLDomainTrusted(url: URI, trustedDomains: string[]): boolean {
 	url = URI.parse(normalizeURL(url));
 	trustedDomains = trustedDomains.map(normalizeURL);
@@ -35,10 +34,10 @@ export function isURLDomainTrusted(url: URI, trustedDomains: string[]): boolean
 
 	return false;
 }
+
 /**
  * Case-normalize some case-insensitive URLs, such as github.
  */
-
 export function normalizeURL(url: string | URI): string {
 	const caseInsensitiveAuthorities = ['github.com'];
 	try {
@@ -50,10 +49,10 @@ export function normalizeURL(url: string | URI): string {
 		}
 	} catch { return url.toString(); }
 }
+
 const rLocalhost = /^(.+\.)?localhost(:\d+)?$/i;
 const r127 = /^127.0.0.1(:\d+)?$/;
 
 export function isLocalhostAuthority(authority: string) {
 	return rLocalhost.test(authority) || r127.test(authority);
 }
-

…/workbench/contrib/url/common/urlGlob.ts src/vs/platform/url/common/urlGlob.tssrc/vs/workbench/contrib/url/common/urlGlob.ts renamed to src/vs/platform/url/common/urlGlob.ts src/vs/workbench/contrib/url/common/urlGlob.ts renamed to src/vs/platform/url/common/urlGlob.ts

@@ -3,7 +3,7 @@
  *  Licensed under the MIT License. See License.txt in the project root for license information.
  *--------------------------------------------------------------------------------------------*/
 
-import { URI } from '../../../../base/common/uri.js';
+import { URI } from '../../../base/common/uri.js';
 
 /**
  * Normalizes a URL by removing trailing slashes and query/fragment components.

src/vs/platform/url/test/common/trustedDomains.test.ts

@@ -0,0 +1,126 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+import assert from 'assert';
+import { isLocalhostAuthority, isURLDomainTrusted, normalizeURL } from '../../common/trustedDomains.js';
+import { URI } from '../../../../base/common/uri.js';
+import { ensureNoDisposablesAreLeakedInTestSuite } from '../../../../base/test/common/utils.js';
+
+suite('trustedDomains', () => {
+
+	ensureNoDisposablesAreLeakedInTestSuite();
+
+	suite('isURLDomainTrusted', () => {
+
+		test('localhost is always trusted', () => {
+			assert.strictEqual(isURLDomainTrusted(URI.parse('http://localhost:3000'), []), true);
+			assert.strictEqual(isURLDomainTrusted(URI.parse('http://127.0.0.1:3000'), []), true);
+			assert.strictEqual(isURLDomainTrusted(URI.parse('http://subdomain.localhost'), []), true);
+		});
+
+		test('wildcard (*) matches everything', () => {
+			assert.strictEqual(isURLDomainTrusted(URI.parse('https://example.com'), ['*']), true);
+			assert.strictEqual(isURLDomainTrusted(URI.parse('http://anything.org'), ['*']), true);
+			assert.strictEqual(isURLDomainTrusted(URI.parse('https://github.com/microsoft'), ['*']), true);
+		});
+
+		test('exact domain match', () => {
+			assert.strictEqual(isURLDomainTrusted(URI.parse('https://example.com'), ['https://example.com']), true);
+			assert.strictEqual(isURLDomainTrusted(URI.parse('https://example.com/path'), ['https://example.com']), true);
+			assert.strictEqual(isURLDomainTrusted(URI.parse('http://example.com'), ['https://example.com']), false);
+		});
+
+		test('subdomain wildcard matching', () => {
+			assert.strictEqual(isURLDomainTrusted(URI.parse('https://api.github.com'), ['https://*.github.com']), true);
+			assert.strictEqual(isURLDomainTrusted(URI.parse('https://github.com'), ['https://*.github.com']), true);
+			assert.strictEqual(isURLDomainTrusted(URI.parse('https://sub.api.github.com'), ['https://*.github.com']), true);
+		});
+
+		test('path matching', () => {
+			assert.strictEqual(isURLDomainTrusted(URI.parse('https://example.com/api/v1'), ['https://example.com/api/*']), true);
+			// Path without trailing content doesn't match a wildcard pattern requiring more path segments
+			assert.strictEqual(isURLDomainTrusted(URI.parse('https://example.com/api'), ['https://example.com/api/*']), false);
+		});
+
+		test('scheme must match', () => {
+			assert.strictEqual(isURLDomainTrusted(URI.parse('https://example.com'), ['http://example.com']), false);
+			assert.strictEqual(isURLDomainTrusted(URI.parse('http://example.com'), ['https://example.com']), false);
+		});
+
+		test('not trusted when no match', () => {
+			assert.strictEqual(isURLDomainTrusted(URI.parse('https://example.com'), ['https://other.com']), false);
+			assert.strictEqual(isURLDomainTrusted(URI.parse('https://example.com'), []), false);
+		});
+
+		test('multiple trusted domains', () => {
+			const trusted = ['https://github.com', 'https://microsoft.com'];
+			assert.strictEqual(isURLDomainTrusted(URI.parse('https://github.com'), trusted), true);
+			assert.strictEqual(isURLDomainTrusted(URI.parse('https://microsoft.com'), trusted), true);
+			assert.strictEqual(isURLDomainTrusted(URI.parse('https://google.com'), trusted), false);
+		});
+
+		test('case normalization for github', () => {
+			assert.strictEqual(isURLDomainTrusted(URI.parse('https://github.com/Microsoft/VSCode'), ['https://github.com/microsoft/vscode']), true);
+			assert.strictEqual(isURLDomainTrusted(URI.parse('https://github.com/microsoft/vscode'), ['https://github.com/Microsoft/VSCode']), true);
+		});
+	});
+
+	suite('normalizeURL', () => {
+
+		test('normalizes github.com URLs to lowercase path', () => {
+			assert.strictEqual(normalizeURL('https://github.com/Microsoft/VSCode'), 'https://github.com/microsoft/vscode');
+			assert.strictEqual(normalizeURL('https://github.com/OWNER/REPO'), 'https://github.com/owner/repo');
+		});
+
+		test('does not normalize non-github URLs', () => {
+			assert.strictEqual(normalizeURL('https://example.com/Path/To/Resource'), 'https://example.com/Path/To/Resource');
+			assert.strictEqual(normalizeURL('https://microsoft.com/Products'), 'https://microsoft.com/Products');
+		});
+
+		test('handles URI objects', () => {
+			const uri = URI.parse('https://github.com/Microsoft/VSCode');
+			assert.strictEqual(normalizeURL(uri), 'https://github.com/microsoft/vscode');
+		});
+
+		test('handles invalid URIs gracefully', () => {
+			const result = normalizeURL('not-a-valid-uri');
+			assert.strictEqual(typeof result, 'string');
+		});
+	});
+
+	suite('isLocalhostAuthority', () => {
+
+		test('recognizes localhost', () => {
+			assert.strictEqual(isLocalhostAuthority('localhost'), true);
+			assert.strictEqual(isLocalhostAuthority('localhost:3000'), true);
+			assert.strictEqual(isLocalhostAuthority('localhost:8080'), true);
+		});
+
+		test('recognizes subdomains of localhost', () => {
+			assert.strictEqual(isLocalhostAuthority('subdomain.localhost'), true);
+			assert.strictEqual(isLocalhostAuthority('api.localhost:3000'), true);
+			assert.strictEqual(isLocalhostAuthority('a.b.c.localhost'), true);
+		});
+
+		test('recognizes 127.0.0.1', () => {
+			assert.strictEqual(isLocalhostAuthority('127.0.0.1'), true);
+			assert.strictEqual(isLocalhostAuthority('127.0.0.1:3000'), true);
+			assert.strictEqual(isLocalhostAuthority('127.0.0.1:8080'), true);
+		});
+
+		test('case insensitive for localhost', () => {
+			assert.strictEqual(isLocalhostAuthority('LOCALHOST'), true);
+			assert.strictEqual(isLocalhostAuthority('LocalHost:3000'), true);
+			assert.strictEqual(isLocalhostAuthority('SUB.LOCALHOST'), true);
+		});
+
+		test('does not match non-localhost authorities', () => {
+			assert.strictEqual(isLocalhostAuthority('example.com'), false);
+			assert.strictEqual(isLocalhostAuthority('notlocalhost.com'), false);
+			assert.strictEqual(isLocalhostAuthority('127.0.0.2'), false);
+			assert.strictEqual(isLocalhostAuthority('192.168.1.1'), false);
+		});
+	});
+});

src/vs/platform/url/test/common/urlGlob.test.ts

@@ -0,0 +1,107 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+import assert from 'assert';
+import { URI } from '../../../../base/common/uri.js';
+import { testUrlMatchesGlob } from '../../common/urlGlob.js';
+import { ensureNoDisposablesAreLeakedInTestSuite } from '../../../../base/test/common/utils.js';
+
+suite('urlGlob', () => {
+
+	ensureNoDisposablesAreLeakedInTestSuite();
+
+	suite('testUrlMatchesGlob', () => {
+
+		test('exact match', () => {
+			assert.strictEqual(testUrlMatchesGlob('https://example.com', 'https://example.com'), true);
+			assert.strictEqual(testUrlMatchesGlob('http://example.com', 'http://example.com'), true);
+			assert.strictEqual(testUrlMatchesGlob('https://example.com/path', 'https://example.com/path'), true);
+		});
+
+		test('trailing slashes are ignored', () => {
+			assert.strictEqual(testUrlMatchesGlob('https://example.com/', 'https://example.com'), true);
+			assert.strictEqual(testUrlMatchesGlob('https://example.com', 'https://example.com/'), true);
+			assert.strictEqual(testUrlMatchesGlob('https://example.com//', 'https://example.com'), true);
+			assert.strictEqual(testUrlMatchesGlob('https://example.com/path/', 'https://example.com/path'), true);
+		});
+
+		test('query and fragment are ignored', () => {
+			assert.strictEqual(testUrlMatchesGlob('https://example.com?query=value', 'https://example.com'), true);
+			assert.strictEqual(testUrlMatchesGlob('https://example.com#fragment', 'https://example.com'), true);
+			assert.strictEqual(testUrlMatchesGlob('https://example.com?query=value#fragment', 'https://example.com'), true);
+		});
+
+		test('scheme matching', () => {
+			assert.strictEqual(testUrlMatchesGlob('https://example.com', 'https://example.com'), true);
+			assert.strictEqual(testUrlMatchesGlob('http://example.com', 'https://example.com'), false);
+			assert.strictEqual(testUrlMatchesGlob('ftp://example.com', 'https://example.com'), false);
+		});
+
+		test('glob without scheme assumes http/https', () => {
+			assert.strictEqual(testUrlMatchesGlob('https://example.com', 'example.com'), true);
+			assert.strictEqual(testUrlMatchesGlob('http://example.com', 'example.com'), true);
+			assert.strictEqual(testUrlMatchesGlob('ftp://example.com', 'example.com'), false);
+		});
+
+		test('wildcard matching in path', () => {
+			assert.strictEqual(testUrlMatchesGlob('https://example.com/anything', 'https://example.com/*'), true);
+			assert.strictEqual(testUrlMatchesGlob('https://example.com/path/to/resource', 'https://example.com/*'), true);
+			assert.strictEqual(testUrlMatchesGlob('https://example.com/path/to/resource', 'https://example.com/path/*'), true);
+			assert.strictEqual(testUrlMatchesGlob('https://example.com/path/to/resource', 'https://example.com/path/*/resource'), true);
+		});
+
+		test('subdomain wildcard matching', () => {
+			assert.strictEqual(testUrlMatchesGlob('https://sub.example.com', 'https://*.example.com'), true);
+			assert.strictEqual(testUrlMatchesGlob('https://sub.domain.example.com', 'https://*.example.com'), true);
+			assert.strictEqual(testUrlMatchesGlob('https://example.com', 'https://*.example.com'), true);
+			// *. matches any number of characters before the domain, including other domains
+			assert.strictEqual(testUrlMatchesGlob('https://notexample.com', 'https://*.example.com'), true);
+		});
+
+		test('port matching', () => {
+			assert.strictEqual(testUrlMatchesGlob('https://example.com:8080', 'https://example.com:8080'), true);
+			assert.strictEqual(testUrlMatchesGlob('https://example.com:8080', 'https://example.com:9090'), false);
+			assert.strictEqual(testUrlMatchesGlob('https://example.com', 'https://example.com:8080'), false);
+		});
+
+		test('wildcard port matching', () => {
+			assert.strictEqual(testUrlMatchesGlob('https://example.com:8080', 'https://example.com:*'), true);
+			assert.strictEqual(testUrlMatchesGlob('https://example.com:9090', 'https://example.com:*'), true);
+			assert.strictEqual(testUrlMatchesGlob('https://example.com', 'https://example.com:*'), true);
+			assert.strictEqual(testUrlMatchesGlob('https://example.com:8080/path', 'https://example.com:*/path'), true);
+		});
+
+		test('root path glob', () => {
+			assert.strictEqual(testUrlMatchesGlob('https://example.com', 'https://example.com/'), true);
+			assert.strictEqual(testUrlMatchesGlob('https://example.com/', 'https://example.com/'), true);
+			assert.strictEqual(testUrlMatchesGlob('https://example.com/path', 'https://example.com/'), true);
+		});
+
+		test('mismatch cases', () => {
+			assert.strictEqual(testUrlMatchesGlob('https://example.com/path', 'https://example.com/other'), false);
+			assert.strictEqual(testUrlMatchesGlob('https://example.com', 'https://other.com'), false);
+			assert.strictEqual(testUrlMatchesGlob('https://sub.example.com', 'https://example.com'), false);
+		});
+
+		test('URI object input', () => {
+			const uri = URI.parse('https://example.com/path');
+			assert.strictEqual(testUrlMatchesGlob(uri, 'https://example.com/path'), true);
+			assert.strictEqual(testUrlMatchesGlob(uri, 'https://example.com/*'), true);
+		});
+
+		test('complex patterns', () => {
+			assert.strictEqual(testUrlMatchesGlob('https://api.github.com/repos/microsoft/vscode', 'https://*.github.com/repos/*/*'), true);
+			assert.strictEqual(testUrlMatchesGlob('https://github.com/microsoft/vscode', 'https://*.github.com/repos/*/*'), false);
+			assert.strictEqual(testUrlMatchesGlob('https://api.github.com:443/repos/microsoft/vscode', 'https://*.github.com:*/repos/*/*'), true);
+		});
+
+		test('edge cases', () => {
+			// Wildcard after authority doesn't match without additional path
+			assert.strictEqual(testUrlMatchesGlob('https://example.com', 'https://example.com*'), false);
+			assert.strictEqual(testUrlMatchesGlob('https://example.com.extra', 'https://example.com*'), true);
+			assert.strictEqual(testUrlMatchesGlob('https://example.com', '*'), true);
+		});
+	});
+});

src/vs/platform/webContentExtractor/common/webContentExtractor.ts

@@ -17,6 +17,11 @@ export interface IWebContentExtractorOptions {
 	 * 'false' by default.
 	 */
 	followRedirects?: boolean;
+
+	/**
+	 * List of trusted domain patterns for redirect validation.
+	 */
+	trustedDomains?: string[];
 }
 
 export type WebContentExtractResult =

src/vs/platform/webContentExtractor/electron-main/webContentExtractorService.ts

@@ -7,6 +7,7 @@ import { BrowserWindow } from 'electron';
 import { Limiter } from '../../../base/common/async.js';
 import { URI } from '../../../base/common/uri.js';
 import { ILogService } from '../../log/common/log.js';
+import { isURLDomainTrusted } from '../../url/common/trustedDomains.js';
 import { IWebContentExtractorOptions, IWebContentExtractorService, WebContentExtractResult } from '../common/webContentExtractor.js';
 import { WebContentCache } from './webContentCache.js';
 import { WebPageLoader } from './webPageLoader.js';
@@ -37,7 +38,13 @@ export class NativeWebContentExtractorService implements IWebContentExtractorSer
 			return cached;
 		}
 
-		const loader = new WebPageLoader((options) => new BrowserWindow(options), this._logger, uri, options);
+		const loader = new WebPageLoader(
+			(options) => new BrowserWindow(options),
+			this._logger,
+			uri,
+			options,
+			(uri) => isURLDomainTrusted(uri, options?.trustedDomains || []));
+
 		try {
 			const result = await loader.load();
 			this._webContentsCache.add(uri, options, result);

src/vs/platform/webContentExtractor/electron-main/webPageLoader.ts

@@ -7,7 +7,6 @@ import type { BeforeSendResponse, BrowserWindow, BrowserWindowConstructorOptions
 import { Queue, raceTimeout, TimeoutTimer } from '../../../base/common/async.js';
 import { createSingleCallFunction } from '../../../base/common/functional.js';
 import { Disposable, toDisposable } from '../../../base/common/lifecycle.js';
-import { equalsIgnoreCase } from '../../../base/common/strings.js';
 import { URI } from '../../../base/common/uri.js';
 import { generateUuid } from '../../../base/common/uuid.js';
 import { ILogService } from '../../log/common/log.js';
@@ -44,7 +43,8 @@ export class WebPageLoader extends Disposable {
 		browserWindowFactory: (options: BrowserWindowConstructorOptions) => BrowserWindow,
 		private readonly _logger: ILogService,
 		private readonly _uri: URI,
-		private readonly _options?: IWebContentExtractorOptions,
+		private readonly _options: IWebContentExtractorOptions | undefined,
+		private readonly _isTrustedDomain: (uri: URI) => boolean,
 	) {
 		super();
 
@@ -201,13 +201,30 @@ export class WebPageLoader extends Disposable {
 		this.trace(`Received 'will-navigate' or 'will-redirect' event, url: ${url}`);
 		if (!this._options?.followRedirects) {
 			const toURI = URI.parse(url);
-			if (!equalsIgnoreCase(toURI.authority, this._uri.authority)) {
-				event.preventDefault();
-				this._onResult({ status: 'redirect', toURI });
+
+			// Allow redirect if authority is the same when ignoring www prefix
+			if (this.normalizeAuthority(toURI.authority) === this.normalizeAuthority(this._uri.authority)) {
+				return;
+			}
+
+			// Allow redirect if target is a trusted domain
+			if (this._isTrustedDomain(toURI)) {
+				return;
 			}
+
+			// Otherwise, prevent redirect and report it
+			event.preventDefault();
+			this._onResult({ status: 'redirect', toURI });
 		}
 	}
 
+	/**
+	 * Normalizes an authority by removing the 'www.' prefix if present.
+	 */
+	private normalizeAuthority(authority: string): string {
+		return authority.toLowerCase().replace(/^www\./, '');
+	}
+
 	/**
 	 * Handles debugger messages related to network requests, tracking their lifecycle.
 	 * @note DO NOT add logging to this function, microsoft.com will freeze when too many logs are generated