Skip to content

Commit a2495da

Browse files
chidozieononiwuazure-sdk
authored andcommitted
Update Verify Links and CSpell to pass Network Isolation
1 parent 500aa84 commit a2495da

3 files changed

Lines changed: 244 additions & 15 deletions

File tree

eng/common/pipelines/templates/steps/check-spelling.yml

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,8 @@
1515
# npm commands in the context of this step. If specified, this
1616
# will be set as the npm_config_userconfig environment variable
1717
# so that npm uses the provided config file instead of the default
18-
# user config.
18+
# user config. If not specified, this template creates and
19+
# authenticates a temporary .npmrc and uses that file.
1920
# This check recognizes the setting of variable "Skip.SpellCheck"
2021
# if set to 'true', spellchecking will not be invoked.
2122

@@ -35,6 +36,11 @@ parameters:
3536

3637
steps:
3738
- ${{ if eq(variables['Build.Reason'], 'PullRequest') }}:
39+
- ${{ if eq(parameters.NpmConfigUserConfig, '') }}:
40+
- template: /eng/common/pipelines/templates/steps/create-authenticated-npmrc.yml
41+
parameters:
42+
npmrcPath: $(Agent.TempDirectory)/check-spelling/.npmrc
43+
3844
- task: PowerShell@2
3945
displayName: Check spelling (cspell)
4046
condition: and(succeeded(), ne(variables['Skip.SpellCheck'],'true'))
@@ -49,6 +55,8 @@ steps:
4955
env:
5056
${{ if ne(parameters.NpmConfigUserConfig, '') }}:
5157
npm_config_userconfig: ${{ parameters.NpmConfigUserConfig }}
58+
${{ if eq(parameters.NpmConfigUserConfig, '') }}:
59+
npm_config_userconfig: "$(Agent.TempDirectory)/check-spelling/.npmrc"
5260
- ${{ if ne('', parameters.ScriptToValidateUpgrade) }}:
5361
- pwsh: |
5462
$changedFiles = ./eng/common/scripts/get-changedfiles.ps1

eng/common/scripts/Verify-Links.ps1

Lines changed: 46 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -167,28 +167,60 @@ function ProcessCratesIoLink([System.Uri]$linkUri, $path) {
167167
}
168168

169169
function ProcessNpmLink([System.Uri]$linkUri) {
170-
# npmjs.com started using Cloudflare which returns 403 and we need to instead check the registry api for existence checks
171-
# https://github.com/orgs/community/discussions/174098#discussioncomment-14461226
172-
173-
# Handle versioned URLs: https://www.npmjs.com/package/@azure/ai-agents/v/1.1.0 -> https://registry.npmjs.org/@azure/ai-agents/1.1.0
174-
# Handle non-versioned URLs: https://www.npmjs.com/package/@azure/ai-agents -> https://registry.npmjs.org/@azure/ai-agents
175-
# The regex captures the package name (which may contain a slash for scoped packages) and optionally the version.
176-
# Query parameters and URL fragments are excluded from the transformation.
170+
# npmjs.com links are verified via the Azure DevOps public feed upstream API to avoid
171+
# direct calls to registry.npmjs.org or npmjs.com under CFS network isolation.
172+
# Upstream versions reflect what npmjs.org publishes, not just what the feed has cached.
173+
#
174+
# Handle versioned URLs: https://www.npmjs.com/package/@azure/ai-agents/v/1.1.0
175+
# -> checks that version 1.1.0 is present in upstream via ADO feed API
176+
# Handle non-versioned URLs: https://www.npmjs.com/package/@azure/ai-agents
177+
# -> checks that at least one upstream version exists via ADO feed API
178+
#
179+
# ADO feed upstream API: https://pkgs.dev.azure.com/azure-sdk/public/_apis/packaging/feeds/azure-sdk-for-js/npm/packages/{package}/upstreamVersions
177180
$urlString = $linkUri.ToString()
181+
$packageName = $null
182+
$version = $null
183+
178184
if ($urlString -match '^https?://(?:www\.)?npmjs\.com/package/([^?#]+)/v/([^?#]+)') {
179-
# Versioned URL: remove the /v/ segment but keep the version
180-
$apiUrl = "https://registry.npmjs.org/$($matches[1])/$($matches[2])"
185+
# Versioned URL: e.g. https://www.npmjs.com/package/@azure/ai-agents/v/1.1.0
186+
$packageName = $matches[1]
187+
$version = $matches[2]
181188
}
182189
elseif ($urlString -match '^https?://(?:www\.)?npmjs\.com/package/([^?#]+)') {
183-
# Non-versioned URL: just replace the domain
184-
$apiUrl = "https://registry.npmjs.org/$($matches[1])"
190+
# Non-versioned URL: e.g. https://www.npmjs.com/package/@azure/ai-agents
191+
$packageName = $matches[1]
185192
}
186193
else {
187-
# Fallback: use the original URL if it doesn't match expected patterns
188-
$apiUrl = $urlString
194+
Write-Verbose "Could not parse npm package name from $linkUri, skipping network verification"
195+
return $true
189196
}
190197

191-
return ProcessStandardLink ([System.Uri]$apiUrl)
198+
# Scoped package names (e.g. @azure/ai-agents) must be percent-encoded for the path segment
199+
$encodedPackageName = [System.Uri]::EscapeDataString($packageName)
200+
$upstreamApiUrl = "https://pkgs.dev.azure.com/azure-sdk/public/_apis/packaging/feeds/azure-sdk-for-js/npm/packages/$encodedPackageName/upstreamVersions?api-version=7.1-preview.1"
201+
202+
Write-Verbose "Checking npm package '$packageName' via ADO upstream feed: $upstreamApiUrl"
203+
204+
# Invoke-RestMethod throws on non-2xx (e.g. 404 for unknown package); CheckLink's catch block handles it
205+
$response = Invoke-RestMethod -Uri $upstreamApiUrl -Method GET -UserAgent $userAgent -TimeoutSec $requestTimeoutSec
206+
207+
if ($version) {
208+
# Versioned: the specific version must exist in upstream
209+
$versionExists = $response.value | Where-Object { $_.version -eq $version }
210+
if (!$versionExists) {
211+
Write-Host "Version '$version' of npm package '$packageName' not found in ADO upstream"
212+
return $false
213+
}
214+
}
215+
else {
216+
# Non-versioned: at least one upstream version must exist
217+
if (!$response.value -or $response.value.Count -eq 0) {
218+
Write-Host "npm package '$packageName' has no upstream versions in ADO feed"
219+
return $false
220+
}
221+
}
222+
223+
return $true
192224
}
193225

194226
function ProcessStandardLink([System.Uri]$linkUri) {
Lines changed: 189 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,189 @@
1+
# Run tests:
2+
# Install-Module -Name Pester -Force -SkipPublisherCheck
3+
# Invoke-Pester -Passthru $PSScriptRoot/Verify-Links.Tests.ps1
4+
5+
BeforeAll {
6+
# Load only the functions we need by dot-sourcing the script with a dummy param block.
7+
# We source the script file and then override Invoke-RestMethod / Invoke-WebRequest
8+
# inside each test via Mock so no real network calls are made.
9+
10+
# Stub out logging functions that the script depends on but are defined in logging.ps1
11+
function LogWarning($msg) { Write-Warning $msg }
12+
function LogError($msg) { Write-Error $msg }
13+
function LogGroupStart($msg) {}
14+
function LogGroupEnd {}
15+
16+
# Source only the function definitions (stop before the script-body runs)
17+
# by dot-sourcing within a script block that replaces the parameter-driven
18+
# body with a no-op after loading.
19+
$scriptPath = (Resolve-Path "$PSScriptRoot/../Verify-Links.ps1").Path
20+
21+
# Extract and invoke only the function definitions by parsing the AST
22+
$ast = [System.Management.Automation.Language.Parser]::ParseFile($scriptPath, [ref]$null, [ref]$null)
23+
$functionDefs = $ast.FindAll({ $args[0] -is [System.Management.Automation.Language.FunctionDefinitionAst] }, $false)
24+
foreach ($fn in $functionDefs) {
25+
Invoke-Expression $fn.Extent.Text
26+
}
27+
28+
# Script-scope variables referenced inside the functions
29+
$script:userAgent = "TestAgent/1.0"
30+
$script:requestTimeoutSec = 15
31+
}
32+
33+
Describe "ProcessNpmLink" {
34+
Context "Unparseable URL" {
35+
It "Returns true and skips verification for a URL that does not match npmjs.com package pattern" {
36+
$result = ProcessNpmLink ([System.Uri]"https://npmjs.com/browse/keyword/azure")
37+
$result | Should -Be $true
38+
}
39+
}
40+
41+
Context "Non-versioned URL - package found in ADO upstream" {
42+
BeforeEach {
43+
Mock Invoke-RestMethod {
44+
return @{
45+
value = @(
46+
@{ version = "1.0.0" },
47+
@{ version = "2.0.0" }
48+
)
49+
}
50+
} -ParameterFilter { $Uri -like "*upstreamVersions*" }
51+
}
52+
53+
It "Returns true when package has upstream versions" {
54+
$result = ProcessNpmLink ([System.Uri]"https://www.npmjs.com/package/@azure/ai-agents")
55+
$result | Should -Be $true
56+
}
57+
58+
It "Encodes scoped package name correctly in the API URL" {
59+
ProcessNpmLink ([System.Uri]"https://www.npmjs.com/package/@azure/ai-agents") | Out-Null
60+
Should -Invoke Invoke-RestMethod -ParameterFilter {
61+
$Uri -like "*%40azure%2Fai-agents*"
62+
} -Times 1 -Exactly
63+
}
64+
65+
It "Calls the ADO feed upstream API, not registry.npmjs.org or npmjs.com" {
66+
ProcessNpmLink ([System.Uri]"https://www.npmjs.com/package/@azure/core-http") | Out-Null
67+
Should -Invoke Invoke-RestMethod -ParameterFilter {
68+
$Uri -like "https://pkgs.dev.azure.com/azure-sdk/public/_apis/packaging/feeds/azure-sdk-for-js/npm/packages/*"
69+
} -Times 1 -Exactly
70+
}
71+
}
72+
73+
Context "Non-versioned URL - package not found (no upstream versions)" {
74+
BeforeEach {
75+
Mock Invoke-RestMethod {
76+
return @{ value = @() }
77+
} -ParameterFilter { $Uri -like "*upstreamVersions*" }
78+
}
79+
80+
It "Returns false when the package has no upstream versions" {
81+
$result = ProcessNpmLink ([System.Uri]"https://www.npmjs.com/package/@azure/nonexistent-package")
82+
$result | Should -Be $false
83+
}
84+
}
85+
86+
Context "Non-versioned URL - package not found (ADO returns 404)" {
87+
BeforeEach {
88+
Mock Invoke-RestMethod {
89+
$response = [System.Net.HttpWebResponse]::new.Invoke(@())
90+
throw [System.Net.WebException]::new(
91+
"The remote server returned an error: (404) Not Found.",
92+
$null,
93+
[System.Net.WebExceptionStatus]::ProtocolError,
94+
$null
95+
)
96+
} -ParameterFilter { $Uri -like "*upstreamVersions*" }
97+
}
98+
99+
It "Propagates exception (to be handled by CheckLink caller)" {
100+
{ ProcessNpmLink ([System.Uri]"https://www.npmjs.com/package/@azure/nonexistent-package") } | Should -Throw
101+
}
102+
}
103+
104+
Context "Versioned URL - correct version present in ADO upstream" {
105+
BeforeEach {
106+
Mock Invoke-RestMethod {
107+
return @{
108+
value = @(
109+
@{ version = "1.0.0" },
110+
@{ version = "1.1.0" },
111+
@{ version = "2.0.0" }
112+
)
113+
}
114+
} -ParameterFilter { $Uri -like "*upstreamVersions*" }
115+
}
116+
117+
It "Returns true when the specific version exists in upstream" {
118+
$result = ProcessNpmLink ([System.Uri]"https://www.npmjs.com/package/@azure/ai-agents/v/1.1.0")
119+
$result | Should -Be $true
120+
}
121+
122+
It "Encodes scoped package name correctly for versioned URLs" {
123+
ProcessNpmLink ([System.Uri]"https://www.npmjs.com/package/@azure/ai-agents/v/1.1.0") | Out-Null
124+
Should -Invoke Invoke-RestMethod -ParameterFilter {
125+
$Uri -like "*%40azure%2Fai-agents*"
126+
} -Times 1 -Exactly
127+
}
128+
}
129+
130+
Context "Versioned URL - version not present in ADO upstream" {
131+
BeforeEach {
132+
Mock Invoke-RestMethod {
133+
return @{
134+
value = @(
135+
@{ version = "1.0.0" },
136+
@{ version = "2.0.0" }
137+
)
138+
}
139+
} -ParameterFilter { $Uri -like "*upstreamVersions*" }
140+
}
141+
142+
It "Returns false when the specific version does not exist in upstream" {
143+
$result = ProcessNpmLink ([System.Uri]"https://www.npmjs.com/package/@azure/ai-agents/v/9.9.9")
144+
$result | Should -Be $false
145+
}
146+
}
147+
148+
Context "Versioned URL - unscoped package" {
149+
BeforeEach {
150+
Mock Invoke-RestMethod {
151+
return @{
152+
value = @(
153+
@{ version = "3.2.1" }
154+
)
155+
}
156+
} -ParameterFilter { $Uri -like "*upstreamVersions*" }
157+
}
158+
159+
It "Handles unscoped package names in versioned URLs" {
160+
$result = ProcessNpmLink ([System.Uri]"https://www.npmjs.com/package/typescript/v/3.2.1")
161+
$result | Should -Be $true
162+
}
163+
164+
It "Returns false when unscoped versioned package version is missing" {
165+
$result = ProcessNpmLink ([System.Uri]"https://www.npmjs.com/package/typescript/v/99.0.0")
166+
$result | Should -Be $false
167+
}
168+
}
169+
170+
Context "Query parameters and fragments are excluded from package name" {
171+
BeforeEach {
172+
Mock Invoke-RestMethod {
173+
return @{
174+
value = @(
175+
@{ version = "1.0.0" }
176+
)
177+
}
178+
} -ParameterFilter { $Uri -like "*upstreamVersions*" }
179+
}
180+
181+
It "Strips query string from non-versioned URL when extracting package name" {
182+
$result = ProcessNpmLink ([System.Uri]"https://www.npmjs.com/package/@azure/ai-agents?activeTab=readme")
183+
$result | Should -Be $true
184+
Should -Invoke Invoke-RestMethod -ParameterFilter {
185+
$Uri -like "*%40azure%2Fai-agents*" -and $Uri -notlike "*activeTab*"
186+
} -Times 1 -Exactly
187+
}
188+
}
189+
}

0 commit comments

Comments
 (0)