Merge branch 'main' into dev/Aman-Jain-14/StorageCache_AutoImport_2025-05-01_API_spec

Author: Aman-Jain-14

.github/workflows/post-apiview.yml

@@ -0,0 +1,29 @@
+name: After APIView
+
+on:
+  check_run:
+    types: [completed]
+    
+permissions:
+  pull-requests: write
+  contents: read
+
+jobs:
+  post-apiview:
+    name: After APIView
+    runs-on: ubuntu-24.04
+    if: |
+      github.event.check_run.check_suite.app.name == 'Azure Pipelines' && (
+      contains(github.event.check_run.name, 'APIView') ||
+      contains(github.event.check_run.name, 'SDK Generation') )
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          sparse-checkout: 'eng/common'
+    
+      - name: Create APIView Comment on PR
+        run: |
+          . "eng/common/scripts/Helpers/ApiView-Helpers.ps1"
+          Set-ApiViewCommentForRelatedIssues -HeadCommitish ${{ github.event.check_run.head_sha }} -AuthToken ${{ secrets.GITHUB_TOKEN }}
+        shell: pwsh

eng/common/pipelines/templates/stages/archetype-sdk-tool-pwsh.yml

@@ -10,6 +10,9 @@ parameters:
   - name: TargetTags
     type: string
     default: ''
+  - name: PreTestSteps
+    type: object
+    default: []
 
 variables:
   - template: /eng/pipelines/templates/variables/globals.yml
@@ -36,6 +39,8 @@ stages:
           vmImage: $(Image)
 
         steps:
+          - ${{ parameters.PreTestSteps }}
+
           - template: /eng/common/pipelines/templates/steps/run-pester-tests.yml
             parameters:
               TargetDirectory: ${{ parameters.TargetDirectory }}

eng/common/scripts/Helpers/ApiView-Helpers.ps1

@@ -168,8 +168,8 @@ function Set-ApiViewCommentForPR {
   $commentText = @()
   $commentText += "## API Change Check"
   try {
-    $response = Invoke-RestMethod -Uri $apiviewEndpoint -Method Get -MaximumRetryCount 3
-    if ($response.Count -eq 0) {
+    $response = Invoke-WebRequest -Uri $apiviewEndpoint -Method Get -MaximumRetryCount 3
+    if ($response.StatusCode -ne 200) {
       LogWarning "API changes are not detected in this pull request."
       $commentText += ""
       $commentText += "API changes are not detected in this pull request."
@@ -179,10 +179,17 @@ function Set-ApiViewCommentForPR {
       $commentText += ""
       $commentText += "APIView identified API level changes in this PR and created the following API reviews"
       $commentText += ""
-      $commentText += "| Language | API Review for Package |"
-      $commentText += "|----------|---------|"
-      $response | ForEach-Object {
-        $commentText += "| $($_.language) | [$($_.packageName)]($($_.url)) |"
+
+      if ($RepoName.StartsWith(("azure-sdk-for-"))) {
+        $response | ForEach-Object {
+          $commentText += "[$($_.packageName)]($($_.url))"
+        }
+      } else {
+        $commentText += "| Language | API Review for Package |"
+        $commentText += "|----------|---------|"
+        $response | ForEach-Object {
+          $commentText += "| $($_.language) | [$($_.packageName)]($($_.url)) |"
+        }
       }
     }
   } catch{

eng/common/scripts/Package-Properties.ps1

@@ -77,11 +77,13 @@ class PackageProps {
         $this.ArtifactName = $artifactName
         $this.Initialize($name, $version, $directoryPath, $serviceDirectory)
     }
-    hidden [PSCustomObject]ParseYmlForArtifact([string]$ymlPath) {
+
+    hidden [PSCustomObject]ParseYmlForArtifact([string]$ymlPath, [bool]$soleCIYml = $false) {
         $content = LoadFrom-Yaml $ymlPath
         if ($content) {
             $artifacts = GetValueSafelyFrom-Yaml $content @("extends", "parameters", "Artifacts")
-            $artifactForCurrentPackage = $null
+            $artifactForCurrentPackage = @{}
+
             if ($artifacts) {
                 # If there's an artifactName match that to the name field from the yml
                 if ($this.ArtifactName) {
@@ -98,8 +100,9 @@ class PackageProps {
                 }
             }
 
-            # if we found an artifact for the current package, we should count this ci file as the source of the matrix for this package
-            if ($artifactForCurrentPackage) {
+            # if we found an artifact for the current package OR this is the sole ci.yml for the given service directory,
+            # we should count this ci file as the source of the matrix for this package
+            if ($artifactForCurrentPackage -or $soleCIYml) {
                 $result = [PSCustomObject]@{
                     ArtifactConfig = [HashTable]$artifactForCurrentPackage
                     ParsedYml = $content
@@ -116,11 +119,12 @@ class PackageProps {
         $RepoRoot = Resolve-Path (Join-Path $PSScriptRoot ".." ".." "..")
 
         $ciFolderPath = Join-Path -Path $RepoRoot -ChildPath (Join-Path "sdk" $this.ServiceDirectory)
-        $ciFiles = Get-ChildItem -Path $ciFolderPath -Filter "ci*.yml" -File
+        $ciFiles = @(Get-ChildItem -Path $ciFolderPath -Filter "ci*.yml" -File)
         $ciArtifactResult = $null
+        $soleCIYml = ($ciFiles.Count -eq 1)
 
         foreach ($ciFile in $ciFiles) {
-            $ciArtifactResult = $this.ParseYmlForArtifact($ciFile.FullName)
+            $ciArtifactResult = $this.ParseYmlForArtifact($ciFile.FullName, $soleCIYml)
             if ($ciArtifactResult) {
                 break
             }
@@ -137,7 +141,7 @@ class PackageProps {
         if (-not $this.ArtifactDetails) {
             $ciArtifactResult = $this.GetCIYmlForArtifact()
 
-            if ($ciArtifactResult) {
+            if ($ciArtifactResult -and $null -ne $ciArtifactResult.ArtifactConfig) {
                 $this.ArtifactDetails = [Hashtable]$ciArtifactResult.ArtifactConfig
 
                 $repoRoot = Resolve-Path (Join-Path $PSScriptRoot ".." ".." "..")
@@ -147,27 +151,32 @@ class PackageProps {
                 if (-not $this.ArtifactDetails["triggeringPaths"]) {
                     $this.ArtifactDetails["triggeringPaths"] = @()
                 }
-                else {
-                    $adjustedPaths = @()
-
-                    # we need to convert relative references to absolute references within the repo
-                    # this will make it extremely easy to compare triggering paths to files in the deleted+changed file list.
-                    for ($i = 0; $i -lt $this.ArtifactDetails["triggeringPaths"].Count; $i++) {
-                        $currentPath = $this.ArtifactDetails["triggeringPaths"][$i]
-                        $newPath = Join-Path $repoRoot $currentPath
-                        if (!$currentPath.StartsWith("/")) {
-                            $newPath = Join-Path $repoRoot $relRoot $currentPath
-                        }
-                        # it is a possibility that users may have a triggerPath dependency on a file that no longer exists.
-                        # before we resolve it to get rid of possible relative references, we should check if the file exists
-                        # if it doesn't, we should just leave it as is. Otherwise we would _crash_ here when a user accidentally
-                        # left a triggeringPath on a file that had been deleted
-                        if (Test-Path $newPath) {
-                            $adjustedPaths += (Resolve-Path -Path $newPath -Relative -RelativeBasePath $repoRoot).TrimStart(".").Replace("`\", "/")
-                        }
+
+                # if we know this is the matrix for our file, we should now see if there is a custom matrix config for the package
+                $serviceTriggeringPaths = GetValueSafelyFrom-Yaml $ciArtifactResult.ParsedYml @("extends", "parameters", "TriggeringPaths")
+                if ($serviceTriggeringPaths){
+                    $this.ArtifactDetails["triggeringPaths"] += $serviceTriggeringPaths
+                }
+
+                $adjustedPaths = @()
+
+                # we need to convert relative references to absolute references within the repo
+                # this will make it extremely easy to compare triggering paths to files in the deleted+changed file list.
+                for ($i = 0; $i -lt $this.ArtifactDetails["triggeringPaths"].Count; $i++) {
+                    $currentPath = $this.ArtifactDetails["triggeringPaths"][$i]
+                    $newPath = Join-Path $repoRoot $currentPath
+                    if (!$currentPath.StartsWith("/")) {
+                        $newPath = Join-Path $repoRoot $relRoot $currentPath
+                    }
+                    # it is a possibility that users may have a triggerPath dependency on a file that no longer exists.
+                    # before we resolve it to get rid of possible relative references, we should check if the file exists
+                    # if it doesn't, we should just leave it as is. Otherwise we would _crash_ here when a user accidentally
+                    # left a triggeringPath on a file that had been deleted
+                    if (Test-Path $newPath) {
+                        $adjustedPaths += (Resolve-Path -Path $newPath -Relative -RelativeBasePath $repoRoot).TrimStart(".").Replace("`\", "/")
                     }
-                    $this.ArtifactDetails["triggeringPaths"] = $adjustedPaths
                 }
+                $this.ArtifactDetails["triggeringPaths"] = $adjustedPaths
                 $this.ArtifactDetails["triggeringPaths"] += $ciYamlPath
 
                 $this.CIParameters["CIMatrixConfigs"] = @()
@@ -215,6 +224,22 @@ function Get-PkgProperties {
     return $null
 }
 
+function Get-PackagesFromPackageInfo([string]$PackageInfoFolder, [bool]$IncludeIndirect, [ScriptBlock]$CustomCompareFunction = $null) {
+    $packages = Get-ChildItem -R -Path $PackageInfoFolder -Filter "*.json" | ForEach-Object {
+        Get-Content $_.FullName | ConvertFrom-Json
+    }
+
+    if (-not $includeIndirect) {
+        $packages = $packages | Where-Object { $_.IncludedForValidation -eq $false }
+    }
+
+    if ($CustomCompareFunction) {
+        $packages = $packages | Where-Object { &$CustomCompareFunction $_ }
+    }
+
+    return $packages
+}
+
 
 function Get-TriggerPaths([PSCustomObject]$AllPackageProperties) {
     $existingTriggeringPaths = @()
@@ -447,7 +472,8 @@ function Get-PrPkgProperties([string]$InputDiffJson) {
     # finally, if we have gotten all the way here and we still don't have any packages, we should include the template service
     # packages. We should never return NO validation.
     if ($packagesWithChanges.Count -eq 0) {
-        $packagesWithChanges += ($allPackageProperties | Where-Object { $_.ServiceDirectory -eq "template" })
+        # most of our languages use `template` as the service directory for the template service, but `go` uses `template/aztemplate`.
+        $packagesWithChanges += ($allPackageProperties | Where-Object { $_.ServiceDirectory -eq "template"-or $_.ServiceDirectory -eq "template/aztemplate" })
         foreach ($package in $packagesWithChanges) {
             $package.IncludedForValidation = $true
         }

eng/common/scripts/logging.ps1

@@ -35,7 +35,7 @@ function LogWarning {
     Write-Host ("##vso[task.LogIssue type=warning;]$args" -replace "`n", "%0D%0A")
   }
   elseif (Test-SupportsGitHubLogging) {
-    Write-Warning ("::warning::$args" -replace "`n", "%0D%0A")
+    Write-Host ("::warning::$args" -replace "`n", "%0D%0A")
   }
   else {
     Write-Warning "$args"
@@ -56,7 +56,7 @@ function LogErrorForFile($file, $errorString)
     Write-Host ("##vso[task.logissue type=error;sourcepath=$file;linenumber=1;columnnumber=1;]$errorString" -replace "`n", "%0D%0A")
   }
   elseif (Test-SupportsGitHubLogging) {
-    Write-Error ("::error file=$file,line=1,col=1::$errorString" -replace "`n", "%0D%0A")
+    Write-Host ("::error file=$file,line=1,col=1::$errorString" -replace "`n", "%0D%0A")
   }
   else {
     Write-Error "[Error in file $file]$errorString"
@@ -68,7 +68,7 @@ function LogError {
     Write-Host ("##vso[task.LogIssue type=error;]$args" -replace "`n", "%0D%0A")
   }
   elseif (Test-SupportsGitHubLogging) {
-    Write-Error ("::error::$args" -replace "`n", "%0D%0A")
+    Write-Host ("::error::$args" -replace "`n", "%0D%0A")
   }
   else {
     Write-Error "$args"
@@ -80,7 +80,7 @@ function LogDebug {
     Write-Host "[debug]$args"
   }
   elseif (Test-SupportsGitHubLogging) {
-    Write-Debug "::debug::$args"
+    Write-Host "::debug::$args"
   }
   else {
     Write-Debug "$args"

eng/pipelines/templates/stages/1es-redirect.yml

@@ -21,7 +21,7 @@ extends:
     template: v1/1ES.Official.PipelineTemplate.yml@1ESPipelineTemplates
     parameters:
       settings:
-          skipBuildTagsForGitHubPullRequests: true
+        skipBuildTagsForGitHubPullRequests: true
       sdl:
         ${{ if and(parameters.GenerateBaselines, eq(variables['Build.SourceBranchName'], 'main'), eq(variables['System.TeamProject'], 'internal')) }}:
           autobaseline:
@@ -40,6 +40,8 @@ extends:
   ${{ else }}:
     template: v1/1ES.Unofficial.PipelineTemplate.yml@1ESPipelineTemplates
     parameters:
+      settings:
+        skipBuildTagsForGitHubPullRequests: true
       sdl: # SDLSources stage still runs even all tools are disabled. 1es team uses it for other purposes and they have the following work item to remove the stage.
         enableAllTools: false # https://dev.azure.com/mseng/1ES/_workitems/edit/2253084
         sourceAnalysisPool:

eng/tools/typespec-requirement/test/typespec-requirement.test.ts

@@ -11,7 +11,12 @@ async function checkAllUnder(path: string, responseCache?: string) {
     command += ` -_ResponseCache ${responseCache}`;
   }
 
-  return await execa("pwsh", ["-Command", command], { cwd: repoRoot, reject: false });
+  const result = await execa("pwsh", ["-Command", command], { cwd: repoRoot, reject: false });
+  return {
+    // Merge stdout and stderr, since script writes to stdout in CI but stderr on dev machine
+    stdout: result.stdout + result.stderr,
+    exitCode: result.exitCode,
+  };
 }
 
 test.concurrent("No files to check", async ({ expect }) => {
@@ -36,9 +41,9 @@ test.concurrent("Parse error", async ({ expect }) => {
 });
 
 test.concurrent("No tspconfig.yaml", async ({ expect }) => {
-  const { stderr, exitCode } = await checkAllUnder("specification/no-tspconfig");
+  const { stdout, exitCode } = await checkAllUnder("specification/no-tspconfig");
 
-  expect(stderr).toContain("no files named 'tspconfig.yaml'");
+  expect(stdout).toContain("no files named 'tspconfig.yaml'");
   expect(exitCode).toBe(1);
 });
 
@@ -74,12 +79,12 @@ test.concurrent("Hand-written, does not exist in main", async ({ expect }) => {
 });
 
 test.concurrent("Hand-written, unexpected response checking main", async ({ expect }) => {
-  const { stdout, stderr, exitCode } = await checkAllUnder(
+  const { stdout, exitCode } = await checkAllUnder(
     "specification/hand-written",
     '@{"https://github.com/Azure/azure-rest-api-specs/tree/main/specification/hand-written/resource-manager/Microsoft.HandWritten/stable"=519}',
   );
 
   expect(stdout).toContain("was not generated from TypeSpec");
-  expect(stderr).toContain("Unexpected response");
+  expect(stdout).toContain("Unexpected response");
   expect(exitCode).toBe(1);
 });

eng/tools/typespec-validation/package.json

@@ -8,8 +8,10 @@
   },
   "dependencies": {
     "globby": "^14.0.1",
+    "picocolors": "^1.1.1",
     "simple-git": "^3.24.0",
     "suppressions": "file:../suppressions",
+    "strip-ansi": "^7.1.0",
     "yaml": "^2.4.2"
   },
   "devDependencies": {