Ensure that eng/ based packages build, test, and analyze appropriately (Azure#24397)

* update language-settings to handle eng or sdk rooted packages in Get-AllPackageInfoFromRepo
* update analyze.yml to operate on packages instead of service directories. this fixes the confusion as to whether a package should be rooted in `eng/` vs rooted in `sdk/`. This fixes the various analyze failures for go - generator and other tools packages
* add sdk/keyvault to the checkout for eng packages
* skip eng linting
* filter out eng from smoketest invocations

Author: scbedd

eng/pipelines/templates/jobs/archetype-sdk-client.yml

@@ -153,16 +153,9 @@ extends:
               os: linux
 
             steps:
-              - template: /eng/common/pipelines/templates/steps/sparse-checkout.yml
+              - template: /eng/pipelines/templates/steps/sparse-checkout-for-servicedirectory.yml
                 parameters:
-                  ${{ if eq(parameters.ServiceDirectory, 'auto') }}:
-                    Paths:
-                      - "/*"
-                      - ".vscode/"
-                  ${{ else }}:
-                    Paths:
-                      - "sdk/${{ parameters.ServiceDirectory }}"
-                      - ".vscode/"
+                  ServiceDirectory: ${{ parameters.ServiceDirectory }}
 
               - template: /eng/common/pipelines/templates/steps/check-spelling.yml
 

eng/pipelines/templates/jobs/ci.tests.yml

@@ -52,14 +52,9 @@ jobs:
     os: ${{ parameters.OSName }}
 
   steps:
-  - template: /eng/common/pipelines/templates/steps/sparse-checkout.yml
+  - template: /eng/pipelines/templates/steps/sparse-checkout-for-servicedirectory.yml
     parameters:
-      ${{ if eq(parameters.ServiceDirectory, 'auto') }}:
-        Paths:
-          - "/*"
-      ${{ else }}:
-        Paths:
-          - "sdk/${{ parameters.ServiceDirectory }}"
+      ServiceDirectory: ${{ parameters.ServiceDirectory }}
 
   - template: /eng/common/pipelines/templates/steps/verify-agent-os.yml
     parameters:

eng/pipelines/templates/steps/analyze.yml

@@ -26,9 +26,9 @@ steps:
       $pkgs = Get-PackagesFromPackageInfo -PackageInfoFolder "$(Build.ArtifactStagingDirectory)/PackageInfo" `
         -IncludeIndirect $false -CustomCompareFunction { param($pkgProp) { return $pkgProp.CIParameters.IsSdkLibrary } }
 
-      $changedServices = @()
+      $apiviewEnabledPackages = @()
       if ($pkgs) {
-        $changedServices = $pkgs | ForEach-Object { $_.ServiceDirectory } | Sort-Object -Unique
+        $apiviewEnabledPackages = $pkgs | ForEach-Object { $_.Name } | Sort-Object -Unique
       }
       else {
         Write-Host "No sdk packages were found in package set. Skipping APIView generation."
@@ -38,7 +38,7 @@ steps:
       $failed = $false
       $directoryToPublish = Join-Path -Path $(Build.ArtifactStagingDirectory) ${{ parameters.ArtifactName }}
 
-      foreach($serviceDirectory in $changedServices) {
+      foreach($serviceDirectory in $apiviewEnabledPackages) {
         New-APIViewArtifacts `
           -ServiceDirectory $serviceDirectory `
           -OutputDirectory $(Build.ArtifactStagingDirectory) `
@@ -151,18 +151,24 @@ steps:
     retryCountOnTaskFailure: 3
 
   - pwsh: |
-      $changedServices = "$(ChangedServices)" -split ","
+      $changedPackages = "$(Packages)" -split ","
       $failed = $false
 
-      foreach($serviceDirectory in $changedServices) {
-        $modDirs = & $(Build.SourcesDirectory)/eng/scripts/get_module_dirs.ps1 "$serviceDirectory"
+      foreach($packageDirectory in $changedPackages) {
+        if ($packageDirectory.startsWith('eng')) {
+          Write-Host "skipping linting on engineering-only package: '$jpackageDirectory'"
+          continue
+        }
+
+        $modDirs = & $(Build.SourcesDirectory)/eng/scripts/get_module_dirs.ps1 "$packageDirectory"
+
         foreach ($md in $modDirs) {
           pushd $md
           Write-Host "##[command]Executing golangci-lint run -c $(System.DefaultWorkingDirectory)/eng/.golangci.yml in $md"
           golangci-lint run -c $(System.DefaultWorkingDirectory)/eng/.golangci.yml
 
           if ($LASTEXITCODE) {
-            Write-Host "golangci-lint for $serviceDirectory failed. Check above output for details."
+            Write-Host "golangci-lint for $packageDirectory failed. Check above output for details."
             $failed = $true
           }
         }
@@ -177,19 +183,19 @@ steps:
     condition: succeededOrFailed()
 
   - pwsh: |
-      $changedServices = "$(ChangedServices)" -split ","
+      $changedPackages = "$(Packages)" -split ","
       $failed = $false
 
-      foreach($serviceDirectory in $changedServices) {
-        $modDirs = & $(Build.SourcesDirectory)/eng/scripts/get_module_dirs.ps1 "$serviceDirectory"
+      foreach($packageDirectory in $changedPackages) {
+        $modDirs = & $(Build.SourcesDirectory)/eng/scripts/get_module_dirs.ps1 "$packageDirectory"
         foreach ($md in $modDirs) {
           pushd $md
           Write-Host "##[command]Executing go get -u all in $md"
           go get -u all
           $diff = (git diff -w .)
           Write-Host ($diff -join "`n")
           if ($diff.Length -gt 0) {
-            Write-Host "Failed to go get $serviceDirectory"
+            Write-Host "Failed to go get $packageDirectory"
             $failed = $true
           }
         }
@@ -204,11 +210,11 @@ steps:
     workingDirectory: $(System.DefaultWorkingDirectory)
 
   - pwsh: |
-      $changedServices = "$(ChangedServices)" -split ","
+      $changedPackages = "$(Packages)" -split ","
       $failed = $false
 
-      foreach($serviceDirectory in $changedServices) {
-        $modDirs = & $(Build.SourcesDirectory)/eng/scripts/get_module_dirs.ps1 "$serviceDirectory"
+      foreach($packageDirectory in $changedPackages) {
+        $modDirs = & $(Build.SourcesDirectory)/eng/scripts/get_module_dirs.ps1 "$packageDirectory"
         foreach ($md in $modDirs) {
           pushd $md
           Write-Host "##[command]Executing go mod tidy in $md"
@@ -217,7 +223,7 @@ steps:
           Write-Host ($diff -join "`n")
           if ($diff.Length -gt 0) {
             $failed = $true
-            Write-Host "Failed to go mod tidy $serviceDirectory"
+            Write-Host "Failed to go mod tidy $packageDirectory"
           }
         }
       }
@@ -231,9 +237,9 @@ steps:
     condition: succeededOrFailed()
 
   - pwsh: |
-      $changedServices = "$(ChangedServices)" -split ","
-      foreach($serviceDirectory in $changedServices) {
-        $modDirs = ./eng/scripts/get_module_dirs.ps1 "$serviceDirectory"
+      $packages = "$(Packages)" -split ","
+      foreach($packageDir in $packages) {
+        $modDirs = ./eng/scripts/get_module_dirs.ps1 "$packageDir"
         foreach ($md in $modDirs) {
           if (-Not $md -Match "/arm") {
             Get-ChildItem $md/doc.go
@@ -282,16 +288,16 @@ steps:
       Condition: succeededOrFailed()
 
   - pwsh: |
-      if (-not "$(ChangedServices)") {
+      if (-not "$(Packages)") {
         Write-Host "No services were changed build. Skipping doccheck."
         exit 0
       }
 
-      $changedServices = "$(ChangedServices)" -split ","
+      $changedServices = "$(Packages)" -split ","
       $failed = $false
 
       foreach ($service in $changedServices) {
-        go run . --directory "$(System.DefaultWorkingDirectory)/sdk/$service"
+        go run . --directory "$(System.DefaultWorkingDirectory)/$service"
         if ($LASTEXITCODE -ne 0) {
           $failed = $true
         }
@@ -308,7 +314,7 @@ steps:
   - template: /eng/common/pipelines/templates/steps/eng-common-workflow-enforcer.yml
 
   - pwsh: |
-      ./eng/scripts/Invoke-SmokeTests.ps1 "$(ChangedServices)"
+      ./eng/scripts/Invoke-SmokeTests.ps1 "$(Packages)"
     displayName: 'Run Nightly SmokeTests'
 
   # There are no more checks after this one, so simply removing all packageInfo that aren't shipping

eng/pipelines/templates/steps/sparse-checkout-for-servicedirectory.yml

@@ -0,0 +1,27 @@
+parameters:
+  - name: ServiceDirectory
+    type: string
+    default: ''
+
+
+steps:
+  - template: /eng/common/pipelines/templates/steps/sparse-checkout.yml
+    parameters:
+      ${{ if eq(parameters.ServiceDirectory, 'auto') }}:
+        Paths:
+          - "/*"
+          - ".vscode/"
+      ${{ else }}:
+        Paths:
+          - ${{ if startsWith(parameters.ServiceDirectory, '../') }}:
+            - ${{ replace(parameters.ServiceDirectory, '../', '') }}
+            # this is a workaround for the eng packages that have integration tests that rely on keyvault
+            # todo: transform this to a more generic solution of passed sparse checkout paths
+            - "sdk/security"
+          - ${{ else }}:
+            - "sdk/${{ parameters.ServiceDirectory }}"
+          - ".vscode/"
+
+  - pwsh: |
+      Get-ChildItem -Path "$(Build.SourcesDirectory)" -Recurse | % { Write-Host $_.FullName }
+    displayName: "Dump visible files"

eng/scripts/Invoke-DependencyCheck.ps1

@@ -47,7 +47,7 @@ $PackageDirectories = $PackageDirectories -split ","
 
 foreach ($sdk in $sdks)
 {
-    if ($sdk.Name -like "*internal*" -or (IsPackageDeprecated $sdk))
+    if ($sdk.Name -like "*internal*" -or (IsPackageDeprecated $sdk) -or $sdk.SdkType -eq "eng")
     {
         continue
     }

eng/scripts/Invoke-SmokeTests.ps1

@@ -7,7 +7,7 @@ Param(
 . (Join-Path $PSScriptRoot ".." common scripts common.ps1)
 
 $packages = Get-AllPackageInfoFromRepo $filter
-$targetServices = $packages | Where-Object { $_.CIParameters.NonShipping -eq $false } | Select-Object -Property ServiceDirectory -Unique
+$targetServices = $packages | Where-Object { $_.CIParameters.NonShipping -eq $false -and $_.SdkType -ne "eng" } | Select-Object -Property ServiceDirectory -Unique
 
 $failed = $false
 

eng/scripts/Language-Settings.ps1

@@ -133,7 +133,7 @@ function ResolveSearchPaths {
   $filters = $FilterString.Split(",")
 
   foreach($filter in $filters) {
-    if ($filter.StartsWith("sdk")) {
+    if ($filter.StartsWith("sdk") -or $filter.StartsWith("eng")) {
       $resolvedPaths += (Join-Path $RepoRoot $filter)
     }
     else {

eng/scripts/run_tests.ps1

@@ -9,12 +9,12 @@ Param(
 
 $ErrorActionPreference = 'Stop'
 
-$services = $TargetDirectories -split ","
+$directories = $TargetDirectories -split ","
 
-foreach($serviceDirectory in $services) {
-    &$PSScriptRoot/test.ps1 $serviceDirectory $testTimeout $enableRaceDetector
+foreach($serviceOrPackageDir in $directories) {
+    &$PSScriptRoot/test.ps1 $serviceOrPackageDir $testTimeout $enableRaceDetector
     if ($LASTEXITCODE) {
-        Write-Host "##[error] a failure occurred testing the directory: $serviceDirectory. Check above details for more information."
+        Write-Host "##[error] a failure occurred testing the directory: $serviceOrPackageDir. Check above details for more information."
         $failed = $true
     }
 }

eng/scripts/test.ps1

@@ -3,10 +3,9 @@ param (
     [string] $testTimeout,
     [bool] $enableRaceDetector
 )
-$targetDirectory = $serviceDirectory
-if (-not $serviceDirectory.StartsWith("sdk")) {
-    $targetDirectory = Join-Path "sdk" $serviceDirectory
-}
+. $PSScriptRoot/../common/scripts/common.ps1
+# we are passing in a single item here, so the first item is all we need as that is all there will ever be
+$targetDirectory = (ResolveSearchPaths $serviceDirectory)[0]
 Push-Location $targetDirectory
 
 if ($enableRaceDetector) {

eng/tools/generator/README.md

@@ -41,4 +41,4 @@ To authenticate this command, you need to either
 Example usage:
 ```shell
 generator issue -t $YOUR_PERSONAL_ACCESS_TOKEN > sdk-release.json
-```
+```