Skip to content

Commit af8b1b9

Browse files
tianleiwutianleiwu
committed
[Build] Fix DML Nuget Pipeline
1 parent 735e69a commit af8b1b9

File tree

5 files changed

+159
-15
lines changed

5 files changed

+159
-15
lines changed

‎tools/ci_build/github/azure-pipelines/dml-nuget-packaging.yml‎

Lines changed: 11 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -75,9 +75,13 @@ extends:
7575
DoNugetPack: 'true'
7676
DoEsrp: ${{ parameters.DoEsrp }}
7777
NuPackScript: |
78+
python -m pip install setuptools
7879
msbuild $(Build.SourcesDirectory)\csharp\OnnxRuntime.CSharp.proj /p:Configuration=RelWithDebInfo /t:CreatePackage /p:OrtPackageId=Microsoft.ML.OnnxRuntime.DirectML /p:IsReleaseBuild=${{ parameters.IsReleaseBuild }} /p:CurrentData=$(BuildDate) /p:CurrentTime=$(BuildTime)
80+
if errorlevel 1 exit /b 1
7981
copy $(Build.SourcesDirectory)\csharp\src\Microsoft.ML.OnnxRuntime\bin\RelWithDebInfo\*.nupkg $(Build.ArtifactStagingDirectory)
80-
copy $(Build.BinariesDirectory)\RelWithDebInfo\RelWithDebInfo\*.nupkg $(Build.ArtifactStagingDirectory)
82+
if errorlevel 1 exit /b 1
83+
powershell -ExecutionPolicy Bypass -File $(Build.SourcesDirectory)\tools\ci_build\github\windows\select_dml_package.ps1 -SourceDir "$(Build.BinariesDirectory)\RelWithDebInfo\RelWithDebInfo" -IsReleaseBuild "${{ parameters.IsReleaseBuild }}" -Action copy -DestinationDir "$(Build.ArtifactStagingDirectory)"
84+
if errorlevel 1 exit /b 1
8185
mkdir $(Build.ArtifactStagingDirectory)\testdata
8286
copy $(Build.BinariesDirectory)\RelWithDebInfo\RelWithDebInfo\custom_op_library.* $(Build.ArtifactStagingDirectory)\testdata
8387
@@ -94,13 +98,16 @@ extends:
9498
DoEsrp: ${{ parameters.DoEsrp }}
9599
RunTests: 'false'
96100
NuPackScript: |
101+
python -m pip install setuptools
97102
msbuild $(Build.SourcesDirectory)\csharp\OnnxRuntime.CSharp.proj /p:Configuration=RelWithDebInfo /p:TargetArchitecture=arm64 /t:CreatePackage /p:OrtPackageId=Microsoft.ML.OnnxRuntime.DirectML /p:IsReleaseBuild=${{ parameters.IsReleaseBuild }}
98-
cd $(Build.BinariesDirectory)\RelWithDebInfo\RelWithDebInfo\
99-
ren Microsoft.ML.OnnxRuntime.DirectML.* win-dml-arm64.zip
103+
if errorlevel 1 exit /b 1
104+
powershell -ExecutionPolicy Bypass -File $(Build.SourcesDirectory)\tools\ci_build\github\windows\select_dml_package.ps1 -SourceDir "$(Build.BinariesDirectory)\RelWithDebInfo\RelWithDebInfo" -IsReleaseBuild "${{ parameters.IsReleaseBuild }}" -Action rename -NewName "win-dml-arm64.zip"
105+
if errorlevel 1 exit /b 1
100106
copy $(Build.BinariesDirectory)\RelWithDebInfo\RelWithDebInfo\win-dml-arm64.zip $(Build.ArtifactStagingDirectory)
107+
if errorlevel 1 exit /b 1
101108
mkdir $(Build.ArtifactStagingDirectory)\testdata
102109
copy $(Build.BinariesDirectory)\RelWithDebInfo\RelWithDebInfo\custom_op_library.* $(Build.ArtifactStagingDirectory)\testdata
103110
104111
- template: stages/nuget_dml_packaging_stage.yml
105112
parameters:
106-
DoEsrp: ${{ parameters.DoEsrp }}
113+
DoEsrp: ${{ parameters.DoEsrp }}

‎tools/ci_build/github/azure-pipelines/templates/validate-package.yml‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@ parameters:
44
PackageType: ''
55
PackageName: ''
66
PackagePath: ''
7+
IsReleaseBuild: false
78
ScriptPath: '$(Build.SourcesDirectory)/tools/nuget/validate_package.py'
89
workingDirectory: "$(Build.BinariesDirectory)"
910

@@ -17,5 +18,5 @@ steps:
1718
displayName: 'Validate Package'
1819
inputs:
1920
scriptPath: '${{parameters.ScriptPath}}'
20-
arguments: '--package_type ${{parameters.PackageType}} --package_name ${{parameters.PackageName}} --package_path ${{parameters.PackagePath}} --platforms_supported ${{parameters.PlatformsSupported}} --verify_nuget_signing ${{parameters.VerifyNugetSigning}}'
21+
arguments: '--package_type ${{parameters.PackageType}} --package_name ${{parameters.PackageName}} --package_path ${{parameters.PackagePath}} --platforms_supported ${{parameters.PlatformsSupported}} --verify_nuget_signing ${{parameters.VerifyNugetSigning}} --is_release_build ${{parameters.IsReleaseBuild}}'
2122
workingDirectory: ${{parameters.workingDirectory}}

‎tools/ci_build/github/windows/bundle_dml_package.ps1‎

Lines changed: 48 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -27,17 +27,27 @@ $arm64ExtractPath = "win-dml-arm64-unzipped"
2727
Write-Host "Extracting $arm64ZipFile to $arm64ExtractPath..."
2828
& $sevenZipPath x $arm64ZipFile -o"$arm64ExtractPath" -y
2929

30+
# Debug: List contents of extracted arm64 zip
31+
Write-Host "Contents of $arm64ExtractPath (recursive):"
32+
Get-ChildItem -Path $arm64ExtractPath -Recurse | ForEach-Object { Write-Host " - $($_.FullName)" }
33+
3034
# 2. Find the target NuGet package.
3135
# It finds all .nupkg files that do not contain "Managed" in their name.
32-
$nupkgFiles = Get-ChildItem -Path . -Recurse -Filter *.nupkg | Where-Object { $_.Name -notlike "*Managed*" }
36+
$nupkgFiles = Get-ChildItem -Path . -Filter *.nupkg | Where-Object { ($_.Name -notlike "*Managed*") -and ($_.Name -notlike "*.symbols.nupkg") }
37+
38+
Write-Host "Found $($nupkgFiles.Count) candidate nupkg file(s) for bundling:"
39+
$nupkgFiles | ForEach-Object { Write-Host " - $($_.FullName)" }
3340

34-
# 3. Validate that exactly one package was found.
35-
if ($nupkgFiles.Count -ne 1) {
36-
Write-Error "Error: Expected to find exactly one non-managed NuGet package, but found $($nupkgFiles.Count)."
41+
# 3. Select the best package (shortest name prefers Release over Dev, and Main over Symbols)
42+
if ($nupkgFiles.Count -eq 0) {
43+
Write-Error "Error: No matching NuGet packages found to bundle into."
3744
exit 1
3845
}
39-
$nupkg = $nupkgFiles[0]
40-
Write-Host "Found package to process: $($nupkg.Name)"
46+
if ($nupkgFiles.Count -gt 1) {
47+
Write-Warning "Found multiple packages. Selecting the one with the shortest filename as the target for bundling."
48+
}
49+
$nupkg = $nupkgFiles | Sort-Object {$_.Name.Length} | Select-Object -First 1
50+
Write-Host "Selected target package: $($nupkg.Name)"
4151

4252
# 4. Validate the package name matches the expected format.
4353
if ($nupkg.Name -notlike "Microsoft.ML.OnnxRuntime.DirectML*.nupkg") {
@@ -61,14 +71,36 @@ New-Item -ItemType Directory -Path $tempDir | Out-Null
6171
Write-Host "Extracting $($nupkg.Name) to $tempDir..."
6272
& $sevenZipPath x $nupkg.FullName -o"$tempDir" -y
6373

74+
# Debug: Print the .nuspec content
75+
$nuspecFile = Get-ChildItem -Path $tempDir -Filter *.nuspec | Select-Object -First 1
76+
if ($nuspecFile) {
77+
Write-Host "Found manifest: $($nuspecFile.FullName)"
78+
Write-Host "--- Manifest Content ---"
79+
Get-Content $nuspecFile.FullName | ForEach-Object { Write-Host $_ }
80+
Write-Host "------------------------"
81+
}
82+
83+
# Debug: List contents of extracted target nupkg
84+
Write-Host "Contents of $tempDir (recursive):"
85+
Get-ChildItem -Path $tempDir -Recurse | ForEach-Object { Write-Host " - $($_.FullName)" }
86+
6487
# Step B: Create the new runtime directory structure.
6588
$newRuntimePath = Join-Path $tempDir "runtimes\win-arm64\native"
89+
Write-Host "Ensuring destination path exists: $newRuntimePath"
6690
New-Item -ItemType Directory -Path $newRuntimePath -Force | Out-Null
6791

6892
# Step C: Copy the ARM64 binaries into the new structure.
6993
$arm64SourcePath = Join-Path . "$arm64ExtractPath\runtimes\win-arm64\native"
70-
Write-Host "Copying ARM64 binaries from $arm64SourcePath to $newRuntimePath..."
71-
Copy-Item -Path "$arm64SourcePath\*" -Destination $newRuntimePath -Recurse -Force
94+
if (Test-Path $arm64SourcePath) {
95+
Write-Host "Copying ARM64 binaries from $arm64SourcePath to $newRuntimePath..."
96+
$filesToCopy = Get-ChildItem -Path "$arm64SourcePath\*"
97+
Write-Host "Files found in source: $($filesToCopy.Count)"
98+
$filesToCopy | ForEach-Object { Write-Host " -> $($_.Name)" }
99+
Copy-Item -Path "$arm64SourcePath\*" -Destination $newRuntimePath -Recurse -Force
100+
} else {
101+
Write-Error "Error: ARM64 source path not found: $arm64SourcePath. Bailing out to avoid creating a broken package."
102+
exit 1
103+
}
72104

73105
# Step D: Delete the original nupkg file.
74106
Remove-Item -Path $nupkg.FullName -Force
@@ -79,6 +111,13 @@ Push-Location $tempDir
79111
& $sevenZipPath a -tzip "$($nupkg.FullName)" ".\" -r
80112
Pop-Location
81113

114+
# Debug: Check final nupkg existence
115+
if (Test-Path $nupkg.FullName) {
116+
Write-Host "Final package created successfully: $($nupkg.FullName)"
117+
$finalSize = (Get-Item $nupkg.FullName).Length
118+
Write-Host "Final package size: $finalSize bytes"
119+
}
120+
82121
# --- Cleanup and Final Steps ---
83122
Write-Host "Cleaning up temporary directory $tempDir..."
84123
Remove-Item -Recurse -Force $tempDir
@@ -91,4 +130,4 @@ Write-Host "Copying final artifact to $ArtifactStagingDirectory..."
91130
Copy-Item -Path ".\Microsoft.ML.OnnxRuntime.DirectML*.nupkg" -Destination $ArtifactStagingDirectory -Force
92131

93132
Write-Host "---"
94-
Write-Host "Script completed successfully."
133+
Write-Host "Script completed successfully."

‎tools/ci_build/github/windows/select_dml_package.ps1‎

Lines changed: 86 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,86 @@
1+
# select_dml_package.ps1
2+
# Helper script to select the correct DML NuGet package based on build type
3+
# Usage: select_dml_package.ps1 -SourceDir <path> -IsReleaseBuild <true|false> -Action <copy|rename> [-DestinationDir <path>] [-NewName <name>]
4+
5+
param(
6+
[Parameter(Mandatory=$true)]
7+
[string]$SourceDir,
8+
9+
[Parameter(Mandatory=$true)]
10+
[string]$IsReleaseBuild,
11+
12+
[Parameter(Mandatory=$true)]
13+
[ValidateSet("copy", "rename")]
14+
[string]$Action,
15+
16+
[Parameter(Mandatory=$false)]
17+
[string]$DestinationDir,
18+
19+
[Parameter(Mandatory=$false)]
20+
[string]$NewName
21+
)
22+
23+
$ErrorActionPreference = "Stop"
24+
25+
Write-Host "Searching for packages in: $SourceDir"
26+
Write-Host "IsReleaseBuild: $IsReleaseBuild"
27+
Write-Host "Action: $Action"
28+
29+
# Convert string to boolean
30+
$isRelease = [System.Convert]::ToBoolean($IsReleaseBuild)
31+
32+
# Find all matching packages
33+
$allPackages = Get-ChildItem -Path $SourceDir -Filter "Microsoft.ML.OnnxRuntime.DirectML.*.nupkg"
34+
Write-Host "Found $($allPackages.Count) total package(s):"
35+
$allPackages | ForEach-Object { Write-Host " - $($_.Name)" }
36+
37+
# Filter packages based on build type
38+
$filteredPackages = $allPackages | Where-Object {
39+
$name = $_.Name
40+
$isSymbols = $name -like "*symbols*"
41+
$isDev = $name -like "*-dev*"
42+
43+
if ($isSymbols) {
44+
return $false
45+
}
46+
47+
if ($isRelease) {
48+
return -not $isDev
49+
} else {
50+
return $isDev
51+
}
52+
}
53+
54+
Write-Host "After filtering (isRelease=$isRelease), found $($filteredPackages.Count) matching package(s):"
55+
$filteredPackages | ForEach-Object { Write-Host " - $($_.Name)" }
56+
57+
if ($filteredPackages.Count -eq 0) {
58+
Write-Error "No matching package found!"
59+
exit 1
60+
}
61+
62+
# Select the first matching package (sorted by name length for consistency)
63+
$selectedPackage = $filteredPackages | Sort-Object { $_.Name.Length } | Select-Object -First 1
64+
Write-Host "Selected package: $($selectedPackage.FullName)"
65+
66+
# Perform the action
67+
if ($Action -eq "copy") {
68+
if (-not $DestinationDir) {
69+
Write-Error "DestinationDir is required for copy action"
70+
exit 1
71+
}
72+
Write-Host "Copying to: $DestinationDir"
73+
Copy-Item -Path $selectedPackage.FullName -Destination $DestinationDir -Force
74+
Write-Host "Copy successful."
75+
}
76+
elseif ($Action -eq "rename") {
77+
if (-not $NewName) {
78+
Write-Error "NewName is required for rename action"
79+
exit 1
80+
}
81+
Write-Host "Renaming to: $NewName"
82+
Rename-Item -Path $selectedPackage.FullName -NewName $NewName -Force
83+
Write-Host "Rename successful."
84+
}
85+
86+
exit 0

‎tools/nuget/validate_package.py‎

Lines changed: 12 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -67,6 +67,10 @@ def parse_arguments():
6767
"--verify_nuget_signing",
6868
help="Flag indicating if Nuget package signing is to be verified. Only accepts 'true' or 'false'",
6969
)
70+
parser.add_argument(
71+
"--is_release_build",
72+
help="Flag indicating if validating a release build or dev build. Only accepts 'true' or 'false'",
73+
)
7074

7175
return parser.parse_args()
7276

@@ -285,7 +289,14 @@ def validate_zip(args):
285289

286290
def validate_nuget(args):
287291
files = glob.glob(os.path.join(args.package_path, args.package_name))
288-
nuget_packages_found_in_path = [i for i in files if i.endswith(".nupkg") and "Managed" not in i]
292+
is_release_build = args.is_release_build and args.is_release_build.lower() == "true"
293+
nuget_packages_found_in_path = [
294+
i
295+
for i in files
296+
if i.endswith(".nupkg")
297+
and "Managed" not in i
298+
and ((is_release_build and "-dev" not in i) or (not is_release_build and "-dev" in i))
299+
]
289300
if len(nuget_packages_found_in_path) != 1:
290301
print("Nuget packages found in path: ")
291302
print(nuget_packages_found_in_path)

0 commit comments

Comments
 (0)