Skip to content

Commit 0240530

Browse files
edgchen1edgchen1
committed
reorder pipeline steps, clean up
1 parent 3831cdb commit 0240530

File tree

2 files changed

+142
-158
lines changed

2 files changed

+142
-158
lines changed

‎tools/ci_build/github/azure-pipelines/templates/setup-build-tools.yml‎

Lines changed: 8 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@ parameters:
88
- name: python_version
99
type: string
1010
default: '3.12'
11-
11+
1212
- name: action_version
1313
type: string
1414
default: 'v0.0.9'
@@ -17,17 +17,9 @@ steps:
1717
- template: telemetry-steps.yml
1818

1919
- task: UsePythonVersion@0
20-
displayName: 'Use Python ${{ parameters.host_cpu_arch }} (macOS)'
21-
condition: and(succeeded(), eq(variables['Agent.OS'], 'Darwin'))
22-
inputs:
23-
versionSpec: ${{ parameters.python_version }}
24-
architecture: ${{ parameters.host_cpu_arch }}
25-
26-
- task: UsePythonVersion@0
27-
displayName: 'Use Python ${{ parameters.host_cpu_arch }} (non-macOS)'
28-
condition: and(succeeded(), ne(variables['Agent.OS'], 'Darwin'))
20+
displayName: 'Use Python ${{ parameters.python_version }} ${{ parameters.host_cpu_arch }}'
2921
inputs:
30-
versionSpec: ${{ parameters.python_version }}
22+
versionSpec: ${{ parameters.python_version }}
3123
architecture: ${{ parameters.host_cpu_arch }}
3224

3325
- task: PipAuthenticate@1
@@ -45,17 +37,18 @@ steps:
4537
displayName: 'Setup Latest Node.js v20 (Win)'
4638
condition: and(succeeded(), eq(variables['Agent.OS'], 'Windows_NT'))
4739
inputs:
48-
filePath: '$(System.DefaultWorkingDirectory)\tools\ci_build\github\windows\setup_nodejs.ps1'
40+
filePath: '$(System.DefaultWorkingDirectory)\tools\ci_build\github\windows\setup_nodejs.ps1'
4941
arguments: '-MajorVersion 22'
5042

5143
- script: |
5244
node -v
5345
npm -v
54-
5546
condition: and(succeeded(), eq(variables['Agent.OS'], 'Windows_NT'))
56-
displayName: 'Verify Node.js Version'
47+
displayName: 'Display Node.js Version'
5748

58-
- script: python3 -m pip install requests
49+
- script: |
50+
python3 -m pip install requests
51+
displayName: 'Install Python requests package'
5952

6053
- task: PythonScript@0
6154
displayName: 'Run GitHub Action via Python Wrapper'

‎tools/ci_build/github/azure-pipelines/templates/stages/mac-ios-packaging-build-stage.yml‎

Lines changed: 134 additions & 143 deletions
Original file line numberDiff line numberDiff line change
@@ -5,9 +5,9 @@ parameters:
55
- release
66
- normal
77

8-
# Note: Keep the Xcode version and iOS simulator version compatible.
9-
# Check the table here to see what iOS simulator versions are supported by a particular Xcode version:
10-
# https://developer.apple.com/support/xcode/
8+
# Note: Keep the Xcode version and iOS simulator version compatible.
9+
# Check the table here to see what iOS simulator versions are supported by a particular Xcode version:
10+
# https://developer.apple.com/support/xcode/
1111

1212
- name: xcodeVersion
1313
type: string
@@ -24,130 +24,110 @@ parameters:
2424
default: "onnxruntime"
2525

2626
stages:
27-
# TODO commented out for testing
28-
# - stage: BuildFrameworks
29-
# dependsOn: []
30-
31-
# jobs:
32-
# - job: SetUpBuildFrameworkJobMatrix
33-
34-
# steps:
35-
# - task: PythonScript@0
36-
# name: SetVariables
37-
# inputs:
38-
# scriptSource: "inline"
39-
# script: |
40-
# import json
41-
# import sys
42-
43-
# utils_path = "$(Build.SourcesDirectory)/tools/ci_build/github/apple"
44-
# sys.path.insert(0, utils_path)
45-
46-
# from build_settings_utils import parse_build_settings_file, get_sysroot_arch_pairs
47-
48-
# build_settings_file = "${{ parameters.buildSettingsFile }}"
49-
# build_settings = parse_build_settings_file(build_settings_file)
50-
# sysroot_arch_pairs = get_sysroot_arch_pairs(build_settings)
51-
52-
# job_matrix = {}
53-
# for sysroot, arch in sysroot_arch_pairs:
54-
# identifier = f"{sysroot}_{arch}"
55-
# job_matrix[identifier] = {
56-
# "sysroot": sysroot,
57-
# "arch": arch,
58-
# }
59-
60-
# job_matrix_json = json.dumps(job_matrix)
61-
62-
# print(f"Build framework job matrix:\n{job_matrix_json}")
63-
# print(f"##vso[task.setvariable variable=BuildFrameworkJobMatrix;isOutput=true]{job_matrix_json}")
64-
# displayName: "Generate build framework job matrix"
65-
66-
# - job: BuildFramework
67-
# dependsOn: SetUpBuildFrameworkJobMatrix
68-
69-
# strategy:
70-
# maxParallel: 8
71-
# matrix: $[ dependencies.SetUpBuildFrameworkJobMatrix.outputs['SetVariables.BuildFrameworkJobMatrix'] ]
72-
73-
# timeoutInMinutes: 120
74-
75-
# steps:
76-
# - template: ../setup-build-tools.yml
77-
# parameters:
78-
# host_cpu_arch: arm64
79-
80-
# - template: ../use-xcode-version.yml
81-
# parameters:
82-
# xcodeVersion: ${{ parameters.xcodeVersion }}
83-
84-
# - script: |
85-
# pip install -r tools/ci_build/github/apple/ios_packaging/requirements.txt
86-
# displayName: "Install Python requirements"
87-
88-
# - script: |
89-
# python tools/ci_build/github/apple/build_apple_framework.py \
90-
# --build_dir "$(Build.BinariesDirectory)/build" \
91-
# --only_build_single_sysroot_arch_framework "$(sysroot)" "$(arch)" \
92-
# --record_sysroot_arch_framework_build_outputs_to_file "$(Build.BinariesDirectory)/build_outputs.txt" \
93-
# "${{ parameters.buildSettingsFile }}"
94-
# displayName: "Build framework for $(sysroot)/$(arch)"
95-
96-
# - script: |
97-
# set -e
98-
99-
# BUILD_OUTPUTS_FILE="$(Build.BinariesDirectory)/build_outputs.txt"
100-
# BUILD_DIR="$(Build.BinariesDirectory)/build"
101-
102-
# pushd "${BUILD_DIR}"
103-
# zip \
104-
# --recurse-paths \
105-
# --symlinks \
106-
# --names-stdin \
107-
# "$(Build.ArtifactStagingDirectory)/build.zip" \
108-
# < "${BUILD_OUTPUTS_FILE}"
109-
# popd
110-
# displayName: "Create framework build archive artifact"
111-
112-
# - task: 1ES.PublishPipelineArtifact@1
113-
# inputs:
114-
# path: $(Build.ArtifactStagingDirectory)
115-
# artifact: framework_$(sysroot)_$(arch)
116-
# displayName: "Publish artifact - framework_$(sysroot)_$(arch)"
117-
118-
- stage: AssemblePackageAndTest
119-
# TODO commented out for testing
120-
# dependsOn: BuildFrameworks
121-
122-
variables:
123-
cPodName: ${{ parameters.podNamePrefix }}-c
124-
objcPodName: ${{ parameters.podNamePrefix }}-objc
27+
# This stage builds the individual sysroot/arch frameworks.
28+
# Since the number of framework build jobs is dynamic, these jobs are put into this stage to allow a later stage to
29+
# wait for all of them to complete.
30+
- stage: BuildFrameworks
31+
dependsOn: []
12532

12633
jobs:
127-
- job: AssemblePackageAndTest
34+
- job: SetUpBuildFrameworkJobMatrix
12835

12936
steps:
130-
- task: DownloadPipelineArtifact@2
37+
- task: PythonScript@0
38+
name: SetVariables
13139
inputs:
132-
itemPattern: framework_*/build.zip
133-
targetPath: $(Build.BinariesDirectory)/artifacts
134-
# TODO for testing
135-
buildType: specific
136-
project: Lotus
137-
definition: "995"
138-
runVersion: specific
139-
pipelineId: "1078531"
40+
scriptSource: "inline"
41+
script: |
42+
import json
43+
import sys
44+
45+
utils_path = "$(Build.SourcesDirectory)/tools/ci_build/github/apple"
46+
sys.path.insert(0, utils_path)
47+
48+
from build_settings_utils import parse_build_settings_file, get_sysroot_arch_pairs
49+
50+
build_settings_file = "${{ parameters.buildSettingsFile }}"
51+
build_settings = parse_build_settings_file(build_settings_file)
52+
sysroot_arch_pairs = get_sysroot_arch_pairs(build_settings)
53+
54+
job_matrix = {}
55+
for sysroot, arch in sysroot_arch_pairs:
56+
identifier = f"{sysroot}_{arch}"
57+
job_matrix[identifier] = {
58+
"sysroot": sysroot,
59+
"arch": arch,
60+
}
61+
62+
job_matrix_json = json.dumps(job_matrix)
63+
64+
print(f"Build framework job matrix:\n{job_matrix_json}")
65+
print(f"##vso[task.setvariable variable=BuildFrameworkJobMatrix;isOutput=true]{job_matrix_json}")
66+
displayName: "Generate build framework job matrix"
67+
68+
- job: BuildFramework
69+
dependsOn: SetUpBuildFrameworkJobMatrix
70+
71+
strategy:
72+
maxParallel: 8
73+
matrix: $[ dependencies.SetUpBuildFrameworkJobMatrix.outputs['SetVariables.BuildFrameworkJobMatrix'] ]
74+
75+
timeoutInMinutes: 120
76+
77+
steps:
78+
- template: ../setup-build-tools.yml
79+
parameters:
80+
host_cpu_arch: arm64
81+
82+
- template: ../use-xcode-version.yml
83+
parameters:
84+
xcodeVersion: ${{ parameters.xcodeVersion }}
85+
86+
- script: |
87+
pip install -r tools/ci_build/github/apple/ios_packaging/requirements.txt
88+
displayName: "Install Python requirements"
89+
90+
- script: |
91+
python tools/ci_build/github/apple/build_apple_framework.py \
92+
--build_dir "$(Build.BinariesDirectory)/build" \
93+
--only_build_single_sysroot_arch_framework "$(sysroot)" "$(arch)" \
94+
--record_sysroot_arch_framework_build_outputs_to_file "$(Build.BinariesDirectory)/build_outputs.txt" \
95+
"${{ parameters.buildSettingsFile }}"
96+
displayName: "Build framework for $(sysroot)/$(arch)"
14097
14198
- script: |
14299
set -e
143100
101+
BUILD_OUTPUTS_FILE="$(Build.BinariesDirectory)/build_outputs.txt"
144102
BUILD_DIR="$(Build.BinariesDirectory)/build"
145-
mkdir -p "${BUILD_DIR}"
146-
find "$(Build.BinariesDirectory)/artifacts" -name "build.zip" -exec unzip {} -d "${BUILD_DIR}" \;
147103
148-
find "${BUILD_DIR}"
149-
displayName: Extract framework build archive artifacts to build directory
104+
cd "${BUILD_DIR}"
105+
zip \
106+
--recurse-paths \
107+
--symlinks \
108+
--names-stdin \
109+
"$(Build.ArtifactStagingDirectory)/build.zip" \
110+
< "${BUILD_OUTPUTS_FILE}"
111+
displayName: "Create framework build archive artifact"
112+
113+
- task: 1ES.PublishPipelineArtifact@1
114+
inputs:
115+
path: $(Build.ArtifactStagingDirectory)
116+
artifact: framework_$(sysroot)_$(arch)
117+
displayName: "Publish artifact - framework_$(sysroot)_$(arch)"
150118

119+
# This stage assembles the frameworks built in the previous stage into a packaging artifacts and tests them.
120+
- stage: AssemblePackageAndTest
121+
dependsOn: BuildFrameworks
122+
123+
variables:
124+
cPodName: ${{ parameters.podNamePrefix }}-c
125+
objcPodName: ${{ parameters.podNamePrefix }}-objc
126+
127+
jobs:
128+
- job: AssemblePackageAndTest
129+
130+
steps:
151131
- script: |
152132
set -e
153133
@@ -168,7 +148,7 @@ stages:
168148
# Do not output ##vso[] commands with `set -x` or they may be parsed again and include a trailing quote.
169149
set +x
170150
echo "##vso[task.setvariable variable=ortPodVersion;]${VERSION}"
171-
echo "ortPodVersion : ${ortPodVersion}, VERSION : ${VERSION}"
151+
echo "ortPodVersion: ${VERSION}"
172152
displayName: "Set ortPodVersion variable"
173153
174154
- task: InstallAppleCertificate@2
@@ -197,6 +177,19 @@ stages:
197177
pip install -r tools/ci_build/github/apple/ios_packaging/requirements.txt
198178
displayName: "Install Python requirements"
199179
180+
- task: DownloadPipelineArtifact@2
181+
inputs:
182+
itemPattern: framework_*/build.zip
183+
targetPath: $(Build.BinariesDirectory)/artifacts
184+
displayName: "Download framework build archive artifacts"
185+
186+
- script: |
187+
set -e
188+
BUILD_DIR="$(Build.BinariesDirectory)/build"
189+
mkdir -p "${BUILD_DIR}"
190+
find "$(Build.BinariesDirectory)/artifacts" -name "build.zip" -exec unzip {} -d "${BUILD_DIR}" \;
191+
displayName: "Extract framework build archive artifacts to build directory"
192+
200193
# Assemble the xcframework and the pod packages.
201194
# The frameworks from the BuildFrameworks stage should already be in the build directory.
202195
- script: |
@@ -211,33 +204,6 @@ stages:
211204
env:
212205
ORT_GET_SIMULATOR_DEVICE_INFO_REQUESTED_RUNTIME_VERSION: ${{ parameters.iosSimulatorRuntimeVersion }}
213206
214-
- script: |
215-
set -e -x
216-
217-
mkdir -p "$(Build.ArtifactStagingDirectory)/package"
218-
219-
for POD_NAME in "${{ variables.cPodName}}" "${{ variables.objcPodName }}";
220-
do
221-
./tools/ci_build/github/apple/assemble_apple_packaging_artifacts.sh \
222-
"$(Build.BinariesDirectory)/staging" \
223-
"$(Build.ArtifactStagingDirectory)/package" \
224-
"${POD_NAME}" \
225-
"$(ortPodVersion)"
226-
done
227-
228-
# copy over helper script for use in release pipeline
229-
cp tools/ci_build/github/apple/package_release_tasks.py "$(Build.ArtifactStagingDirectory)/package"
230-
displayName: "Assemble packaging artifacts"
231-
232-
# Publish package artifact
233-
- task: 1ES.PublishPipelineArtifact@1
234-
inputs:
235-
path: $(Build.ArtifactStagingDirectory)/package
236-
artifact: ios_packaging_artifacts_full
237-
displayName: "Publish artifact - ios_packaging_artifacts_full"
238-
239-
# Run more tests on the package
240-
241207
- script: |
242208
python tools/ci_build/github/apple/test_apple_packages.py \
243209
--fail_if_cocoapods_missing \
@@ -332,6 +298,31 @@ stages:
332298
BROWSERSTACK_ID: $(browserstack_username)
333299
BROWSERSTACK_TOKEN: $(browserstack_access_key)
334300
301+
- script: |
302+
set -e -x
303+
304+
mkdir -p "$(Build.ArtifactStagingDirectory)/package"
305+
306+
for POD_NAME in "${{ variables.cPodName}}" "${{ variables.objcPodName }}";
307+
do
308+
./tools/ci_build/github/apple/assemble_apple_packaging_artifacts.sh \
309+
"$(Build.BinariesDirectory)/staging" \
310+
"$(Build.ArtifactStagingDirectory)/package" \
311+
"${POD_NAME}" \
312+
"$(ortPodVersion)"
313+
done
314+
315+
# copy over helper script for use in release pipeline
316+
cp tools/ci_build/github/apple/package_release_tasks.py "$(Build.ArtifactStagingDirectory)/package"
317+
displayName: "Assemble packaging artifacts"
318+
319+
# Publish packaging artifacts
320+
- task: 1ES.PublishPipelineArtifact@1
321+
inputs:
322+
path: $(Build.ArtifactStagingDirectory)/package
323+
artifact: ios_packaging_artifacts_full
324+
displayName: "Publish artifact - ios_packaging_artifacts_full"
325+
335326
- script: |
336327
set -e -x
337328
ls -R "$(Build.ArtifactStagingDirectory)"

0 commit comments

Comments
 (0)