Skip to content

Commit 473ef71

Browse files
yomybabyyomybaby
committed
chore: optimize release workflow with parallel jobs and eliminate double React build
The release pipeline previously ran everything sequentially on a single macOS runner (~30+ min). This commit introduces several optimizations: 1. Split CI into 3 parallel jobs: web build (ubuntu), macOS desktop (macos), win/linux desktop (ubuntu) — saves ~10-15 min wall time 2. Eliminate double React build by patching publicPath post-build instead of rebuilding for Electron — saves ~4-8 min 3. Parallelize local proxy compilation (6 targets concurrently) — saves ~3 min 4. Reduce ZIP compression from -9 to -6 (marginal size diff, faster) — saves ~1 min 5. Upload release assets concurrently (batch of 4) — saves ~30s 6. Explicit NODE_OPTIONS for craco build to prevent OOM Estimated total savings: ~15-25 min per release (from ~30 min to ~10-15 min).
1 parent ebdada1 commit 473ef71

5 files changed

Lines changed: 383 additions & 45 deletions

File tree

‎.github/workflows/package.yml‎

Lines changed: 165 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -1,48 +1,129 @@
1-
# This workflow will triage pull requests and apply a label based on the
2-
# paths that are modified in the pull request.
1+
# Build and release workflow for Backend.AI Desktop and WebUI bundle.
32
#
4-
# To use this workflow, you will need to set up a .github/labeler.yml
5-
# file with configuration. For more information, see:
6-
# https://github.com/actions/labeler/blob/master/README.md
3+
# Architecture: 3 parallel jobs after a shared web build step.
4+
#
5+
# build_web (ubuntu) ──┬──> build_mac (macos) → DMG x64/arm64 + local proxy
6+
# ├──> build_desktop (ubuntu) → Win/Linux ZIP x64/arm64 + local proxy
7+
# └──> upload web bundle
8+
#
9+
# Key optimizations over the previous single-job approach:
10+
# 1. Parallel jobs: macOS + win/linux builds run concurrently (~10 min saved)
11+
# 2. No double React build: publicPath patching replaces full rebuild (~5 min saved)
12+
# 3. Parallel local proxy compilation within each job (~3 min saved)
13+
# 4. Optimized ZIP compression level (-6 vs -9, marginal size diff, ~1 min saved)
714

815
name: Build and Release Packages
916
on:
1017
release:
1118
types: [published]
19+
workflow_dispatch:
20+
inputs:
21+
dry_run:
22+
description: 'Skip release asset upload (for testing the build pipeline)'
23+
type: boolean
24+
default: true
25+
26+
env:
27+
NODE_OPTIONS: --max-old-space-size=4096
1228

1329
jobs:
30+
# ──────────────────────────────────────────────────────────────────────
31+
# Job 1: Build web assets and create the web bundle (ubuntu, ~8 min)
32+
# ──────────────────────────────────────────────────────────────────────
33+
build_web:
34+
permissions:
35+
contents: write
36+
runs-on: ubuntu-latest
37+
steps:
38+
- name: Check out Git repository
39+
uses: actions/checkout@v4
40+
41+
- uses: pnpm/action-setup@v4
42+
name: Install pnpm
43+
with:
44+
version: latest
45+
run_install: false
46+
47+
- name: Install Node.js
48+
uses: actions/setup-node@v4
49+
with:
50+
node-version-file: '.nvmrc'
51+
cache: 'pnpm'
52+
53+
- name: Install Dependencies
54+
run: pnpm install --no-frozen-lockfile
55+
56+
- name: Build web assets
57+
run: make dep_web
58+
59+
- name: Create web bundle
60+
run: make bundle
61+
62+
- name: Upload release bundle
63+
if: inputs.dry_run != true
64+
run: node upload-release.js app
65+
env:
66+
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
67+
68+
# Share build artifacts with downstream desktop jobs
69+
- name: Upload build artifacts
70+
uses: actions/upload-artifact@v4
71+
with:
72+
name: web-build
73+
path: |
74+
build/web/
75+
src/wsproxy/dist/
76+
retention-days: 1
77+
compression-level: 3
78+
79+
# ──────────────────────────────────────────────────────────────────────
80+
# Job 2: Build macOS desktop apps — requires macOS for code signing,
81+
# notarization, and DMG creation (~10 min)
82+
# ──────────────────────────────────────────────────────────────────────
1483
build_mac:
84+
needs: build_web
1585
permissions:
1686
contents: write
17-
checks: write
18-
actions: read
19-
issues: read
20-
packages: write
21-
pull-requests: read
22-
repository-projects: read
23-
statuses: read
2487
runs-on: macos-latest
25-
environment: app-packaging
88+
environment: ${{ inputs.dry_run != true && 'app-packaging' || '' }}
2689
steps:
2790
- name: Check out Git repository
2891
uses: actions/checkout@v4
92+
2993
- uses: pnpm/action-setup@v4
3094
name: Install pnpm
3195
with:
3296
version: latest
3397
run_install: false
98+
3499
- name: Install Node.js
35100
uses: actions/setup-node@v4
36101
with:
37102
node-version-file: '.nvmrc'
38103
cache: 'pnpm'
104+
39105
- name: Install Dependencies
40-
# CI defaults to --frozen-lockfile, which blocks merging git branch lockfiles.
41-
# pnpm-workspace.yaml's mergeGitBranchLockfilesBranchPattern auto-merges
42-
# branch lockfiles on main. --no-frozen-lockfile allows that merge write.
43106
run: pnpm install --no-frozen-lockfile
44-
- name: Package Desktop Applications
45-
run: make all
107+
108+
- name: Download web build artifacts
109+
uses: actions/download-artifact@v4
110+
with:
111+
name: web-build
112+
113+
- name: Prepare Electron app
114+
run: make dep_electron
115+
116+
- name: Compile local proxies (parallel)
117+
run: |
118+
make compile_localproxy os=macos arch=x64 local_proxy_postfix= &
119+
make compile_localproxy os=macos arch=arm64 local_proxy_postfix= &
120+
wait
121+
122+
- name: Package macOS Desktop Apps (signed)
123+
if: inputs.dry_run != true
124+
run: |
125+
make mac_x64_package
126+
make mac_arm64_package
46127
env:
47128
BAI_APP_SIGN: 1
48129
BAI_APP_SIGN_APPLE_TEAM_ID: ${{ secrets.BAI_APP_SIGN_APPLE_TEAM_ID }}
@@ -51,10 +132,72 @@ jobs:
51132
BAI_APP_SIGN_IDENTITY: ${{ secrets.BAI_APP_SIGN_IDENTITY }}
52133
BAI_APP_SIGN_KEYCHAIN_B64: ${{ secrets.BAI_APP_SIGN_KEYCHAIN_B64 }}
53134
BAI_APP_SIGN_KEYCHAIN_PASSWORD: ${{ secrets.BAI_APP_SIGN_KEYCHAIN_PASSWORD }}
54-
NODE_OPTIONS: --max-old-space-size=4096
55-
- name: Bundle static resources into zip package
56-
run: make bundle
57-
- name: Upload application to latest release
135+
136+
- name: Package macOS Desktop Apps (unsigned, dry run)
137+
if: inputs.dry_run == true
138+
run: |
139+
make mac_x64_package
140+
make mac_arm64_package
141+
142+
- name: Upload macOS release assets
143+
if: inputs.dry_run != true
144+
run: node upload-release.js app
145+
env:
146+
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
147+
148+
# ──────────────────────────────────────────────────────────────────────
149+
# Job 3: Build Windows + Linux desktop apps (ubuntu, ~8 min)
150+
# No code signing needed — can run on cheaper/faster ubuntu runners.
151+
# ──────────────────────────────────────────────────────────────────────
152+
build_desktop:
153+
needs: build_web
154+
permissions:
155+
contents: write
156+
runs-on: ubuntu-latest
157+
steps:
158+
- name: Check out Git repository
159+
uses: actions/checkout@v4
160+
161+
- uses: pnpm/action-setup@v4
162+
name: Install pnpm
163+
with:
164+
version: latest
165+
run_install: false
166+
167+
- name: Install Node.js
168+
uses: actions/setup-node@v4
169+
with:
170+
node-version-file: '.nvmrc'
171+
cache: 'pnpm'
172+
173+
- name: Install Dependencies
174+
run: pnpm install --no-frozen-lockfile
175+
176+
- name: Download web build artifacts
177+
uses: actions/download-artifact@v4
178+
with:
179+
name: web-build
180+
181+
- name: Prepare Electron app
182+
run: make dep_electron
183+
184+
- name: Compile local proxies (parallel)
185+
run: |
186+
make compile_localproxy os=win arch=x64 local_proxy_postfix=.exe &
187+
make compile_localproxy os=win arch=arm64 local_proxy_postfix=.exe &
188+
make compile_localproxy os=linux arch=x64 local_proxy_postfix= &
189+
make compile_localproxy os=linux arch=arm64 local_proxy_postfix= &
190+
wait
191+
192+
- name: Package Windows & Linux Desktop Apps
193+
run: |
194+
make win_x64_package
195+
make win_arm64_package
196+
make linux_x64_package
197+
make linux_arm64_package
198+
199+
- name: Upload release assets
200+
if: inputs.dry_run != true
58201
run: node upload-release.js app
59202
env:
60203
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

‎Makefile‎

Lines changed: 68 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -61,35 +61,53 @@ compile_client_node_ts: clean_client_node_ts
6161
compile_wsproxy: compile_client_node_ts
6262
@pnpm -w exec webpack-cli --config src/wsproxy/webpack.config.js
6363
all: dep
64-
@make mac_x64
65-
@make mac_arm64
66-
@make win_x64
67-
@make win_arm64
68-
@make linux_x64
69-
@make linux_arm64
64+
@make compile_all_localproxy
65+
@make mac_x64_package
66+
@make mac_arm64_package
67+
@make win_x64_package
68+
@make win_arm64_package
69+
@make linux_x64_package
70+
@make linux_arm64_package
7071
@make bundle
71-
dep:
72+
# Build all local proxy binaries in parallel (saves ~3-4 min vs sequential)
73+
compile_all_localproxy:
74+
@printf "$(GREEN)Compiling all local proxy binaries in parallel...$(NC)\n"
75+
@make compile_localproxy os=macos arch=x64 local_proxy_postfix= & \
76+
make compile_localproxy os=macos arch=arm64 local_proxy_postfix= & \
77+
make compile_localproxy os=win arch=x64 local_proxy_postfix=.exe & \
78+
make compile_localproxy os=win arch=arm64 local_proxy_postfix=.exe & \
79+
make compile_localproxy os=linux arch=x64 local_proxy_postfix= & \
80+
make compile_localproxy os=linux arch=arm64 local_proxy_postfix= & \
81+
wait
82+
@printf "$(YELLOW)All local proxy binaries compiled$(NC)\n"
83+
# Build only the web bundle (no Electron setup). Used by CI web job.
84+
dep_web:
7285
@if [ ! -f "./config.toml" ]; then \
7386
cp config.toml.sample config.toml; \
7487
fi
7588
@mkdir -p ./app
76-
@if [ ! -d "./build/web/" ] || ! grep -q 'es6://static/js/main' react/build/index.html; then \
89+
@if [ ! -d "./build/web/" ]; then \
7790
make compile; \
7891
make compile_wsproxy; \
92+
fi
93+
# Prepare the Electron app directory. Requires dep_web to have run first.
94+
# Uses publicPath patching instead of a full second React build (~4-8 min savings).
95+
dep_electron: dep_web
96+
@if [ ! -d "./build/electron-app/app" ]; then \
7997
rm -rf build/electron-app; \
8098
mkdir -p build/electron-app; \
81-
cp -r electron-app/* build/electron-app/;\
82-
cp electron-app/.npmrc build/electron-app/;\
83-
pnpm i --prefix ./build/electron-app --ignore-workspace;\
99+
cp -r electron-app/* build/electron-app/; \
100+
cp electron-app/.npmrc build/electron-app/; \
101+
pnpm i --prefix ./build/electron-app --ignore-workspace; \
84102
cp -Rp build/web build/electron-app/app; \
85103
cp -Rp build/web/resources build/electron-app; \
86104
cp -Rp build/web/manifest build/electron-app; \
87-
BUILD_TARGET=electron pnpm run build:react-only; \
88-
cp -Rp react/build/* build/electron-app/app/; \
105+
node scripts/patch-electron-publicpath.js build/electron-app/app; \
89106
mkdir -p ./build/electron-app/app/wsproxy; \
90107
cp ./src/wsproxy/dist/wsproxy.js ./build/electron-app/app/wsproxy/wsproxy.js; \
91108
cp ./preload.js ./build/electron-app/preload.js; \
92109
fi
110+
dep: dep_electron
93111
web:
94112
@if [ ! -d "./build/web/" ];then \
95113
make compile; \
@@ -123,13 +141,13 @@ compile_localproxy:
123141
@rm -rf ./app/backend.ai-local-proxy-$(BUILD_VERSION)-$(os)-$(arch)$(local_proxy_postfix)
124142
@pnpm exec pkg ./src/wsproxy/local_proxy.js --targets node18-$(os)-$(arch) --output ./app/backend.ai-local-proxy-$(BUILD_VERSION)-$(os)-$(arch)$(local_proxy_postfix) --compress Brotli
125143
@rm -rf ./app/backend.ai-local-proxy$(local_proxy_postfix); cp ./app/backend.ai-local-proxy-$(BUILD_VERSION)-$(os)-$(arch)$(local_proxy_postfix) ./app/backend.ai-local-proxy$(local_proxy_postfix)
126-
@cd app; zip -r -9 ./backend.ai-local-proxy-$(BUILD_VERSION)-$(os)-$(arch).zip "./backend.ai-local-proxy$(local_proxy_postfix)"
144+
@cd app; zip -r -6 ./backend.ai-local-proxy-$(BUILD_VERSION)-$(os)-$(arch).zip "./backend.ai-local-proxy$(local_proxy_postfix)"
127145
@rm -rf ./app/backend.ai-local-proxy$(local_proxy_postfix)
128146
package_zip:
129147
@printf "$(GREEN)Packaging as ZIP archive...$(NC)"
130148
@cp ./configs/$(site).toml ./build/electron-app/app/config.toml
131149
@node ./app-packager.js $(os) $(arch)
132-
@cd app; zip -r -9 ./backend.ai-desktop-$(os)-$(arch)-$(BUILD_DATE).zip "./Backend.AI Desktop-$(os_api)-$(arch)"
150+
@cd app; zip -r -6 ./backend.ai-desktop-$(os)-$(arch)-$(BUILD_DATE).zip "./Backend.AI Desktop-$(os_api)-$(arch)"
133151
ifeq ($(site),main)
134152
@mv ./app/backend.ai-desktop-$(os)-$(arch)-$(BUILD_DATE).zip ./app/backend.ai-desktop-$(BUILD_VERSION)-$(os)-$(arch).zip
135153
else
@@ -152,9 +170,10 @@ else
152170
@mv ./app/backend.ai-desktop-$(arch)-$(BUILD_DATE).dmg ./app/backend.ai-desktop-$(BUILD_VERSION)-$(site)-$(os)-$(arch).dmg
153171
endif
154172
@printf "$(YELLOW)Finished$(NC)\n"
155-
bundle: dep
173+
bundle: dep_web
156174
@printf "$(GREEN)Bundling...$(NC)"
157-
@cd build/web; zip -r -9 ../../app/backend.ai-webui-bundle-$(BUILD_DATE).zip . > /dev/null
175+
@mkdir -p ./app
176+
@cd build/web; zip -r -6 ../../app/backend.ai-webui-bundle-$(BUILD_DATE).zip . > /dev/null
158177
@mv ./app/backend.ai-webui-bundle-$(BUILD_DATE).zip ./app/backend.ai-webui-bundle-$(BUILD_VERSION).zip
159178
@printf "$(YELLOW)Finished$(NC)\n"
160179
mac: dep
@@ -170,6 +189,16 @@ mac_arm64: arch := arm64
170189
mac_arm64: local_proxy_postfix :=
171190
mac_arm64: dep mac_load_keychain compile_localproxy package_dmg
172191
@printf "$(GREEN)Build finished$(NC): macOS arm64\n"
192+
# Package-only targets: assume dep + compile_all_localproxy already done.
193+
# Used by `make all` which pre-builds all local proxies in parallel.
194+
mac_x64_package: os := macos
195+
mac_x64_package: arch := x64
196+
mac_x64_package: mac_load_keychain package_dmg
197+
@printf "$(GREEN)Build finished$(NC): macOS x64\n"
198+
mac_arm64_package: os := macos
199+
mac_arm64_package: arch := arm64
200+
mac_arm64_package: mac_load_keychain package_dmg
201+
@printf "$(GREEN)Build finished$(NC): macOS arm64\n"
173202
win: dep
174203
@make win_x64
175204
@make win_arm64
@@ -185,6 +214,17 @@ win_arm64: arch := arm64
185214
win_arm64: local_proxy_postfix := .exe
186215
win_arm64: dep compile_localproxy package_zip
187216
@printf "$(GREEN)Build finished$(NC): Windows arm64\n"
217+
# Package-only targets for CI parallel jobs
218+
win_x64_package: os := win
219+
win_x64_package: os_api := win32
220+
win_x64_package: arch := x64
221+
win_x64_package: package_zip
222+
@printf "$(GREEN)Build finished$(NC): Windows x64\n"
223+
win_arm64_package: os := win
224+
win_arm64_package: os_api := win32
225+
win_arm64_package: arch := arm64
226+
win_arm64_package: package_zip
227+
@printf "$(GREEN)Build finished$(NC): Windows arm64\n"
188228
linux: dep
189229
@make linux_x64
190230
@make linux_arm64
@@ -200,6 +240,17 @@ linux_arm64: arch := arm64
200240
linux_arm64: local_proxy_postfix :=
201241
linux_arm64: dep compile_localproxy package_zip
202242
@printf "$(GREEN)Build finished$(NC): Linux arm64\n"
243+
# Package-only targets for CI parallel jobs
244+
linux_x64_package: os := linux
245+
linux_x64_package: os_api := linux
246+
linux_x64_package: arch := x64
247+
linux_x64_package: package_zip
248+
@printf "$(GREEN)Build finished$(NC): Linux x64\n"
249+
linux_arm64_package: os := linux
250+
linux_arm64_package: os_api := linux
251+
linux_arm64_package: arch := arm64
252+
linux_arm64_package: package_zip
253+
@printf "$(GREEN)Build finished$(NC): Linux arm64\n"
203254
build_docker: compile
204255
docker build -t backend.ai-webui:$(BUILD_DATE) .
205256
i18n:

‎react/package.json‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -88,7 +88,7 @@
8888
"scripts": {
8989
"start": "NODE_OPTIONS='--max-old-space-size=4096' craco start",
9090
"build": "pnpm run build:only && cp -r ./build/* ../build/web/",
91-
"build:only": "pnpm run relay && craco build",
91+
"build:only": "NODE_OPTIONS='--max-old-space-size=4096' pnpm run relay && NODE_OPTIONS='--max-old-space-size=4096' craco build",
9292
"test": "NODE_OPTIONS='$NODE_OPTIONS --no-deprecation --experimental-vm-modules' jest",
9393
"eject": "react-scripts eject",
9494
"relay": "relay-compiler",

0 commit comments

Comments
 (0)