Skip to content

Commit 529357f

Browse files
NeptuneHubNeptuneHub
committed
NATIVE BUILD IMPROVEMENT
1 parent ae6a280 commit 529357f

28 files changed

Lines changed: 1063 additions & 985 deletions

‎.dockerignore‎

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -21,12 +21,15 @@ test/
2121
# Student CLAP training (not needed in production)
2222
student_clap/
2323

24-
# macOS and Linux standalone app: source trees (PyInstaller spec / nfpm config,
25-
# vendored redis/pg binaries, assets) and build artifacts. None of it is used by
26-
# the Linux container, so keep it out of the image.
24+
# Native standalone app (macOS/Linux/Windows): the per-platform source trees
25+
# (vendored redis/pg binaries, assets, nfpm config, runtime launchers), the shared
26+
# PyInstaller spec + build tooling, and build artifacts. None of it is used by the
27+
# Linux container, so keep it out of the image.
2728
macos/
2829
linux/
2930
windows/
31+
AudioMuse-AI.spec
32+
scripts/standalone/
3033
build/
3134
dist/
3235

‎.github/workflows/build-linux.yml‎

Lines changed: 4 additions & 81 deletions
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,7 @@
1414
#
1515
# The small native build inputs (redis-server, the unaccent/pg_trgm contrib
1616
# modules) are built from source in this workflow (linux/vendor/*) rather than
17-
# committed, then baked into the bundle by linux/AudioMuse-AI.spec.
17+
# committed, then baked into the bundle by the shared AudioMuse-AI.spec.
1818
#
1919
# The ~5 GB of models are NOT in git. This workflow assembles ./model from the
2020
# same GitHub releases the Dockerfile/macOS build use, INCLUDING the HuggingFace
@@ -146,87 +146,10 @@ jobs:
146146
- name: Assemble ./model (mirrors the Dockerfile/macOS models stage)
147147
env:
148148
GH_TOKEN: ${{ github.token }}
149-
run: |
150-
set -euo pipefail
151-
mkdir -p model
152-
153-
echo "==> musicnn + CLAP text models (from ${MODEL_RELEASE})"
154-
gh release download "$MODEL_RELEASE" -R "$GITHUB_REPOSITORY" -D model --clobber \
155-
-p musicnn_embedding.onnx \
156-
-p musicnn_prediction.onnx \
157-
-p clap_text_model.onnx
158-
159-
echo "==> DCLAP audio model (from ${DCLAP_RELEASE} in the -DCLAP repo)"
160-
gh release download "$DCLAP_RELEASE" -R NeptuneHub/AudioMuse-AI-DCLAP -D model --clobber \
161-
-p model_epoch_36.onnx \
162-
-p model_epoch_36.onnx.data
163-
164-
echo "==> HuggingFace cache (roberta/bert/bart) -- HF_HOME points at model/huggingface"
165-
tmp_hf="$(mktemp -d)"
166-
gh release download "$MODEL_RELEASE" -R "$GITHUB_REPOSITORY" -D "$tmp_hf" --clobber \
167-
-p huggingface_models.tar.gz
168-
mkdir -p model/huggingface
169-
tar -xzf "$tmp_hf/huggingface_models.tar.gz" -C model/huggingface
170-
rm -rf "$tmp_hf"
171-
172-
echo "==> Trim HF cache to just the roberta-base tokenizer (~1.4 GB saved)"
173-
# The app's ONLY runtime HF dependency is the roberta-base *tokenizer*
174-
# (tasks/clap_analyzer.py: AutoTokenizer.from_pretrained("roberta-base")).
175-
# bert-base-uncased and bart-base are never loaded by the app, and a
176-
# tokenizer does not need model weights. Dropping them keeps the release
177-
# assets under the 2 GB GitHub limit without removing any model the app
178-
# uses. (This prunes only this bundle copy -- the shared release tarball
179-
# and the Docker build are unaffected.)
180-
hf="model/huggingface/hub"
181-
rm -rf "$hf/models--bert-base-uncased" "$hf/models--facebook--bart-base"
182-
rb="$hf/models--roberta-base"
183-
if [ -d "$rb" ]; then
184-
find "$rb/blobs" -type f -size +10M -delete
185-
find "$rb/snapshots" \( -name "model.safetensors" -o -name "pytorch_model.bin" \) -delete
186-
du -sh "$rb"
187-
fi
188-
189-
echo "==> lyrics bundles (whisper / silero / gte)"
190-
tmp="$(mktemp -d)"
191-
gh release download "$MODEL_RELEASE" -R "$GITHUB_REPOSITORY" -D "$tmp" --clobber \
192-
-p lyrics_model_whisper.tar.gz \
193-
-p lyrics_model_silero_vad.tar.gz \
194-
-p lyrics_model_gte_vnni.tar.gz
195-
for t in lyrics_model_whisper lyrics_model_silero_vad lyrics_model_gte_vnni; do
196-
tar -xzf "$tmp/$t.tar.gz" -C model
197-
done
198-
rm -rf "$tmp"
149+
run: python3 scripts/standalone/assemble_model.py
199150

200151
- name: Verify the assembled model/ is complete
201-
run: |
202-
set -euo pipefail
203-
required=(
204-
model/musicnn_embedding.onnx
205-
model/musicnn_prediction.onnx
206-
model/clap_text_model.onnx
207-
model/model_epoch_36.onnx
208-
model/model_epoch_36.onnx.data
209-
model/huggingface/hub/models--roberta-base/snapshots
210-
model/silero_vad.onnx
211-
model/gte-multilingual-base-int8.onnx
212-
model/whisper-small-onnx/encoder_model.onnx
213-
model/whisper-small-onnx/decoder_model_merged.onnx
214-
model/gte-multilingual-base/tokenizer.json
215-
)
216-
missing=0
217-
for f in "${required[@]}"; do
218-
# Flag a file that is missing OR a zero-byte/truncated download. The
219-
# size test only applies to regular files (directory entries in the
220-
# list pass on existence alone).
221-
if [ ! -e "$f" ] || { [ -f "$f" ] && [ ! -s "$f" ]; }; then
222-
echo "::error::Missing or empty: $f"; missing=1
223-
fi
224-
done
225-
if [ -z "$(find model/huggingface/hub/models--roberta-base -name tokenizer.json -print -quit)" ]; then
226-
echo "::error::roberta-base tokenizer.json missing after HF-cache prune"; missing=1
227-
fi
228-
[ "$missing" -eq 0 ] || { echo "::error::Refusing to build an incomplete bundle."; exit 1; }
229-
du -sh model
152+
run: python3 scripts/standalone/assemble_model.py --verify
230153

231154
- name: Build the packages (.deb + .rpm)
232155
run: |
@@ -241,7 +164,7 @@ jobs:
241164
else
242165
PKG_VERSION="0.0.0"
243166
fi
244-
PKG_VERSION="$PKG_VERSION" bash linux/build.sh
167+
PKG_VERSION="$PKG_VERSION" python scripts/standalone/build.py --platform linux
245168
env:
246169
GITHUB_REF_NAME: ${{ github.ref_name }}
247170
GITHUB_REF_TYPE: ${{ github.ref_type }}

‎.github/workflows/build-macos.yml‎

Lines changed: 3 additions & 78 deletions
Original file line numberDiff line numberDiff line change
@@ -104,85 +104,10 @@ jobs:
104104
- name: Assemble ./model (mirrors the Dockerfile models stage)
105105
env:
106106
GH_TOKEN: ${{ github.token }}
107-
run: |
108-
set -euo pipefail
109-
mkdir -p model
110-
111-
echo "==> musicnn + CLAP text models (from ${MODEL_RELEASE})"
112-
gh release download "$MODEL_RELEASE" -R "$GITHUB_REPOSITORY" -D model --clobber \
113-
-p musicnn_embedding.onnx \
114-
-p musicnn_prediction.onnx \
115-
-p clap_text_model.onnx
116-
117-
echo "==> DCLAP audio model (from ${DCLAP_RELEASE} in the -DCLAP repo)"
118-
gh release download "$DCLAP_RELEASE" -R NeptuneHub/AudioMuse-AI-DCLAP -D model --clobber \
119-
-p model_epoch_36.onnx \
120-
-p model_epoch_36.onnx.data
121-
122-
echo "==> HuggingFace cache (roberta/bert/bart) — HF_HOME points at model/huggingface"
123-
tmp_hf="$(mktemp -d)"
124-
gh release download "$MODEL_RELEASE" -R "$GITHUB_REPOSITORY" -D "$tmp_hf" --clobber \
125-
-p huggingface_models.tar.gz
126-
mkdir -p model/huggingface
127-
tar -xzf "$tmp_hf/huggingface_models.tar.gz" -C model/huggingface
128-
rm -rf "$tmp_hf"
129-
130-
echo "==> Trim HF cache to just the roberta-base tokenizer (~1.4 GB saved)"
131-
# The macOS app's ONLY runtime HF dependency is the roberta-base *tokenizer*
132-
# (tasks/clap_analyzer.py: AutoTokenizer.from_pretrained("roberta-base")).
133-
# bert-base-uncased and bart-base are never loaded by the app, and a tokenizer
134-
# does not need model weights. Dropping them keeps the release zip under the
135-
# 2 GB GitHub release-asset limit without removing any model the app uses.
136-
# Note: this prunes only the macOS bundle copy — the shared release tarball
137-
# and the Docker build are unaffected.
138-
hf="model/huggingface/hub"
139-
rm -rf "$hf/models--bert-base-uncased" "$hf/models--facebook--bart-base"
140-
rb="$hf/models--roberta-base"
141-
if [ -d "$rb" ]; then
142-
# Drop the ~476 MB weight blob; AutoTokenizer reads only the tiny
143-
# tokenizer.json/vocab.json/merges.txt/config files (all < 2 MB).
144-
find "$rb/blobs" -type f -size +10M -delete
145-
find "$rb/snapshots" \( -name "model.safetensors" -o -name "pytorch_model.bin" \) -delete
146-
du -sh "$rb"
147-
fi
148-
149-
echo "==> lyrics bundles (whisper / silero / gte) — downloaded then extracted into ./model"
150-
tmp="$(mktemp -d)"
151-
gh release download "$MODEL_RELEASE" -R "$GITHUB_REPOSITORY" -D "$tmp" --clobber \
152-
-p lyrics_model_whisper.tar.gz \
153-
-p lyrics_model_silero_vad.tar.gz \
154-
-p lyrics_model_gte_vnni.tar.gz
155-
for t in lyrics_model_whisper lyrics_model_silero_vad lyrics_model_gte_vnni; do
156-
tar -xzf "$tmp/$t.tar.gz" -C model
157-
done
158-
rm -rf "$tmp"
107+
run: python3 scripts/standalone/assemble_model.py
159108

160109
- name: Verify the assembled model/ is complete
161-
run: |
162-
set -euo pipefail
163-
required=(
164-
model/musicnn_embedding.onnx
165-
model/musicnn_prediction.onnx
166-
model/clap_text_model.onnx
167-
model/model_epoch_36.onnx
168-
model/model_epoch_36.onnx.data
169-
model/huggingface/hub/models--roberta-base/snapshots
170-
model/silero_vad.onnx
171-
model/gte-multilingual-base-int8.onnx
172-
model/whisper-small-onnx/encoder_model.onnx
173-
model/whisper-small-onnx/decoder_model_merged.onnx
174-
model/gte-multilingual-base/tokenizer.json
175-
)
176-
missing=0
177-
for f in "${required[@]}"; do
178-
if [ ! -e "$f" ]; then echo "::error::Missing or empty: $f"; missing=1; fi
179-
done
180-
# The roberta tokenizer files must survive the HF-cache prune above.
181-
if [ -z "$(find model/huggingface/hub/models--roberta-base -name tokenizer.json -print -quit)" ]; then
182-
echo "::error::roberta-base tokenizer.json missing after HF-cache prune"; missing=1
183-
fi
184-
[ "$missing" -eq 0 ] || { echo "::error::Refusing to build an incomplete bundle."; exit 1; }
185-
du -sh model
110+
run: python3 scripts/standalone/assemble_model.py --verify
186111

187112
- name: Install Python dependencies
188113
run: |
@@ -196,7 +121,7 @@ jobs:
196121
run: |
197122
set -euo pipefail
198123
source .venv-macos/bin/activate
199-
bash macos/build.sh
124+
python scripts/standalone/build.py --platform macos
200125
201126
- name: Upload zip as a workflow artifact
202127
uses: actions/upload-artifact@v4

‎.github/workflows/build-windows.yml‎

Lines changed: 4 additions & 74 deletions
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@
88
#
99
# The small native build inputs (redis-server.exe, the unaccent/pg_trgm contrib
1010
# modules) are built or downloaded in this workflow, then baked into the bundle
11-
# by windows/AudioMuse-AI.spec.
11+
# by the shared AudioMuse-AI.spec.
1212
#
1313
# The ~5 GB of models are NOT in git. This workflow assembles ./model from the
1414
# same GitHub releases the Dockerfile/macOS/Linux builds use, INCLUDING the
@@ -103,92 +103,22 @@ jobs:
103103
shell: powershell
104104
env:
105105
GH_TOKEN: ${{ github.token }}
106-
run: |
107-
New-Item -ItemType Directory -Force -Path model | Out-Null
108-
$modelRelease = "$env:MODEL_RELEASE"
109-
$dclapRelease = "$env:DCLAP_RELEASE"
110-
111-
Write-Host "==> musicnn + CLAP text models (from $modelRelease)"
112-
gh release download $modelRelease -R $env:GITHUB_REPOSITORY -D model --clobber `
113-
-p musicnn_embedding.onnx `
114-
-p musicnn_prediction.onnx `
115-
-p clap_text_model.onnx
116-
117-
Write-Host "==> DCLAP audio model (from $dclapRelease in the -DCLAP repo)"
118-
gh release download $dclapRelease -R NeptuneHub/AudioMuse-AI-DCLAP -D model --clobber `
119-
-p model_epoch_36.onnx `
120-
-p model_epoch_36.onnx.data
121-
122-
Write-Host "==> HuggingFace cache (roberta/bert/bart)"
123-
$tmp_hf = Join-Path $env:RUNNER_TEMP "hf_models"
124-
New-Item -ItemType Directory -Force -Path $tmp_hf | Out-Null
125-
gh release download $modelRelease -R $env:GITHUB_REPOSITORY -D $tmp_hf --clobber `
126-
-p huggingface_models.tar.gz
127-
New-Item -ItemType Directory -Force -Path model\huggingface | Out-Null
128-
tar -xzf "$tmp_hf\huggingface_models.tar.gz" -C model\huggingface
129-
if ($LASTEXITCODE -ne 0) { Write-Error "tar failed extracting huggingface models"; exit 1 }
130-
Remove-Item -Recurse -Force $tmp_hf
131-
132-
Write-Host "==> Trim HF cache to just the roberta-base tokenizer (~1.4 GB saved)"
133-
$hf = "model\huggingface\hub"
134-
if (Test-Path "$hf\models--bert-base-uncased") { Remove-Item -Recurse -Force "$hf\models--bert-base-uncased" }
135-
if (Test-Path "$hf\models--facebook--bart-base") { Remove-Item -Recurse -Force "$hf\models--facebook--bart-base" }
136-
$rb = "$hf\models--roberta-base"
137-
if (Test-Path $rb) {
138-
Get-ChildItem -Recurse -File "$rb\blobs" | Where-Object { $_.Length -gt 10MB } | Remove-Item -Force
139-
Get-ChildItem -Recurse "$rb\snapshots" -Include "model.safetensors","pytorch_model.bin" | Remove-Item -Force
140-
}
141-
142-
Write-Host "==> lyrics bundles (whisper / silero / gte)"
143-
$tmp = Join-Path $env:RUNNER_TEMP "lyrics_models"
144-
New-Item -ItemType Directory -Force -Path $tmp | Out-Null
145-
gh release download $modelRelease -R $env:GITHUB_REPOSITORY -D $tmp --clobber `
146-
-p lyrics_model_whisper.tar.gz `
147-
-p lyrics_model_silero_vad.tar.gz `
148-
-p lyrics_model_gte_vnni.tar.gz
149-
foreach ($t in @("lyrics_model_whisper", "lyrics_model_silero_vad", "lyrics_model_gte_vnni")) {
150-
tar -xzf "$tmp\$t.tar.gz" -C model
151-
if ($LASTEXITCODE -ne 0) { Write-Error "tar failed extracting $t"; exit 1 }
152-
}
153-
Remove-Item -Recurse -Force $tmp
106+
run: python scripts/standalone/assemble_model.py
154107

155108
- name: Verify the assembled model/ is complete
156109
shell: powershell
157-
run: |
158-
$required = @(
159-
"model\musicnn_embedding.onnx",
160-
"model\musicnn_prediction.onnx",
161-
"model\clap_text_model.onnx",
162-
"model\model_epoch_36.onnx",
163-
"model\model_epoch_36.onnx.data",
164-
"model\huggingface\hub\models--roberta-base\snapshots",
165-
"model\silero_vad.onnx",
166-
"model\gte-multilingual-base-int8.onnx",
167-
"model\whisper-small-onnx\encoder_model.onnx",
168-
"model\whisper-small-onnx\decoder_model_merged.onnx",
169-
"model\gte-multilingual-base\tokenizer.json"
170-
)
171-
$missing = 0
172-
foreach ($f in $required) {
173-
if (-not (Test-Path $f)) {
174-
Write-Error "Missing or empty: $f"
175-
$missing = 1
176-
}
177-
}
178-
if ($missing -ne 0) { Write-Error "Refusing to build an incomplete bundle."; exit 1 }
179-
Write-Host "Model assembly verified."
110+
run: python scripts/standalone/assemble_model.py --verify
180111

181112
- name: Build the bundle
182113
shell: powershell
183114
run: |
184-
.venv-windows\Scripts\activate
185115
if ($env:GITHUB_REF_TYPE -eq "tag") {
186116
$ver = $env:GITHUB_REF_NAME -replace '^v', ''
187117
} else {
188118
$ver = "0.0.0"
189119
}
190120
$env:PKG_VERSION = $ver
191-
cmd /c windows\build.bat
121+
.venv-windows\Scripts\python scripts\standalone\build.py --platform windows
192122
env:
193123
GITHUB_REF_NAME: ${{ github.ref_name }}
194124
GITHUB_REF_TYPE: ${{ github.ref_type }}

0 commit comments

Comments
 (0)