diff --git a/.dockerignore b/.dockerignore index 8975a980..9cc988e1 100644 --- a/.dockerignore +++ b/.dockerignore @@ -21,12 +21,15 @@ test/ # Student CLAP training (not needed in production) student_clap/ -# macOS and Linux standalone app: source trees (PyInstaller spec / nfpm config, -# vendored redis/pg binaries, assets) and build artifacts. None of it is used by -# the Linux container, so keep it out of the image. +# Native standalone app (macOS/Linux/Windows): the per-platform source trees +# (vendored redis/pg binaries, assets, nfpm config, runtime launchers), the shared +# PyInstaller spec + build tooling, and build artifacts. None of it is used by the +# Linux container, so keep it out of the image. macos/ linux/ windows/ +AudioMuse-AI.spec +scripts/standalone/ build/ dist/ diff --git a/.github/workflows/build-linux.yml b/.github/workflows/build-linux.yml index a71320cf..901a3b0b 100644 --- a/.github/workflows/build-linux.yml +++ b/.github/workflows/build-linux.yml @@ -14,7 +14,7 @@ # # The small native build inputs (redis-server, the unaccent/pg_trgm contrib # modules) are built from source in this workflow (linux/vendor/*) rather than -# committed, then baked into the bundle by linux/AudioMuse-AI.spec. +# committed, then baked into the bundle by the shared AudioMuse-AI.spec. # # The ~5 GB of models are NOT in git. This workflow assembles ./model from the # same GitHub releases the Dockerfile/macOS build use, INCLUDING the HuggingFace @@ -146,105 +146,16 @@ jobs: - name: Assemble ./model (mirrors the Dockerfile/macOS models stage) env: GH_TOKEN: ${{ github.token }} - run: | - set -euo pipefail - mkdir -p model - - echo "==> musicnn + CLAP text models (from ${MODEL_RELEASE})" - gh release download "$MODEL_RELEASE" -R "$GITHUB_REPOSITORY" -D model --clobber \ - -p musicnn_embedding.onnx \ - -p musicnn_prediction.onnx \ - -p clap_text_model.onnx - - echo "==> DCLAP audio model (from ${DCLAP_RELEASE} in the -DCLAP repo)" - gh release download "$DCLAP_RELEASE" -R NeptuneHub/AudioMuse-AI-DCLAP -D model --clobber \ - -p model_epoch_36.onnx \ - -p model_epoch_36.onnx.data - - echo "==> HuggingFace cache (roberta/bert/bart) -- HF_HOME points at model/huggingface" - tmp_hf="$(mktemp -d)" - gh release download "$MODEL_RELEASE" -R "$GITHUB_REPOSITORY" -D "$tmp_hf" --clobber \ - -p huggingface_models.tar.gz - mkdir -p model/huggingface - tar -xzf "$tmp_hf/huggingface_models.tar.gz" -C model/huggingface - rm -rf "$tmp_hf" - - echo "==> Trim HF cache to just the roberta-base tokenizer (~1.4 GB saved)" - # The app's ONLY runtime HF dependency is the roberta-base *tokenizer* - # (tasks/clap_analyzer.py: AutoTokenizer.from_pretrained("roberta-base")). - # bert-base-uncased and bart-base are never loaded by the app, and a - # tokenizer does not need model weights. Dropping them keeps the release - # assets under the 2 GB GitHub limit without removing any model the app - # uses. (This prunes only this bundle copy -- the shared release tarball - # and the Docker build are unaffected.) - hf="model/huggingface/hub" - rm -rf "$hf/models--bert-base-uncased" "$hf/models--facebook--bart-base" - rb="$hf/models--roberta-base" - if [ -d "$rb" ]; then - find "$rb/blobs" -type f -size +10M -delete - find "$rb/snapshots" \( -name "model.safetensors" -o -name "pytorch_model.bin" \) -delete - du -sh "$rb" - fi - - echo "==> lyrics bundles (whisper / silero / gte)" - tmp="$(mktemp -d)" - gh release download "$MODEL_RELEASE" -R "$GITHUB_REPOSITORY" -D "$tmp" --clobber \ - -p lyrics_model_whisper.tar.gz \ - -p lyrics_model_silero_vad.tar.gz \ - -p lyrics_model_gte_vnni.tar.gz - for t in lyrics_model_whisper lyrics_model_silero_vad lyrics_model_gte_vnni; do - tar -xzf "$tmp/$t.tar.gz" -C model - done - rm -rf "$tmp" + run: python3 scripts/standalone/assemble_model.py - name: Verify the assembled model/ is complete - run: | - set -euo pipefail - required=( - model/musicnn_embedding.onnx - model/musicnn_prediction.onnx - model/clap_text_model.onnx - model/model_epoch_36.onnx - model/model_epoch_36.onnx.data - model/huggingface/hub/models--roberta-base/snapshots - model/silero_vad.onnx - model/gte-multilingual-base-int8.onnx - model/whisper-small-onnx/encoder_model.onnx - model/whisper-small-onnx/decoder_model_merged.onnx - model/gte-multilingual-base/tokenizer.json - ) - missing=0 - for f in "${required[@]}"; do - # Flag a file that is missing OR a zero-byte/truncated download. The - # size test only applies to regular files (directory entries in the - # list pass on existence alone). - if [ ! -e "$f" ] || { [ -f "$f" ] && [ ! -s "$f" ]; }; then - echo "::error::Missing or empty: $f"; missing=1 - fi - done - if [ -z "$(find model/huggingface/hub/models--roberta-base -name tokenizer.json -print -quit)" ]; then - echo "::error::roberta-base tokenizer.json missing after HF-cache prune"; missing=1 - fi - [ "$missing" -eq 0 ] || { echo "::error::Refusing to build an incomplete bundle."; exit 1; } - du -sh model + run: python3 scripts/standalone/assemble_model.py --verify - name: Build the packages (.deb + .rpm) run: | set -euo pipefail source .venv-linux/bin/activate - # Derive a package-manager-safe version. On a tag push github.ref_name - # is `vX.Y.Z` -> `X.Y.Z`; on a PR it is `/merge` (and on dispatch a - # branch name), neither of which is a valid deb/rpm version, so use a - # placeholder for non-tag builds. - if [ "${GITHUB_REF_TYPE}" = "tag" ]; then - PKG_VERSION="${GITHUB_REF_NAME#v}" - else - PKG_VERSION="0.0.0" - fi - PKG_VERSION="$PKG_VERSION" bash linux/build.sh - env: - GITHUB_REF_NAME: ${{ github.ref_name }} - GITHUB_REF_TYPE: ${{ github.ref_type }} + python scripts/standalone/build.py --platform linux - name: Upload .deb as a workflow artifact uses: actions/upload-artifact@v4 diff --git a/.github/workflows/build-macos.yml b/.github/workflows/build-macos.yml index de12377f..4afc6c04 100644 --- a/.github/workflows/build-macos.yml +++ b/.github/workflows/build-macos.yml @@ -104,85 +104,10 @@ jobs: - name: Assemble ./model (mirrors the Dockerfile models stage) env: GH_TOKEN: ${{ github.token }} - run: | - set -euo pipefail - mkdir -p model - - echo "==> musicnn + CLAP text models (from ${MODEL_RELEASE})" - gh release download "$MODEL_RELEASE" -R "$GITHUB_REPOSITORY" -D model --clobber \ - -p musicnn_embedding.onnx \ - -p musicnn_prediction.onnx \ - -p clap_text_model.onnx - - echo "==> DCLAP audio model (from ${DCLAP_RELEASE} in the -DCLAP repo)" - gh release download "$DCLAP_RELEASE" -R NeptuneHub/AudioMuse-AI-DCLAP -D model --clobber \ - -p model_epoch_36.onnx \ - -p model_epoch_36.onnx.data - - echo "==> HuggingFace cache (roberta/bert/bart) — HF_HOME points at model/huggingface" - tmp_hf="$(mktemp -d)" - gh release download "$MODEL_RELEASE" -R "$GITHUB_REPOSITORY" -D "$tmp_hf" --clobber \ - -p huggingface_models.tar.gz - mkdir -p model/huggingface - tar -xzf "$tmp_hf/huggingface_models.tar.gz" -C model/huggingface - rm -rf "$tmp_hf" - - echo "==> Trim HF cache to just the roberta-base tokenizer (~1.4 GB saved)" - # The macOS app's ONLY runtime HF dependency is the roberta-base *tokenizer* - # (tasks/clap_analyzer.py: AutoTokenizer.from_pretrained("roberta-base")). - # bert-base-uncased and bart-base are never loaded by the app, and a tokenizer - # does not need model weights. Dropping them keeps the release zip under the - # 2 GB GitHub release-asset limit without removing any model the app uses. - # Note: this prunes only the macOS bundle copy — the shared release tarball - # and the Docker build are unaffected. - hf="model/huggingface/hub" - rm -rf "$hf/models--bert-base-uncased" "$hf/models--facebook--bart-base" - rb="$hf/models--roberta-base" - if [ -d "$rb" ]; then - # Drop the ~476 MB weight blob; AutoTokenizer reads only the tiny - # tokenizer.json/vocab.json/merges.txt/config files (all < 2 MB). - find "$rb/blobs" -type f -size +10M -delete - find "$rb/snapshots" \( -name "model.safetensors" -o -name "pytorch_model.bin" \) -delete - du -sh "$rb" - fi - - echo "==> lyrics bundles (whisper / silero / gte) — downloaded then extracted into ./model" - tmp="$(mktemp -d)" - gh release download "$MODEL_RELEASE" -R "$GITHUB_REPOSITORY" -D "$tmp" --clobber \ - -p lyrics_model_whisper.tar.gz \ - -p lyrics_model_silero_vad.tar.gz \ - -p lyrics_model_gte_vnni.tar.gz - for t in lyrics_model_whisper lyrics_model_silero_vad lyrics_model_gte_vnni; do - tar -xzf "$tmp/$t.tar.gz" -C model - done - rm -rf "$tmp" + run: python3 scripts/standalone/assemble_model.py - name: Verify the assembled model/ is complete - run: | - set -euo pipefail - required=( - model/musicnn_embedding.onnx - model/musicnn_prediction.onnx - model/clap_text_model.onnx - model/model_epoch_36.onnx - model/model_epoch_36.onnx.data - model/huggingface/hub/models--roberta-base/snapshots - model/silero_vad.onnx - model/gte-multilingual-base-int8.onnx - model/whisper-small-onnx/encoder_model.onnx - model/whisper-small-onnx/decoder_model_merged.onnx - model/gte-multilingual-base/tokenizer.json - ) - missing=0 - for f in "${required[@]}"; do - if [ ! -e "$f" ]; then echo "::error::Missing or empty: $f"; missing=1; fi - done - # The roberta tokenizer files must survive the HF-cache prune above. - if [ -z "$(find model/huggingface/hub/models--roberta-base -name tokenizer.json -print -quit)" ]; then - echo "::error::roberta-base tokenizer.json missing after HF-cache prune"; missing=1 - fi - [ "$missing" -eq 0 ] || { echo "::error::Refusing to build an incomplete bundle."; exit 1; } - du -sh model + run: python3 scripts/standalone/assemble_model.py --verify - name: Install Python dependencies run: | @@ -196,7 +121,7 @@ jobs: run: | set -euo pipefail source .venv-macos/bin/activate - bash macos/build.sh + python scripts/standalone/build.py --platform macos - name: Upload zip as a workflow artifact uses: actions/upload-artifact@v4 diff --git a/.github/workflows/build-windows.yml b/.github/workflows/build-windows.yml index a0d999a5..41200ce5 100644 --- a/.github/workflows/build-windows.yml +++ b/.github/workflows/build-windows.yml @@ -8,7 +8,7 @@ # # The small native build inputs (redis-server.exe, the unaccent/pg_trgm contrib # modules) are built or downloaded in this workflow, then baked into the bundle -# by windows/AudioMuse-AI.spec. +# by the shared AudioMuse-AI.spec. # # The ~5 GB of models are NOT in git. This workflow assembles ./model from the # same GitHub releases the Dockerfile/macOS/Linux builds use, INCLUDING the @@ -103,95 +103,15 @@ jobs: shell: powershell env: GH_TOKEN: ${{ github.token }} - run: | - New-Item -ItemType Directory -Force -Path model | Out-Null - $modelRelease = "$env:MODEL_RELEASE" - $dclapRelease = "$env:DCLAP_RELEASE" - - Write-Host "==> musicnn + CLAP text models (from $modelRelease)" - gh release download $modelRelease -R $env:GITHUB_REPOSITORY -D model --clobber ` - -p musicnn_embedding.onnx ` - -p musicnn_prediction.onnx ` - -p clap_text_model.onnx - - Write-Host "==> DCLAP audio model (from $dclapRelease in the -DCLAP repo)" - gh release download $dclapRelease -R NeptuneHub/AudioMuse-AI-DCLAP -D model --clobber ` - -p model_epoch_36.onnx ` - -p model_epoch_36.onnx.data - - Write-Host "==> HuggingFace cache (roberta/bert/bart)" - $tmp_hf = Join-Path $env:RUNNER_TEMP "hf_models" - New-Item -ItemType Directory -Force -Path $tmp_hf | Out-Null - gh release download $modelRelease -R $env:GITHUB_REPOSITORY -D $tmp_hf --clobber ` - -p huggingface_models.tar.gz - New-Item -ItemType Directory -Force -Path model\huggingface | Out-Null - tar -xzf "$tmp_hf\huggingface_models.tar.gz" -C model\huggingface - if ($LASTEXITCODE -ne 0) { Write-Error "tar failed extracting huggingface models"; exit 1 } - Remove-Item -Recurse -Force $tmp_hf - - Write-Host "==> Trim HF cache to just the roberta-base tokenizer (~1.4 GB saved)" - $hf = "model\huggingface\hub" - if (Test-Path "$hf\models--bert-base-uncased") { Remove-Item -Recurse -Force "$hf\models--bert-base-uncased" } - if (Test-Path "$hf\models--facebook--bart-base") { Remove-Item -Recurse -Force "$hf\models--facebook--bart-base" } - $rb = "$hf\models--roberta-base" - if (Test-Path $rb) { - Get-ChildItem -Recurse -File "$rb\blobs" | Where-Object { $_.Length -gt 10MB } | Remove-Item -Force - Get-ChildItem -Recurse "$rb\snapshots" -Include "model.safetensors","pytorch_model.bin" | Remove-Item -Force - } - - Write-Host "==> lyrics bundles (whisper / silero / gte)" - $tmp = Join-Path $env:RUNNER_TEMP "lyrics_models" - New-Item -ItemType Directory -Force -Path $tmp | Out-Null - gh release download $modelRelease -R $env:GITHUB_REPOSITORY -D $tmp --clobber ` - -p lyrics_model_whisper.tar.gz ` - -p lyrics_model_silero_vad.tar.gz ` - -p lyrics_model_gte_vnni.tar.gz - foreach ($t in @("lyrics_model_whisper", "lyrics_model_silero_vad", "lyrics_model_gte_vnni")) { - tar -xzf "$tmp\$t.tar.gz" -C model - if ($LASTEXITCODE -ne 0) { Write-Error "tar failed extracting $t"; exit 1 } - } - Remove-Item -Recurse -Force $tmp + run: python scripts/standalone/assemble_model.py - name: Verify the assembled model/ is complete shell: powershell - run: | - $required = @( - "model\musicnn_embedding.onnx", - "model\musicnn_prediction.onnx", - "model\clap_text_model.onnx", - "model\model_epoch_36.onnx", - "model\model_epoch_36.onnx.data", - "model\huggingface\hub\models--roberta-base\snapshots", - "model\silero_vad.onnx", - "model\gte-multilingual-base-int8.onnx", - "model\whisper-small-onnx\encoder_model.onnx", - "model\whisper-small-onnx\decoder_model_merged.onnx", - "model\gte-multilingual-base\tokenizer.json" - ) - $missing = 0 - foreach ($f in $required) { - if (-not (Test-Path $f)) { - Write-Error "Missing or empty: $f" - $missing = 1 - } - } - if ($missing -ne 0) { Write-Error "Refusing to build an incomplete bundle."; exit 1 } - Write-Host "Model assembly verified." + run: python scripts/standalone/assemble_model.py --verify - name: Build the bundle shell: powershell - run: | - .venv-windows\Scripts\activate - if ($env:GITHUB_REF_TYPE -eq "tag") { - $ver = $env:GITHUB_REF_NAME -replace '^v', '' - } else { - $ver = "0.0.0" - } - $env:PKG_VERSION = $ver - cmd /c windows\build.bat - env: - GITHUB_REF_NAME: ${{ github.ref_name }} - GITHUB_REF_TYPE: ${{ github.ref_type }} + run: .venv-windows\Scripts\python scripts\standalone\build.py --platform windows - name: Upload zip as a workflow artifact uses: actions/upload-artifact@v4 diff --git a/AudioMuse-AI.spec b/AudioMuse-AI.spec new file mode 100644 index 00000000..1a71c328 --- /dev/null +++ b/AudioMuse-AI.spec @@ -0,0 +1,127 @@ +# -*- mode: python ; coding: utf-8 -*- +import glob +import importlib.util +import os +import platform + +from PyInstaller.utils.hooks import collect_data_files, collect_dynamic_libs, collect_submodules + +ROOT = SPECPATH + +_cfg_path = os.path.join(ROOT, "scripts", "standalone", "config.py") +_cfg_spec = importlib.util.spec_from_file_location("_amai_build_config", _cfg_path) +_cfg = importlib.util.module_from_spec(_cfg_spec) +_cfg_spec.loader.exec_module(_cfg) + +target = _cfg.resolve_target(os.environ.get("AUDIOMUSE_BUILD_TARGET")) +cfg = _cfg.PLATFORMS[target] +arch = _cfg.normalize_arch(platform.machine(), target) +USE_PGSERVER = _cfg.use_pgserver(cfg["use_pgserver"], arch) + +_app_ver = _cfg.read_app_version(ROOT) +if cfg["bundle"]: + cfg["bundle"]["info_plist"]["CFBundleShortVersionString"] = _app_ver or "0.0.0" + +datas = [ + (os.path.join(ROOT, "templates"), "templates"), + (os.path.join(ROOT, "static"), "static"), + (os.path.join(ROOT, "model"), "model"), + (os.path.join(ROOT, "mood_centroids_real_080_clap.json"), "."), +] +for _src, _dst in cfg["extra_datas"]: + datas.append((os.path.join(ROOT, _src), _dst)) + +if USE_PGSERVER: + try: + datas += collect_data_files("pgserver") + except Exception: + USE_PGSERVER = False +if not USE_PGSERVER: + datas += [(os.path.join(ROOT, cfg["vendor_dir"], "postgres", arch), "pgsql")] + +for _pkg in ("librosa", "resampy", "flasgger", "wn", "langdetect"): + datas += collect_data_files(_pkg) +datas += collect_data_files("transformers", include_py_files=False) + +binaries = [ + (os.path.join(ROOT, cfg["vendor_dir"], "redis", arch, cfg["redis_bin"]), "."), +] +for _pkg in ("av", "voyager", "psycopg2"): + binaries += collect_dynamic_libs(_pkg) + +if USE_PGSERVER: + _pg_contrib = os.path.join(ROOT, cfg["vendor_dir"], "pg-contrib", arch) + _pg_dst = "pgserver/pginstall" + for _f in glob.glob(os.path.join(_pg_contrib, "extension", "*")): + datas.append((_f, f"{_pg_dst}/share/postgresql/extension")) + for _f in glob.glob(os.path.join(_pg_contrib, "tsearch_data", "*")): + datas.append((_f, f"{_pg_dst}/share/postgresql/tsearch_data")) + for _f in glob.glob(os.path.join(_pg_contrib, "lib", cfg["pg_contrib_glob"])): + binaries.append((_f, f"{_pg_dst}/lib/postgresql")) + +hiddenimports = [ + "app", + "rq_worker", + "rq_worker_high_priority", + "rq_janitor", + "restart_listener", + "waitress", + "flasgger", +] +hiddenimports += cfg["extra_hiddenimports"] +for _mod in ("tasks", "lyrics", "sklearn", *cfg["collect_submodules"]): + hiddenimports += collect_submodules(_mod) +hiddenimports = list(dict.fromkeys(hiddenimports)) + +excludes = list(cfg["excludes_base"]) +if not USE_PGSERVER: + excludes.append("pgserver") + hiddenimports = [h for h in hiddenimports if not h.startswith("pgserver")] + +a = Analysis( + [os.path.join(ROOT, cfg["launcher"])], + pathex=[ROOT], + binaries=binaries, + datas=datas, + hiddenimports=hiddenimports, + hookspath=[os.path.join(ROOT, "macos/hooks")], + hooksconfig={}, + runtime_hooks=[], + excludes=excludes, + noarchive=False, +) + +pyz = PYZ(a.pure) + +exe = EXE( + pyz, + a.scripts, + [], + exclude_binaries=True, + name="AudioMuse-AI", + debug=False, + bootloader_ignore_signals=False, + strip=False, + upx=False, + console=cfg["console"], + **({"icon": os.path.join(ROOT, cfg["exe_icon"])} if cfg["exe_icon"] else {}), +) + +coll = COLLECT( + exe, + a.binaries, + a.datas, + strip=False, + upx=False, + name="AudioMuse-AI", +) + +if cfg["bundle"]: + _b = cfg["bundle"] + app = BUNDLE( + coll, + name=_b["name"], + icon=os.path.join(ROOT, _b["icon"]), + bundle_identifier=_b["bundle_identifier"], + info_plist=_b["info_plist"], + ) diff --git a/linux/AudioMuse-AI.spec b/linux/AudioMuse-AI.spec deleted file mode 100644 index feae192e..00000000 --- a/linux/AudioMuse-AI.spec +++ /dev/null @@ -1,152 +0,0 @@ -# -*- mode: python ; coding: utf-8 -*- -"""PyInstaller spec for the standalone Linux build. - -Run from the repo root: ``pyinstaller linux/AudioMuse-AI.spec --noconfirm``. -Produces a one-dir bundle at ``dist/AudioMuse-AI/`` (the executable plus an -``_internal`` tree with Python, the libraries, the models and the embedded -PostgreSQL/Redis). ``linux/build.sh`` then turns that tree into a ``.deb`` and a -``.rpm``. - -Builds for the architecture of the running Python (CI builds x86_64 and -aarch64). Embedded PostgreSQL is bundled per arch: the pgserver wheel on x86_64, -or a from-source PostgreSQL tree under ``pgsql/`` on aarch64 (pgserver has no -arm64 wheel) -- see ``USE_PGSERVER`` below. -""" - -import glob -import os -import platform - -from PyInstaller.utils.hooks import collect_data_files, collect_dynamic_libs, collect_submodules - -arch = platform.machine() # 'x86_64' or 'aarch64' -# x86_64 embeds PostgreSQL via the pgserver wheel; aarch64 has no pgserver wheel -# and instead bundles a from-source PostgreSQL tree (see -# linux/vendor/postgres/build-postgres.sh). -USE_PGSERVER = arch in ('x86_64', 'amd64') - -# ``SPECPATH`` is the directory containing this spec (``/linux``); the repo -# root is its parent. Anchor every relative source path to the root so the build -# works regardless of CWD or PyInstaller version. -ROOT = os.path.dirname(SPECPATH) - -datas = [ - (os.path.join(ROOT, 'templates'), 'templates'), - (os.path.join(ROOT, 'static'), 'static'), - (os.path.join(ROOT, 'model'), 'model'), - # Root-level data file the app loads relative to config.py's __file__ - # (config.MOOD_CENTROIDS_FILE). config.py freezes into _internal/, so this - # must land at the bundle root ('.', i.e. _internal/) or the mood-similarity - # path scoring and /api/mood_centroids fall back/fail. Without it the log - # shows "Could not load mood centroids from .../_internal/mood_centroids_real_080_clap.json". - (os.path.join(ROOT, 'mood_centroids_real_080_clap.json'), '.'), -] -if USE_PGSERVER: - datas += collect_data_files('pgserver') -else: - # Bundle the entire from-source PostgreSQL install as opaque data under - # ``pgsql/`` (preserving the bin/lib/share layout so the relocatable server - # finds its support files). build.sh chmod +x's pgsql/bin/* afterwards - # (datas don't carry the exec bit). - datas += [(os.path.join(ROOT, 'linux/vendor/postgres', arch), 'pgsql')] -datas += collect_data_files('librosa') -datas += collect_data_files('resampy') -datas += collect_data_files('transformers', include_py_files=False) -datas += collect_data_files('flasgger') -datas += collect_data_files('wn') -datas += collect_data_files('langdetect') - -binaries = [ - (os.path.join(ROOT, f'linux/vendor/redis/{arch}/redis-server'), '.'), -] -binaries += collect_dynamic_libs('av') -binaries += collect_dynamic_libs('voyager') -binaries += collect_dynamic_libs('psycopg2') - -# pgserver (x86_64 only) bundles a minimal PostgreSQL (only plpgsql + pgvector); -# the schema needs the ``unaccent`` and ``pg_trgm`` contrib extensions, which it -# lacks. We vendor them (compiled against pgserver's own headers/ABI -- see -# linux/vendor/pg-contrib/README.md) and graft them into the bundled pgserver -# tree. (On aarch64 the from-source PostgreSQL already includes these contrib -# modules in its own tree, bundled wholesale as ``pgsql/`` above.) -if USE_PGSERVER: - _pg_contrib = os.path.join(ROOT, 'linux/vendor/pg-contrib', arch) - _pg_dst = 'pgserver/pginstall' - for _f in glob.glob(os.path.join(_pg_contrib, 'extension', '*')): - datas.append((_f, f'{_pg_dst}/share/postgresql/extension')) - for _f in glob.glob(os.path.join(_pg_contrib, 'tsearch_data', '*')): - datas.append((_f, f'{_pg_dst}/share/postgresql/tsearch_data')) - for _f in glob.glob(os.path.join(_pg_contrib, 'lib', '*.so')): - binaries.append((_f, f'{_pg_dst}/lib/postgresql')) - -hiddenimports = [ - 'app', - 'rq_worker', - 'rq_worker_high_priority', - 'rq_janitor', - 'restart_listener', - 'waitress', - # Platform-agnostic helpers reused by linux.supervisor from the macos - # package (no GUI deps). Listed explicitly so the frozen build includes - # them without pulling in the rumps-based macos.launcher. - 'macos.control_ipc', - 'macos.reverse_log', -] -hiddenimports += collect_submodules('linux') -hiddenimports += collect_submodules('tasks') -hiddenimports += collect_submodules('lyrics') -hiddenimports += collect_submodules('sklearn') - -# rumps/AppKit are macOS-only. On aarch64 also exclude pgserver: it isn't -# installed (no arm64 wheel) and the shared database.py only references it -# lazily inside functions the aarch64 build never calls. -excludes = ['rumps', 'AppKit', 'Foundation', 'objc'] -if not USE_PGSERVER: - excludes.append('pgserver') - -a = Analysis( - [os.path.join(ROOT, 'linux/launcher.py')], - pathex=[ROOT], - binaries=binaries, - datas=datas, - hiddenimports=hiddenimports, - hookspath=[os.path.join(ROOT, 'macos/hooks')], # hook-tasks.py is platform-agnostic - hooksconfig={}, - runtime_hooks=[], - excludes=excludes, - noarchive=False, -) - -pyz = PYZ(a.pure) - -# strip=False: stripping is DISABLED on Linux because it corrupts the bundle. -# Most of our native deps are manylinux wheels whose shared libraries were -# already rewritten by auditwheel/patchelf (mangled, hashed sonames such as -# ``libscipy_openblas-b75cc656.so`` and pgserver's ``libpq-084d956f.so.5.16``, -# plus injected RPATHs). Running GNU ``strip`` over a patchelf-modified ELF -# breaks it -- the result either SIGSEGVs at load (pgserver's initdb/psql) or -# fails to load with "ELF load command address/offset not page-aligned" (scipy's -# OpenBLAS, which crashes the Flask/worker import of sklearn -> scipy). Stripping -# would save a few hundred MB, but a corrupted bundle does not start at all, so -# correctness wins. (The macOS spec also keeps stripping off.) -exe = EXE( - pyz, - a.scripts, - [], - exclude_binaries=True, - name='AudioMuse-AI', - debug=False, - bootloader_ignore_signals=False, - strip=False, - upx=False, - console=True, -) - -coll = COLLECT( - exe, - a.binaries, - a.datas, - strip=False, - upx=False, - name='AudioMuse-AI', -) diff --git a/linux/README.md b/linux/README.md deleted file mode 100644 index 2745e98c..00000000 --- a/linux/README.md +++ /dev/null @@ -1,181 +0,0 @@ -# AudioMuse-AI — Standalone Linux build (`.deb` / `.rpm`) - -This folder builds AudioMuse-AI into native Linux packages with **no Docker and -no separately-installed PostgreSQL or Redis**. Installing the `.deb` or `.rpm` -drops a fully self-contained app under `/opt/AudioMuse-AI` and a launcher in your -application menu. It is the Linux counterpart of the [`macos/`](../macos) build. - -The package bundles: - -- **Embedded PostgreSQL** with the `unaccent` / `pg_trgm` extensions the schema - needs. On **x86_64** this is [`pgserver`](https://github.com/orm011/pgserver) - (+ the two contrib modules we vendor); on **aarch64** (no pgserver wheel) it's - a relocatable PostgreSQL built from source in CI and driven by `initdb` / - `pg_ctl` (`linux/embedded_pg.py`). Both look identical to the app. -- **Embedded Redis** via a bundled `redis-server` binary. -- The Flask web UI (served by `waitress`) on `http://127.0.0.1:8000`. -- The two RQ workers, the janitor, and the config-restart listener. -- The ~5 GB of ONNX models (MusiCNN, CLAP audio+text, the RoBERTa tokenizer - cache, Whisper-small, gte-multilingual, Silero VAD) — so analysis **and** - lyrics transcription work fully offline. - -A single `audiomuse-ai` process supervises all of the above (start, health-check, -auto-restart, clean shutdown), exactly like the macOS menu-bar agent and the -container's supervisord. - -## Using the package - -```bash -# Debian / Ubuntu -sudo apt install ./AudioMuse-AI-x86_64.deb - -# Fedora / RHEL / openSUSE -sudo dnf install ./AudioMuse-AI-x86_64.rpm -``` - -> Use `apt install ./file.deb` (not `sudo dpkg -i file.deb`). `dpkg -i` does -> **not** pull the two declared dependencies (`libgomp1`, `xdg-utils`) and leaves -> the package half-configured if they are absent — its post-install step (which -> refreshes the menu/icon caches) then never runs, so the launcher may not show -> up. If you already used `dpkg -i`, finish it with `sudo apt-get -f install`. - -### Starting it - -**The app is started on demand — it is not a background daemon, so nothing -listens on `http://127.0.0.1:8000` until you start it** (just like the macOS -menu-bar app). Launch **AudioMuse-AI** from your application menu, or from a -terminal: - -```bash -audiomuse-ai start # start everything + open the browser (foreground) -audiomuse-ai stop # cleanly stop PostgreSQL, Redis and all workers -audiomuse-ai status # is it running? -audiomuse-ai open # open the web UI (starting the stack first if needed) -``` - -The web UI is at `http://127.0.0.1:8000`. After first launch, configure your -media server (Jellyfin, Navidrome, Lyrion, Emby or MPD) exactly as for the -container version. - -If the **AudioMuse-AI** launcher does not appear in your menu right after -install, log out and back in (some desktops only rescan `/usr/share/applications` -on session start). - -### Optional: start automatically at login - -The package ships a **systemd user service** (disabled by default). Enable it to -have AudioMuse-AI start for your user on login and stay supervised in the -background: - -```bash -systemctl --user enable --now audiomuse-ai # start now + on every login -loginctl enable-linger "$USER" # also keep it running after logout -systemctl --user status audiomuse-ai # check it -systemctl --user disable --now audiomuse-ai # turn it back off -``` - -It runs as your user over the same per-user data dir (no root, no system -service), and starts without opening a browser (`AUDIOMUSE_OPEN_BROWSER=0`). - -### Where state lives - -All **writable** state lives under your home (never inside the read-only -`/opt/AudioMuse-AI`), following the XDG spec: - -- Database / Redis / scratch / backups: `~/.local/share/AudioMuse-AI/` -- Logs: `~/.local/state/AudioMuse-AI/logs/audiomuse.log` — written **newest line - first**, bounded to the most recent ~40k lines (see `macos/reverse_log.py`). - -Uninstalling the package leaves this directory in place (so your analysis -database survives a reinstall); delete it by hand if you want a clean slate. - -## Why PostgreSQL and Redis are **bundled**, not system dependencies - -The goal was to prefer hard package dependencies on system PostgreSQL/Redis (so -`apt`/`dnf` pull and auto-start them). We bundle them instead, because a single -portable `.deb` + `.rpm` cannot reliably depend on the database the app needs: - -- **pgvector + exact PostgreSQL minor.** The schema needs `pgvector` *and* the - `unaccent` / `pg_trgm` contrib extensions. PostgreSQL loadable modules are not - ABI-stable across minor releases, and `pgvector` is packaged inconsistently - across distros (a separate `postgresql-NN-pgvector`, or only via the PGDG - repo, or absent). Pinning the exact server version the app was tested against - is exactly what `pgserver` already gives us — and it is the same mechanism the - repo's own integration tests use. -- **Per-distro version skew.** A hard `Depends: postgresql-16` would fail to - install on distros that ship a different major (Ubuntu 22.04 → 14, Debian 12 → - 15, etc.). One package cannot name one "precise version" that exists - everywhere. -- **No root, no auth dance.** The bundled servers run per-user over unix sockets - under `~/.local/share`, with the supervisor owning their lifecycle - (start / health-check / auto-restart / shutdown). There is no system cluster - to provision, no role/password to create, and nothing runs as root. - -This satisfies the goal's fallback ("if NOT possible, bundle everything … in a -similar approach to the macOS one"): the macOS build bundles them the same way, -and the supervisor here gives the same autostart/restart guarantees the goal -asked for — just without depending on the host's database. - -The only genuine system dependencies declared by the package are `libgomp` -(OpenMP runtime onnxruntime links against) and `xdg-utils` (for `xdg-open`). - -## Building (CI) - -`.github/workflows/build-linux.yml` builds **x86_64** (on `ubuntu-22.04`) and -**aarch64** (on `ubuntu-22.04-arm`) on every `v*.*.*` tag and on PRs, and -attaches the `.deb`/`.rpm` to the release. For each arch it: - -1. installs the build toolchain + Python deps (`requirements/linux.txt`; - `pgserver` is x86_64-only via a platform marker), -2. builds the vendored `redis-server` (`linux/vendor/build-redis.sh`), -3. provides embedded PostgreSQL per arch: - - **x86_64** — builds `unaccent`/`pg_trgm` against the pgserver wheel's - PostgreSQL (`linux/vendor/pg-contrib/build-pg-contrib.sh`), - - **aarch64** — builds a relocatable PostgreSQL + those contrib modules from - source (`linux/vendor/postgres/build-postgres.sh`), -4. assembles `./model` from the model releases (trimming the HF cache to the - roberta tokenizer so the assets stay under GitHub's 2 GB limit), -5. runs `linux/build.sh` → PyInstaller (`linux/AudioMuse-AI.spec`) → `nfpm`. - -## Building (developer machine) - -```bash -python3.12 -m venv .venv-linux -source .venv-linux/bin/activate -pip install -r requirements/linux.txt - -# Native build inputs (need build-essential, bison, flex, zlib1g-dev, rpm) -bash linux/vendor/build-redis.sh -# Embedded PostgreSQL — pick the one for your arch: -bash linux/vendor/pg-contrib/build-pg-contrib.sh # x86_64 (against pgserver) -bash linux/vendor/postgres/build-postgres.sh # aarch64 (from source) - -# Models: assemble ./model exactly as the workflow does (see build-linux.yml), -# or copy an existing ./model tree into the repo root. - -# Package (needs nfpm on PATH: https://nfpm.goreleaser.com) -PKG_VERSION=1.0.0 bash linux/build.sh -# -> dist/AudioMuse-AI-.deb and dist/AudioMuse-AI-.rpm -``` - -## Layout of this folder - -| File | Purpose | -| --- | --- | -| `launcher.py` | PyInstaller entry point: `start`/`stop`/`status`/`open` and the `--role=` child entry points. | -| `supervisor.py` | Process supervisor (embedded PG/Redis + Flask + workers); port of `macos/supervisor.py`. | -| `db_backend.py` | Selects the embedded-Postgres backend by arch (pgserver on x86_64, `embedded_pg` on aarch64). | -| `embedded_pg.py` | `initdb`/`pg_ctl` manager for the from-source PostgreSQL bundled on aarch64. | -| `paths.py` | XDG-based writable dirs + bundled-resource locations (incl. per-arch Postgres paths). | -| `env.py` | The environment handed to each child (embedded DB/queue, model paths). | -| `AudioMuse-AI.spec` | PyInstaller one-dir spec. | -| `build.sh` | PyInstaller build + `nfpm` packaging into `.deb`/`.rpm`. | -| `packaging/` | `nfpm` config template, `.desktop` entries, the systemd **user** service, the square app icons (`icons/`), post-install/-remove scripts. | -| `vendor/` | Helper scripts that build `redis-server`, the x86_64 PG contrib modules, and the aarch64 from-source PostgreSQL in CI. | - -> **No shared code is modified by this build.** The `linux/` package only *adds* -> helpers. It reuses the platform-agnostic `macos.control_ipc` / -> `macos.reverse_log` helpers, and it reports `AUDIOMUSE_PLATFORM=macos` to the -> shared `restart_manager.py` so the UI's "restart workers" flow uses the -> control-socket path (the only platform-keyed branch there) — see -> `linux/env.py` for the full rationale. diff --git a/linux/build.sh b/linux/build.sh deleted file mode 100755 index 253f24ce..00000000 --- a/linux/build.sh +++ /dev/null @@ -1,192 +0,0 @@ -#!/usr/bin/env bash -# -# Build the standalone Linux bundle with PyInstaller, then package it as a .deb -# and a .rpm with nfpm. Run from the repo root, inside the build venv: -# -# source .venv-linux/bin/activate -# PKG_VERSION=1.0.0 bash linux/build.sh -# -# Prerequisites (the CI workflow installs these): -# * the Python deps from requirements/linux.txt (incl. pyinstaller, pgserver) -# * the vendored redis-server + pg-contrib for this arch -# (linux/vendor/...; built by linux/vendor/build-redis.sh and -# linux/vendor/pg-contrib/build-pg-contrib.sh) -# * nfpm on PATH (https://nfpm.goreleaser.com) -set -euo pipefail - -PKG_VERSION="${PKG_VERSION:-0.0.0}" -# Strip a leading v from a git tag like v1.2.3 -> 1.2.3 (deb/rpm want a bare ver). -PKG_VERSION="${PKG_VERSION#v}" -# Defensive sanitize: a version with a '/' (e.g. a PR ref like "601/merge") makes -# nfpm emit a path with a missing subdirectory and the build fails. Replace any -# char that is not alphanumeric, dot, plus, tilde or hyphen with a hyphen. -PKG_VERSION="$(printf '%s' "$PKG_VERSION" | tr -c 'A-Za-z0-9.+~-' '-')" -# Collapse/trim stray hyphens and fall back if we sanitized it to nothing. -PKG_VERSION="$(printf '%s' "$PKG_VERSION" | sed -e 's/-\{2,\}/-/g' -e 's/^-//' -e 's/-$//')" -PKG_VERSION="${PKG_VERSION:-0.0.0}" -echo "==> Package version: ${PKG_VERSION}" - -UNAME_ARCH="$(uname -m)" # x86_64 | aarch64 -case "$UNAME_ARCH" in - x86_64) NFPM_ARCH="amd64" ;; - aarch64) NFPM_ARCH="arm64" ;; - *) echo "Unsupported arch: $UNAME_ARCH" >&2; exit 1 ;; -esac - -echo "==> Cleaning previous build" -rm -rf build dist - -echo "==> Verifying vendored native build inputs are present" -# Embedded PostgreSQL differs by arch: x86_64 grafts contrib into the pgserver -# wheel; aarch64 bundles a whole from-source PostgreSQL tree (pgserver has no -# arm64 wheel). -if [ "$UNAME_ARCH" = "x86_64" ]; then - required=( - "linux/vendor/redis/${UNAME_ARCH}/redis-server" - "linux/vendor/pg-contrib/${UNAME_ARCH}/lib/unaccent.so" - "linux/vendor/pg-contrib/${UNAME_ARCH}/lib/pg_trgm.so" - "linux/vendor/pg-contrib/${UNAME_ARCH}/extension/unaccent.control" - "linux/vendor/pg-contrib/${UNAME_ARCH}/extension/pg_trgm.control" - "linux/vendor/pg-contrib/${UNAME_ARCH}/tsearch_data/unaccent.rules" - ) -else - # Fixed-path inputs (the from-source server binaries). - required=( - "linux/vendor/redis/${UNAME_ARCH}/redis-server" - "linux/vendor/postgres/${UNAME_ARCH}/bin/postgres" - "linux/vendor/postgres/${UNAME_ARCH}/bin/initdb" - "linux/vendor/postgres/${UNAME_ARCH}/bin/pg_ctl" - ) -fi -missing=0 -for f in "${required[@]}"; do - if [ ! -s "$f" ]; then echo "::error::Missing vendored file: $f"; missing=1; fi -done -if [ "$UNAME_ARCH" != "x86_64" ]; then - # Contrib modules live in the server's pkglibdir/sharedir, whose exact layout - # depends on the from-source build (plain --prefix layout, not Debian's), so - # locate them rather than hardcoding the subdir. - PGTREE="linux/vendor/postgres/${UNAME_ARCH}" - for f in unaccent.so pg_trgm.so unaccent.control pg_trgm.control; do - if [ -z "$(find "$PGTREE" -name "$f" -print -quit 2>/dev/null)" ]; then - echo "::error::Missing contrib artifact in $PGTREE: $f"; missing=1 - fi - done -fi -[ "$missing" -eq 0 ] || { echo "Vendored inputs missing (see linux/vendor/*/README.md)." >&2; exit 1; } -chmod +x "linux/vendor/redis/${UNAME_ARCH}/redis-server" - -echo "==> Running PyInstaller" -pyinstaller linux/AudioMuse-AI.spec --noconfirm - -BUNDLE="dist/AudioMuse-AI" -[ -x "$BUNDLE/AudioMuse-AI" ] || { echo "::error::PyInstaller did not produce $BUNDLE/AudioMuse-AI"; exit 1; } - -# The aarch64 bundle ships a from-source PostgreSQL under pgsql/ (PyInstaller 6 -# puts data under _internal/). It was added as PyInstaller *data*, which does not -# preserve the executable bit, so restore +x on its binaries (the server -# relocates via rpath; embedded_pg also sets LD_LIBRARY_PATH defensively). -if [ "$UNAME_ARCH" != "x86_64" ]; then - PGBIN="$(find "$BUNDLE" -type d -path '*/pgsql/bin' -print -quit)" - if [ -n "$PGBIN" ]; then - find "$PGBIN" -type f -exec chmod +x {} + - echo "==> Restored +x on bundled PostgreSQL binaries ($PGBIN)" - else - echo "::error::Expected bundled PostgreSQL (pgsql/bin) in $BUNDLE (aarch64 build)"; exit 1 - fi -else - # x86_64: repair the bundled pgserver (pgserver wheel) PostgreSQL tree. - # - # The spec pulls the tree in via collect_data_files('pgserver'), but that - # helper EXCLUDES shared libraries, so every loadable module under - # pginstall/lib/postgresql is dropped: plpgsql.so, vector.so (pgvector), - # dict_snowball.so (initdb's post-bootstrap text-search setup loads it -> - # initdb fails without it), pgoutput, the encoding converters, etc. Without - # them initdb cannot create the cluster and supervisor startup fails. - # - # NOTE: the executables themselves are fine -- the spec sets strip=False - # (stripping is disabled on Linux precisely because it corrupts pgserver's - # patchelf-modified ELFs; see AudioMuse-AI.spec), and collect_data_files - # copies binaries verbatim, so the only thing actually missing is the .so - # modules above. Do NOT re-enable strip=True to "shrink" the bundle: that is - # what would corrupt initdb/psql/pg_dump and make them SIGSEGV at load. - # - # Overlay the COMPLETE, pristine pginstall tree from the installed wheel onto - # the bundle. cp merges: it adds the missing .so modules (and harmlessly - # re-copies the already-correct executables), while leaving the vendored - # unaccent/pg_trgm contrib the spec grafted in (those are not in the wheel). - # Also refresh the external libs (pgserver.libs/). Must run inside the build - # venv where pgserver is importable (the CI step and build.sh's own usage both - # activate it). - PG_PKG="$(python -c 'import os, pgserver; print(os.path.dirname(os.path.abspath(pgserver.__file__)))')" - PG_SITE="$(dirname "$PG_PKG")" - DST_PGINSTALL="$BUNDLE/_internal/pgserver/pginstall" - [ -d "$PG_PKG/pginstall/bin" ] && [ -d "$DST_PGINSTALL" ] || { - echo "::error::Cannot locate pgserver pginstall to restore (src=$PG_PKG/pginstall dst=$DST_PGINSTALL)"; exit 1; } - cp -af "$PG_PKG/pginstall/." "$DST_PGINSTALL/" - if [ -d "$PG_SITE/pgserver.libs" ] && [ -d "$BUNDLE/_internal/pgserver.libs" ]; then - cp -af "$PG_SITE/pgserver.libs/." "$BUNDLE/_internal/pgserver.libs/" - fi - echo "==> Restored complete unstripped pgserver tree into $DST_PGINSTALL" - # Smoke-test: a real initdb into a temp dir must succeed (catches a regression - # in this restore, a broken wheel, or a still-missing module like - # dict_snowball). Fail the build loudly rather than ship a package that cannot - # start. - _pgt="$(mktemp -d)" - if ! "$DST_PGINSTALL/bin/initdb" -D "$_pgt/d" --auth=trust --encoding=utf8 -U postgres >"$_pgt/log" 2>&1; then - echo "::error::Bundled initdb failed after restore:"; cat "$_pgt/log"; rm -rf "$_pgt"; exit 1 - fi - echo "==> Verified bundled initdb creates a cluster ($("$DST_PGINSTALL/bin/initdb" --version))" - rm -rf "$_pgt" -fi - -echo "==> Bundle size breakdown (to spot what dominates the package)" -echo " total bundle:" -du -sh "$BUNDLE" || true -echo " largest dirs under _internal:" -du -sh "$BUNDLE"/_internal/* 2>/dev/null | sort -rh | head -20 || true -echo " largest single files:" -find "$BUNDLE" -type f -printf '%s\t%p\n' 2>/dev/null | sort -rn | head -20 \ - | awk '{ printf " %8.1f MB %s\n", $1/1048576, $2 }' || true - -echo "==> Staging package payload" -STAGE="dist/_pkg" -rm -rf "$STAGE" -mkdir -p "$STAGE/opt" "$STAGE/usr/share/applications" \ - "$STAGE/usr/lib/systemd/user" -cp -a "$BUNDLE" "$STAGE/opt/AudioMuse-AI" -cp linux/packaging/AudioMuse-AI.desktop "$STAGE/usr/share/applications/AudioMuse-AI.desktop" -cp linux/packaging/AudioMuse-AI-stop.desktop "$STAGE/usr/share/applications/AudioMuse-AI-stop.desktop" -cp linux/packaging/audiomuse-ai.service "$STAGE/usr/lib/systemd/user/audiomuse-ai.service" - -for size in 512 256 128 64 48 32; do - src="linux/packaging/icons/audiomuse-ai_${size}.png" - [ -s "$src" ] || { echo "::error::Missing square icon source: $src"; exit 1; } - dst="$STAGE/usr/share/icons/hicolor/${size}x${size}/apps" - mkdir -p "$dst" - cp "$src" "$dst/audiomuse-ai.png" -done - -echo "==> Generating nfpm config" -mkdir -p dist -sed -e "s|@VERSION@|${PKG_VERSION}|g" \ - -e "s|@ARCH@|${NFPM_ARCH}|g" \ - -e "s|@STAGE@|${STAGE}|g" \ - linux/packaging/nfpm.yaml.in > dist/nfpm.yaml - -echo "==> Building .deb and .rpm with nfpm (arch=${NFPM_ARCH})" -mkdir -p dist/pkg -nfpm package --config dist/nfpm.yaml --packager deb --target dist/pkg/ -nfpm package --config dist/nfpm.yaml --packager rpm --target dist/pkg/ - -echo "==> Built packages:" -ls -lh dist/pkg/ - -# Normalize the output names so the workflow can find them deterministically. -DEB="$(ls dist/pkg/*.deb | head -n1)" -RPM="$(ls dist/pkg/*.rpm | head -n1)" -cp "$DEB" "dist/AudioMuse-AI-${UNAME_ARCH}-linux.deb" -cp "$RPM" "dist/AudioMuse-AI-${UNAME_ARCH}-linux.rpm" -echo "==> Done:" -echo " dist/AudioMuse-AI-${UNAME_ARCH}-linux.deb" -echo " dist/AudioMuse-AI-${UNAME_ARCH}-linux.rpm" diff --git a/linux/packaging/nfpm.yaml.in b/linux/packaging/nfpm.yaml.in index 338fa819..8bfa6820 100644 --- a/linux/packaging/nfpm.yaml.in +++ b/linux/packaging/nfpm.yaml.in @@ -1,5 +1,5 @@ # nfpm config template for the AudioMuse-AI native Linux package. -# linux/build.sh expands @VERSION@ / @ARCH@ / @STAGE@ and runs: +# scripts/standalone/platforms/linux.py expands @VERSION@ / @ARCH@ / @STAGE@ and runs: # nfpm package --packager deb ... # nfpm package --packager rpm ... # See https://nfpm.goreleaser.com/configuration/ diff --git a/linux/vendor/README.md b/linux/vendor/README.md deleted file mode 100644 index 2cf5bc0f..00000000 --- a/linux/vendor/README.md +++ /dev/null @@ -1,25 +0,0 @@ -# Vendored native build inputs (Linux) - -The native Linux bundle embeds Redis and two PostgreSQL contrib extensions that -`pgserver`'s minimal PostgreSQL does not ship. Both are **built fresh in CI** -(not committed to git) by the helper scripts here, then copied into the bundle -by `linux/AudioMuse-AI.spec`. - -``` -linux/vendor/build-redis.sh # -> redis//redis-server -linux/vendor/pg-contrib/build-pg-contrib.sh # -> pg-contrib//{lib,extension,tsearch_data}/... -``` - -`` is `x86_64` or `aarch64` (the output of `uname -m`). - -## Why built in CI rather than committed - -The macOS build commits its `redis-server` / pg-contrib artifacts for -byte-for-byte reproducibility. On Linux we build them per-arch in the workflow -instead: committing per-distro ELF binaries to git is heavy, and building from -source on the oldest supported runner (Ubuntu 22.04) gives broad glibc -compatibility. The pinned versions (`REDIS_VERSION`, and PostgreSQL = whatever -`pgserver` bundles) make the result deterministic enough for our purposes. - -See `pg-contrib/README.md` for why the contrib modules must be compiled against -the *exact* PostgreSQL version `pgserver` ships. diff --git a/linux/vendor/pg-contrib/README.md b/linux/vendor/pg-contrib/README.md deleted file mode 100644 index c76c6622..00000000 --- a/linux/vendor/pg-contrib/README.md +++ /dev/null @@ -1,49 +0,0 @@ -# Vendored PostgreSQL contrib extensions (`unaccent`, `pg_trgm`) — Linux - -`pgserver` ships a **minimal** PostgreSQL (only `plpgsql` and `pgvector`). -The AudioMuse-AI schema (`app_helper.py::init_db`) requires two contrib -extensions that pgserver does **not** include: - -```sql -CREATE EXTENSION IF NOT EXISTS unaccent; -CREATE EXTENSION IF NOT EXISTS pg_trgm; -``` - -So we build those two modules from PostgreSQL source and the PyInstaller spec -(`linux/AudioMuse-AI.spec`) grafts them into the bundled `pgserver/pginstall` -tree (`.so` → `lib/postgresql/`, `*.control`/`*--*.sql` → -`share/postgresql/extension/`, `unaccent.rules` → -`share/postgresql/tsearch_data/`). - -## Why these must be built from the exact pgserver PG version (not the distro's) - -PostgreSQL loadable modules are **not** ABI-stable across minor releases. A -module copied from the distro's `postgresql-16` package fails to load in -pgserver's bundled server if the minor versions differ. The modules must be -compiled against the **exact** server version pgserver ships -(`pgserver==0.1.4` → PostgreSQL 16.2). pgserver conveniently bundles -`pg_config`, the server headers and `pgxs`, so the modules compile directly -against its own install. - -## How to (re)generate - -Run on the target arch (the CI workflow runs it on x86_64 and aarch64 runners), -from the repo root inside the build venv: - -```bash -source .venv-linux/bin/activate # so pgserver is importable -bash linux/vendor/pg-contrib/build-pg-contrib.sh -``` - -Output lands in `linux/vendor/pg-contrib//` (`x86_64` or `aarch64`): - -``` -/lib/unaccent.so -/lib/pg_trgm.so -/extension/{unaccent,pg_trgm}.control -/extension/{unaccent,pg_trgm}--*.sql -/tsearch_data/unaccent.rules -``` - -These are built fresh in CI and are **not** committed to git (see -`../README.md`). diff --git a/linux/vendor/postgres/README.md b/linux/vendor/postgres/README.md deleted file mode 100644 index f05e62bd..00000000 --- a/linux/vendor/postgres/README.md +++ /dev/null @@ -1,42 +0,0 @@ -# Vendored PostgreSQL (Linux aarch64) - -`pgserver` (used on x86_64) ships **no Linux/aarch64 wheel**, so the arm64 build -bundles its own PostgreSQL. `build-postgres.sh` compiles a relocatable -PostgreSQL — server, client tools, and the `unaccent` / `pg_trgm` contrib -extensions the schema needs — from source and installs it into -`linux/vendor/postgres//`. The PyInstaller spec then bundles that tree as -`pgsql/`, and `linux/embedded_pg.py` drives it at runtime with -`initdb` + `pg_ctl`. - -> **pgvector is intentionally not built.** The app does vector similarity -> in-process via the `voyager` library and never runs `CREATE EXTENSION vector` -> (it only creates `unaccent` and `pg_trgm`), so the arm64 server doesn't need -> pgvector. (The x86_64 `pgserver` wheel happens to include it; it's unused.) - -## How to (re)generate - -Run on the target arch (the CI workflow runs it on the aarch64 runner), from the -repo root. Needs `build-essential`, `bison`, `flex`, `zlib1g-dev`: - -```bash -bash linux/vendor/postgres/build-postgres.sh # PG_VERSION=16.9 by default -``` - -Output (built fresh in CI, **not** committed — see `../README.md`): - -``` -/bin/{postgres,initdb,pg_ctl,psql,pg_dump,pg_restore,...} -/lib/... # server libs -/lib/postgresql/{unaccent,pg_trgm}.so -/share/postgresql/... # incl. extension/*.{control,sql}, tsearch_data/unaccent.rules -``` - -## Why from source (not a prebuilt binary) - -Building the `unaccent` / `pg_trgm` contrib modules requires the server headers -and `pgxs`, which runtime-only binary distributions (e.g. zonky's -embedded-postgres-binaries) do not ship. Building from source also lets us pass -`--without-icu` (no libicu runtime dependency; initdb uses the libc locale -provider) and `--without-readline`, keeping the bundle self-contained. -PostgreSQL is relocatable — it derives its support-file paths from the running -executable — so the installed tree works from wherever the package is installed. diff --git a/macos/AudioMuse-AI.spec b/macos/AudioMuse-AI.spec deleted file mode 100644 index d16dfc0b..00000000 --- a/macos/AudioMuse-AI.spec +++ /dev/null @@ -1,130 +0,0 @@ -# -*- mode: python ; coding: utf-8 -*- -"""PyInstaller spec for the standalone macOS app. - -Run from the repo root: ``pyinstaller macos/AudioMuse-AI.spec --noconfirm``. -Builds for the architecture of the running Python (build once on Apple Silicon, -once on Intel -- universal2 is avoided because onnxruntime/PyAV/voyager wheels -are not reliably universal2). -""" - -import glob -import os -import platform - -from PyInstaller.utils.hooks import collect_data_files, collect_dynamic_libs, collect_submodules - -arch = platform.machine() - -# ``SPECPATH`` is the directory containing this spec (``/macos``); the repo -# root is its parent. Anchor every relative source path to the root so the build -# works regardless of CWD or PyInstaller version (PyInstaller 6.x resolves -# relative spec paths against SPECPATH, older versions against the CWD). -ROOT = os.path.dirname(SPECPATH) - -datas = [ - (os.path.join(ROOT, 'templates'), 'templates'), - (os.path.join(ROOT, 'static'), 'static'), - (os.path.join(ROOT, 'model'), 'model'), - (os.path.join(ROOT, 'macos/assets'), 'assets'), - # Root-level data file the app loads relative to config.py's __file__ - # (config.MOOD_CENTROIDS_FILE). config.py freezes into _internal/, so this - # must land at the bundle root ('.', i.e. _internal/) or the mood-similarity - # path scoring and /api/mood_centroids fall back/fail. Without it the log - # shows "Could not load mood centroids from .../_internal/mood_centroids_real_080_clap.json". - (os.path.join(ROOT, 'mood_centroids_real_080_clap.json'), '.'), -] -datas += collect_data_files('pgserver') -datas += collect_data_files('librosa') -datas += collect_data_files('resampy') -datas += collect_data_files('transformers', include_py_files=False) -datas += collect_data_files('flasgger') -datas += collect_data_files('wn') -datas += collect_data_files('langdetect') - -binaries = [ - (os.path.join(ROOT, f'macos/vendor/redis/{arch}/redis-server'), '.'), -] -binaries += collect_dynamic_libs('av') -binaries += collect_dynamic_libs('voyager') -binaries += collect_dynamic_libs('psycopg2') - -# pgserver bundles a minimal PostgreSQL 16.2 (only plpgsql + pgvector); the -# schema needs the ``unaccent`` and ``pg_trgm`` contrib extensions, which it -# lacks. We vendor them (compiled from PostgreSQL 16.2 source against pgserver's -# own headers/ABI -- see macos/vendor/pg-contrib/README.md) and graft them into -# the bundled pgserver tree. ``collect_data_files('pgserver')`` above lays down -# pginstall under ``pgserver/pginstall``; these land beside it. -_pg_contrib = os.path.join(ROOT, 'macos/vendor/pg-contrib', arch) -_pg_dst = 'pgserver/pginstall' -for _f in glob.glob(os.path.join(_pg_contrib, 'extension', '*')): - datas.append((_f, f'{_pg_dst}/share/postgresql/extension')) -for _f in glob.glob(os.path.join(_pg_contrib, 'tsearch_data', '*')): - datas.append((_f, f'{_pg_dst}/share/postgresql/tsearch_data')) -for _f in glob.glob(os.path.join(_pg_contrib, 'lib', '*.dylib')): - binaries.append((_f, f'{_pg_dst}/lib/postgresql')) - -hiddenimports = [ - 'app', - 'numeric_bootstrap', - 'rq_worker', - 'rq_worker_high_priority', - 'rq_janitor', - 'restart_listener', - 'waitress', - 'rumps', -] -hiddenimports += collect_submodules('tasks') -hiddenimports += collect_submodules('lyrics') -hiddenimports += collect_submodules('macos') -hiddenimports += collect_submodules('sklearn') - -a = Analysis( - [os.path.join(ROOT, 'macos/launcher.py')], - pathex=[ROOT], - binaries=binaries, - datas=datas, - hiddenimports=hiddenimports, - hookspath=[os.path.join(ROOT, 'macos/hooks')], - hooksconfig={}, - runtime_hooks=[], - excludes=[], - noarchive=False, -) - -pyz = PYZ(a.pure) - -exe = EXE( - pyz, - a.scripts, - [], - exclude_binaries=True, - name='AudioMuse-AI', - debug=False, - bootloader_ignore_signals=False, - strip=False, - upx=False, - console=False, -) - -coll = COLLECT( - exe, - a.binaries, - a.datas, - strip=False, - upx=False, - name='AudioMuse-AI', -) - -app = BUNDLE( - coll, - name='AudioMuse-AI.app', - icon=os.path.join(ROOT, 'macos/assets/AudioMuse-AI.icns'), - bundle_identifier='ai.audiomuse.standalone', - info_plist={ - 'LSUIElement': True, - 'NSHighResolutionCapable': True, - 'CFBundleName': 'AudioMuse-AI', - 'CFBundleDisplayName': 'AudioMuse-AI', - 'CFBundleShortVersionString': '1.0.0', - }, -) diff --git a/macos/README.md b/macos/README.md deleted file mode 100644 index 6dd566f8..00000000 --- a/macos/README.md +++ /dev/null @@ -1,362 +0,0 @@ -# AudioMuse-AI — Standalone macOS App - -This folder builds AudioMuse-AI into a single double-clickable **`AudioMuse-AI.app`** -with **no Docker, no separately-installed PostgreSQL or Redis**. The app runs as a -menu-bar agent that starts everything for you: - -- **Embedded PostgreSQL** via [`pgserver`](https://github.com/orm011/pgserver) (the same - embedded server already used in this repo's integration tests). -- **Embedded Redis** via a bundled `redis-server` binary. -- The Flask web UI (served by `waitress`) on `http://127.0.0.1:8000`. -- The two RQ workers, the janitor, and the config-restart listener. - -The ~6.5 GB of models (MusiCNN, CLAP audio+text, the RoBERTa HF cache, Whisper-small, -gte-multilingual, Silero VAD) are bundled inside the app, so analysis **and** lyrics -transcription work fully offline. - -All writable state lives in your Library, never inside the (read-only, signed) app: - -- Database / Redis / scratch: `~/Library/AudioMuse-AI/` -- Logs: `~/Library/Logs/AudioMuse-AI/audiomuse.log` — written **newest line first**, so - opening it shows the latest activity at the top. Bounded to the most recent ~40k - lines (see `macos/reverse_log.py`). - -## Menu-bar items - -- **Open in Browser** — opens the web UI at `http://127.0.0.1:8000`. -- **Pause Server / Start Server** — stops or restarts all embedded services and workers. -- **Open Log** — opens the log (newest line first) in Console.app. -- **Quit** — cleanly shuts down PostgreSQL, Redis and every worker (no orphans). - -After first launch, open the UI and configure your media server (Jellyfin, Navidrome, -Lyrion, Emby or MPD) exactly as you would for the container version. - ---- - -## Building (developer machine) - -> **Apple Silicon only** (M1/M2/M3/M4...). The macOS build targets arm64 -> exclusively — there is no Intel/x86_64 build, and universal2 is not used -> (onnxruntime/PyAV/voyager wheels aren't reliably universal2). Build on an -> Apple Silicon Mac. - -### Prerequisites - -1. macOS with **Xcode Command Line Tools** (`xcode-select --install`) — provides - `codesign`, `sips`, `iconutil`, `ditto`. -2. **Python 3.12** (matching the project venv). -3. The native build inputs are **committed in git** and used as-is — you do **not** - regenerate them per build: - - `macos/vendor/redis/arm64/redis-server` (Redis 8.8.0, no-TLS, self-contained) - - `macos/vendor/pg-contrib/arm64/` (the `unaccent`/`pg_trgm` Postgres extensions, - compiled against pgserver's PostgreSQL 16.2) - - See `macos/vendor/redis/README.md` and `macos/vendor/pg-contrib/README.md` for - how to regenerate them (only needed when bumping Redis or the `pgserver`/PG - version). -4. The **models** are NOT in git (they are ~6.5 GB). Assemble `./model` from the - project releases before building — the CI workflow `.github/workflows/build-macos.yml` - does exactly this and is the source of truth. In short, into `./model`: - - musicnn + CLAP text (`musicnn_embedding.onnx`, `musicnn_prediction.onnx`, - `clap_text_model.onnx`) and the DCLAP audio model - (`model_epoch_36.onnx` + `.data`); - - the HuggingFace cache → `model/huggingface/` (from `huggingface_models.tar.gz`); - - the lyrics bundles → `whisper-small-onnx/`, `silero_vad.onnx`, - `gte-multilingual-base-int8.onnx`, `gte-multilingual-base/` (from - `lyrics_model_whisper.tar.gz`, `lyrics_model_silero_vad.tar.gz`, - `lyrics_model_gte_vnni.tar.gz`). - -### Steps - -```bash -cd -python3.12 -m venv .venv-macos -source .venv-macos/bin/activate -pip install -r requirements/macos.txt - -bash macos/build.sh -``` - -`build.sh` will: -1. Generate `AudioMuse-AI.icns` + the menu-bar icon from `screenshot/audiomuseai.png`. -2. Run PyInstaller against `macos/AudioMuse-AI.spec`. -3. **Ad-hoc sign** every nested binary (Postgres, Redis, dylibs) and then the bundle. -4. Produce `dist/AudioMuse-AI-.zip`. - -The build is **not notarized and not Developer-ID signed** — we have no Apple -Developer account. That is expected; see the next section for how users open it. - ---- - -## Installing & authorizing (end users) - -Because the app is **unsigned** (built without a paid Apple Developer account), -macOS Gatekeeper will refuse to open it on first try, and on macOS Sequoia (15+) -the old right-click → Open shortcut no longer works. Use **one** of these: - -### Option A — Terminal (recommended, one command) - -1. Unzip and move `AudioMuse-AI.app` to `/Applications`. -2. Run: - ```bash - xattr -dr com.apple.quarantine /Applications/AudioMuse-AI.app - ``` - This removes the download "quarantine" flag from the whole app (including the - bundled Postgres/Redis binaries), so Gatekeeper stops blocking it. -3. Double-click the app. The AudioMuse-AI icon appears in your menu bar. - -### Option B — System Settings (no Terminal) - -1. Move the app to `/Applications` and double-click it. macOS shows a warning; - dismiss it. -2. Open **System Settings → Privacy & Security**, scroll to the **Security** - section, and click **Open Anyway** next to AudioMuse-AI. Authenticate. -3. Launch the app again and confirm. - -> The `xattr` command is safe and expected for unsigned open-source apps; it only -> removes the "downloaded from the internet" marker. If you prefer not to run it, -> use Option B. - -### First launch notes - -- First boot initializes the embedded PostgreSQL data directory and can take a - little longer; the menu-bar status shows **Starting…** then **Running**. -- If something looks stuck, use **Open Log** to inspect - `~/Library/Logs/AudioMuse-AI/audiomuse.log`. - ---- - -## Note to the AI (architecture & debugging handoff) - -> Read this first if you are an AI assistant picking up this work in a fresh session -> or on another machine. It explains *what* was changed and *why* so you don't have -> to re-derive it. Verify any file/line reference still exists before acting on it. - -### The core problem and the chosen strategy - -AudioMuse-AI was container-only: a Flask app + RQ workers needing an **external -PostgreSQL** and **external Redis**. The goal was a single double-clickable macOS -`.app` with neither. The blockers were that database access (~897 raw-SQL sites -across ~37 files, leaning on Postgres-only features: `ON CONFLICT`, `RETURNING`, -`JSONB`, `pg_trgm`/`unaccent`, PL/pgSQL triggers, advisory locks, -`information_schema`) and Redis/RQ usage (across ~51 files) are spread everywhere. - -**Decision: embed real PostgreSQL + real Redis rather than port to SQLite / a -non-Redis queue.** This keeps the SQL dialect and all RQ semantics (Lua `EVAL`, -pub/sub, job registries, `job.meta`, `Job.fetch`, `send_stop_job_command`) -**byte-for-byte unchanged**. No call site was rewritten — that is the whole point, -and it is how the "no miss anyone" requirement is satisfied structurally. - -### The seam (this is the key idea) - -Connection/queue construction was already centralized in **one** place -(`app_helper.py`) and config is read from env in **one** place (`config.py`). So the -abstraction is config-driven dispatch in the spirit of `tasks/mediaserver.py`: - -- **`database.py`** (repo root) owns `get_db()`/`close_db()`, dispatched on - `config.DATABASE_TYPE`. It also exposes `start_embedded(data_dir)` / - `stop_embedded()` (pgserver) used **only by the macOS supervisor**. -- **`taskqueue.py`** (repo root) owns `redis_conn`, `rq_queue_high`, - `rq_queue_default`, dispatched on `config.QUEUE_TYPE`. It re-exports the RQ - symbols the codebase uses and exposes `build_embedded_redis_argv(...)`. -- **`app_helper.py`** no longer constructs anything; it does - `from database import get_db, close_db` and - `from taskqueue import redis_conn, rq_queue_high, rq_queue_default, Job, NoSuchJobError, send_stop_job_command`. - Every existing `from app_helper import …` keeps working unchanged. - -Both `postgres`/`embedded` (DB) and `redis`/`embedded` (queue) connect identically — -they read `config.DATABASE_URL` / `config.REDIS_URL`. The **only** difference in -embedded mode is *who starts the server* (the supervisor) and *what URL the env -points at* (the embedded socket). Connections are still plain `psycopg2.connect` / -`Redis.from_url`. - -### Config switches (defined once in `config.py`, defaults preserve cloud behavior) - -`DATABASE_TYPE` (`postgres`), `QUEUE_TYPE` (`redis`), `APP_DATA_DIR` (`""`), -`AUDIOMUSE_PLATFORM` (`""`), `AUDIOMUSE_CONTROL_SOCKET` (`""`). The macOS supervisor -sets `DATABASE_TYPE=embedded`, `QUEUE_TYPE=embedded`, `AUDIOMUSE_PLATFORM=macos` and -the embedded `DATABASE_URL`/`REDIS_URL` **into each child's environment**. Because -`config.py` reads env at import, each child imports `config` already pointed at the -embedded services. With defaults unset, Docker/K8s behavior is unchanged. - -### macOS process model - -The frozen binary is **one** executable that behaves two ways: -- No args → runs the **rumps menu-bar agent** (`macos/launcher.py::_run_menubar`), - which constructs a `ProcessSupervisor` and auto-starts it. It first takes an - `flock` **single-instance lock** (`_acquire_single_instance_lock`): a second - agent would be catastrophic because both manage the *same* embedded - Postgres/Redis and a freshly started `redis-server` unlinks the live unix - socket out from under the running stack — so the second launch just opens the - UI and exits. The OS drops the lock if the holder dies, so a crash-relaunch - cleanly takes over (and `_reap_stale_infra` reaps any orphaned children). -- `--role=` → runs **one child service** (`_run_role`). Workers/janitor/listener - are re-run via `runpy.run_module(, run_name="__main__")` (they have no - reusable `main()` except `restart_listener`); the web server is `waitress.serve` - on `app.app`. Children are spawned by the supervisor as - `[sys.executable, "--role=…"]`. - -**`ProcessSupervisor` (`macos/supervisor.py`)** boots in order: embedded Postgres -(`database.start_embedded`) → embedded Redis (bundled binary, args from -`taskqueue.build_embedded_redis_argv`) → `flask` (gated on an HTTP readiness probe; -this is what runs `init_db()` and creates the schema) → `rq-worker-high` → -`rq-worker-default` → `rq-janitor` → `restart-listener`. Children spawn with -`start_new_session=True`; shutdown is reverse-order `killpg(SIGTERM)` → `SIGKILL` -backstop, then `database.stop_embedded()`. A health thread restarts children that -exit while `RUNNING` (mirrors supervisord `autorestart`; note RQ workers -intentionally exit after `max_jobs` and are meant to be respawned). It also -keeps the **infrastructure** alive each cycle: it runs `SELECT 1` against -embedded Postgres (`_ensure_postgres_healthy` → `database.ensure_embedded_running`) -and pings embedded Redis (`_ensure_redis_healthy`), restarting either if it died -or stopped answering. Postgres and Redis are NOT `_desired` children, so without -these checks a death of either leaves the workers crash-looping forever (see -gotcha #14). Restarts reuse the same unix-socket paths, so children's -`DATABASE_URL`/`REDIS_URL` stay valid and they reconnect on their own. Orphans from a crashed previous run are -reaped on startup via a PID file + `psutil`, **plus** a path sweep -(`_reap_stale_infra`) that kills any `redis-server`/`postgres` referencing our own -data dirs — the pidfile misses processes left by a force-quit, and stale Redis -instances share the one socket path and unlink it on exit. - -**Control plane.** The container uses supervisord; macOS has none. The web UI's -"save config → restart workers" flow publishes to Redis → `restart_listener` -(a supervised child) → `restart_manager`. On macOS (`config.AUDIOMUSE_PLATFORM == -"macos"`) `restart_manager._run_supervisorctl` / `_spawn_supervisorctl` instead send -one JSON line `{"action", "services"}` to `config.AUDIOMUSE_CONTROL_SOCKET`, served -by `macos/control_ipc.ControlServer`, which calls -`ProcessSupervisor.dispatch_control`. The service names (`flask`, -`rq-worker-default`, `rq-worker-high`, `rq-janitor`) map 1:1 to supervisor children. - -### Surgical edits outside `/macos` (and why) - -- `config.py:40` — `TEMP_DIR` was a hardcoded `/app/temp_audio` constant (unwritable - inside a read-only bundle); now `os.environ.get(...)`. -- `flask_app.py` — `template_folder`/`static_folder` resolve via `sys._MEIPASS` when - `getattr(sys, "frozen", False)`; identical to before in dev. -- `restart_manager.py` — the macOS control-socket branch described above. -- Model paths were **already** env-driven (`EMBEDDING_MODEL_PATH`, - `PREDICTION_MODEL_PATH`, `CLAP_AUDIO_MODEL_PATH`, `CLAP_TEXT_MODEL_PATH`, - `LYRICS_MODEL_DIR`); `macos/env.py` just repoints them into the bundled `model/` - dir. No code change there. - -### Gotchas / where bugs will hide (ranked) - -1. **Unsigned nested binaries** (Postgres, Redis, dylibs) get killed by - Gatekeeper/quarantine. Mitigated by ad-hoc signing everything in `build.sh` + - the `xattr -dr com.apple.quarantine` user step. No hardened runtime, no - notarization (we have no Apple Developer account). -2. **RQ job funcs not importable when frozen** — RQ imports `tasks.foo.bar` by - string at run time; static analysis misses them. Fixed by - `macos/hooks/hook-tasks.py` (`collect_submodules("tasks")`). If jobs fail with - ModuleNotFoundError only at run time, this hook (or the spec hiddenimports) is - the place to look. -3. **pgserver binaries under PyInstaller** — collected via - `collect_data_files("pgserver")`. If `pgserver.get_server` can't find - initdb/postgres, set `pgserver.POSTGRES_BIN_PATH` to the bundled path before - calling it. (pgserver API used here: `get_server(data_dir)` → `.get_uri()` → - `.cleanup()`, mirroring `test/test_provider_migration_integration.py`.) -4. **pgserver ships a minimal Postgres without `unaccent`/`pg_trgm`** — its 16.2 - build only has `plpgsql` + `pgvector`, but `init_db` does - `CREATE EXTENSION unaccent`/`pg_trgm`, so first boot dies with - `extension "unaccent" is not available`. Fix: the two contrib modules are - vendored under `macos/vendor/pg-contrib//` and grafted into the bundled - `pgserver/pginstall` tree by the spec. They must be **compiled from the exact - PostgreSQL source version pgserver ships** (16.2) against pgserver's own - headers/pgxs — a module from a different minor (e.g. Homebrew's 16.14) fails to - load with `Symbol not found`. See `macos/vendor/pg-contrib/README.md` to - regenerate (and to add the `x86_64/` set when building on Intel). -5. **Spaces in the writable data dir break embedded Postgres** — pgserver uses - the data dir as the cluster's unix-socket dir and forwards it to `postgres` - via `pg_ctl -o '-k '`, a single string `postgres` re-splits on - whitespace. A path like `~/Library/Application Support/AudioMuse-AI/pgdata` - fails at startup with `postgres: invalid argument: "Support/..."`. That is why - `macos/paths.py::app_support_dir` uses the space-free `~/Library/AudioMuse-AI` - instead of `Application Support`. Keep the whole writable root space-free. -6. **numba "cannot cache function"** in frozen apps via librosa — `macos/env.py` - sets `NUMBA_CACHE_DIR` to a writable dir before any import. -7. **LSUIElement may be ignored** by the PyInstaller bootloader, showing a Dock - icon. Belt-and-braces: Info.plist key in the spec **and** a runtime - `NSApp().setActivationPolicy_(Accessory)` in `_run_menubar`. -8. **Apple Silicon only, not universal2** — onnxruntime/PyAV/voyager wheels aren't - reliably universal2, and the target is arm64 exclusively (no Intel build). Build - on an Apple Silicon Mac; the committed vendor binaries are arm64-only. -9. **waitress, not gunicorn**, for the macOS web server — avoids fork-in-frozen-app - fragility. The RQ workers still fork per job (that's separate and unchanged). -10. **RQ worker fork + Objective-C = crash on macOS** — the per-job `fork()` in - the workers aborts the child with `+[NSNumber initialize] may have been in - progress in another thread when fork() was called. Crashing instead.` - (Foundation is pulled in transitively), so jobs silently never run even - though Redis/queues look healthy. Fixed by exporting - `OBJC_DISABLE_INITIALIZE_FORK_SAFETY=YES` into every child env in - `macos/env.py`. Symptom if it regresses: web UI works, workers "Listening", - but analysis never progresses and the log shows repeated `objc[...]` aborts. -11. **TCP-only Redis socket options over a unix socket** — embedded mode connects - via `unix://`, whose `UnixDomainSocketConnection` rejects `socket_keepalive` - (`TypeError: ... unexpected keyword argument 'socket_keepalive'`), killing the - workers. `taskqueue.redis_socket_options(url)` drops that kwarg for `unix://` - URLs and keeps it for the TCP `redis://` URLs used by the container/cloud. -12. **HF_HOME must point at the bundled model cache** — analysis lazy-loads the - CLAP RoBERTa tokenizer with `AutoTokenizer.from_pretrained("roberta-base", - local_files_only=True)`. The container pre-bakes that under - `/app/.cache/huggingface` and sets `HF_HOME`; we bundle the same cache at - `model/huggingface` (already collected by the spec's `('model','model')`) and - `macos/env.py` sets `HF_HOME` there plus `HF_HUB_OFFLINE/TRANSFORMERS_OFFLINE`. - Symptom if it regresses: audio analysis runs but CLAP text embeddings fail - with `LocalEntryNotFoundError` / "couldn't connect to huggingface.co", and - `other_features` come out as zeros. -13. **Lyrics ONNX models need explicit path overrides** — `lyrics/silero_onnx.py` - and `lyrics/gte_onnx.py` hardcode container `/app/model/...` defaults and do - NOT derive from `LYRICS_MODEL_DIR`, so `macos/env.py` must set - `SILERO_VAD_ONNX_PATH`, `LYRICS_GTE_ONNX_PATH` and `LYRICS_GTE_TOKENIZER_DIR` - (plus `LYRICS_WHISPER_MODEL_DIR`) at `/model/...`, and the - `whisper-small-onnx/`, `silero_vad.onnx`, `gte-multilingual-base-int8.onnx` - and `gte-multilingual-base/` files must be present in `./model`. Symptom if it - regresses: audio+CLAP succeed but lyrics fail with `... not found at - /app/model/...` and `other_features`/lyrics are skipped. -14. **Embedded Postgres/Redis deaths are unrecoverable without explicit - supervision** — both are spawned outside the `_desired` child set, so the - child health check does NOT cover them. If either exits (crash/OOM) — or - Redis's unix socket is unlinked — every worker/flask/janitor crash-loops - (`Error 2 connecting to .../redis.sock. No such file or directory`, or - `could not connect to server` for Postgres) and never recovers. The health - loop therefore probes both every cycle (`_ensure_postgres_healthy` runs - `SELECT 1`; `_ensure_redis_healthy` pings) and restarts whichever is down - (`database.ensure_embedded_running` for PG — needed because - `pgserver.get_server` returns its cached object without restarting a dead - postmaster). Related Redis trap: all Redis instances share ONE socket path, - so a leftover `redis-server` from a force-quit (which skips `stop_all`) - unlinks the live socket on exit — `_reap_stale_infra` sweeps stale - `redis-server`/`postgres` by data-dir match at startup. Symptom: mass - `redis.sock`/Postgres connection tracebacks, often with flask `Address - already in use` (port 8000) from the restart churn. **Root cause, not just - recovery:** the app code never signals/kills processes (no `killpg`/`os.kill` - in `tasks/` or the workers), and a finishing task only publishes an - `index-updates` reload — so task completion does NOT kill infra. The real - trigger is a *second* supervisor starting (double-launch, or a crash-relaunch - while old children linger) whose new `redis-server` unlinks the live socket; - the `flock` single-instance guard in `launcher.py` prevents that, the reaper - cleans orphans, and the health loop recovers from any genuine death (e.g. - macOS memory-pressure jetsam during the memory-heavy index rebuild). - -### Debugging entry points - -- **Logs:** `~/Library/Logs/AudioMuse-AI/audiomuse.log` (**newest line first**, - bounded ~40k lines; see `macos/reverse_log.py`). Every child's stdout/stderr is - pumped here, tagged `[flask]`, `[rq-worker-default]`, etc., by - `ProcessSupervisor._pump`. -- **State dir:** `~/Library/AudioMuse-AI/` — `pgdata/`, - `redis/redis.sock`, `temp_audio/`, `numba_cache/`, `control.sock`, - `supervisor_pids.json`. (Deliberately *not* under `Application Support`: the - embedded Postgres socket dir is passed to `postgres` via `pg_ctl -o '-k '`, - which re-splits on whitespace, so the writable root must be space-free — - see `macos/paths.py::app_support_dir`.) -- **Run a single role by hand** (after building): `dist/AudioMuse-AI.app/Contents/MacOS/AudioMuse-AI --role=flask` - (or `--role=worker-default`, etc.) — but it expects the supervisor's env vars, so - reproduce them (or export `DATABASE_URL`/`REDIS_URL`/model paths) first. -- **Verified on Linux/WSL (no Mac needed):** the whole non-GUI chain imports; the - full unit suite (1069 passed, 1 skipped) and the real-Postgres integration suite - (18 passed, via `pgserver`) both pass with defaults — i.e. the seam refactor is - regression-clean. Only the bundle build, the menu bar (rumps/AppKit), waitress, - the bundled `redis-server`, icon generation and signing are Mac-only. -- **If cloud/container behavior regressed:** check that `DATABASE_TYPE`/`QUEUE_TYPE` - default to `postgres`/`redis` and that `app_helper` still re-exports the handles — - those two facts are what keep the non-macOS path identical. diff --git a/macos/build.sh b/macos/build.sh deleted file mode 100644 index 7d52a174..00000000 --- a/macos/build.sh +++ /dev/null @@ -1,58 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -APP="dist/AudioMuse-AI.app" -ENTITLEMENTS="macos/entitlements.plist" - -echo "==> Cleaning previous build" -rm -rf build dist - -echo "==> Generating icons from screenshot/audiomuseai.png" -bash macos/make_icns.sh - -echo "==> Running PyInstaller" -pyinstaller macos/AudioMuse-AI.spec --noconfirm - -echo "==> Ad-hoc signing nested binaries" -find "$APP/Contents" \ - \( -name "*.dylib" -o -name "*.so" -o -name "redis-server" -o -name "postgres" \ - -o -name "initdb" -o -name "pg_ctl" -o -name "psql" -o -name "pg_isready" \) -print0 \ - | while IFS= read -r -d '' f; do - codesign --force --timestamp=none --sign - --entitlements "$ENTITLEMENTS" "$f" 2>/dev/null || true - done - -echo "==> Ad-hoc signing the bundle" -codesign --force --deep --timestamp=none --sign - --entitlements "$ENTITLEMENTS" "$APP" - -echo "==> Verifying signature (rejection by spctl is expected for an unsigned app)" -codesign --verify --verbose "$APP" || true -spctl -a -vv "$APP" || true - -ARCH="$(uname -m)" -ZIP="dist/AudioMuse-AI-${ARCH}-macos.zip" -echo "==> Packaging ${ZIP} (AudioMuse-AI.app + readme.md)" -# Stage the app + a plain-text install note, then archive both at the zip root. -# We archive with `ditto` (not `zip`): the bundle is >4 GB and needs ZIP64, which -# the legacy `zip` tool mishandles ("extra bytes" / corrupt archive). `cp -c` -# clones the app via APFS copy-on-write, so staging is instant and costs no extra -# disk while preserving the signature and the bundle's symlinks (ditto copy is the -# fallback on a non-APFS volume). -STAGE="dist/_pkg" -rm -rf "$STAGE" -mkdir -p "$STAGE" -cp -cR "$APP" "$STAGE/AudioMuse-AI.app" 2>/dev/null || ditto "$APP" "$STAGE/AudioMuse-AI.app" -cat > "$STAGE/readme.md" <<'EOF' -This AudioMuse-AI app is not signed to avoid Apple recurrent subscription cost. To have it working you need to: -- Move AudioMuse-AI.app in /Applications -- Open a terminal and run this command to authorize: -xattr -dr com.apple.quarantine /Applications/AudioMuse-AI.app - -After this you can just open it like any other application. -EOF -rm -f "$ZIP" -ditto -c -k "$STAGE" "$ZIP" # no --keepParent: app + readme land at the zip root -rm -rf "$STAGE" - -echo "==> Done: ${ZIP} (expands to AudioMuse-AI.app + readme.md)" -echo " End users must clear quarantine after download:" -echo " xattr -dr com.apple.quarantine /Applications/AudioMuse-AI.app" diff --git a/macos/vendor/pg-contrib/README.md b/macos/vendor/pg-contrib/README.md deleted file mode 100644 index 3986acd5..00000000 --- a/macos/vendor/pg-contrib/README.md +++ /dev/null @@ -1,61 +0,0 @@ -# Vendored PostgreSQL contrib extensions (`unaccent`, `pg_trgm`) - -`pgserver` ships a **minimal** PostgreSQL 16.2 — only `plpgsql` and `pgvector`. -The AudioMuse-AI schema (`app_helper.py::init_db`) requires two contrib -extensions that pgserver does **not** include: - -```sql -CREATE EXTENSION IF NOT EXISTS unaccent; -CREATE EXTENSION IF NOT EXISTS pg_trgm; -``` - -So we vendor those two modules here and the PyInstaller spec -(`macos/AudioMuse-AI.spec`) grafts them into the bundled `pgserver/pginstall` -tree (`.dylib` → `lib/postgresql/`, `*.control`/`*--*.sql` → -`share/postgresql/extension/`, `unaccent.rules` → -`share/postgresql/tsearch_data/`). - -## Why these must be built from 16.2 source (not copied from Homebrew) - -PostgreSQL loadable modules are **not** ABI-stable across minor releases. A -module copied from Homebrew's `postgresql@16` (currently 16.14) fails to load in -pgserver's 16.2 server with e.g. `Symbol not found: _pg_mblen_cstr` — a symbol -added after 16.2. The modules must be compiled against the **exact** server -version pgserver ships. - -pgserver conveniently bundles `pg_config`, the server headers and `pgxs`, so the -modules can be compiled directly against its own install. - -## How to regenerate (per architecture) - -Run on the target arch (arm64 build host for `arm64/`, Intel for `x86_64/`): - -```bash -# 1. pgserver's pg_config (in the build venv) -PGC="$(python -c 'import pgserver,os;print(os.path.join(os.path.dirname(pgserver.__file__),"pginstall","bin","pg_config"))')" -PGVER="$("$PGC" --version | awk '{print $2}')" # e.g. 16.2 - -# 2. matching PostgreSQL source -curl -fsSL "https://ftp.postgresql.org/pub/source/v${PGVER}/postgresql-${PGVER}.tar.bz2" | tar xj -cd "postgresql-${PGVER}" - -# 3. build against pgserver's ABI. PG_SYSROOT overrides the (often stale) SDK -# path baked into pgserver's pg_config on its CI builder. -SDK="$(xcrun --show-sdk-path)" -for m in unaccent pg_trgm; do - make -C "contrib/$m" USE_PGXS=1 PG_CONFIG="$PGC" PG_SYSROOT="$SDK" -done - -# 4. drop the artifacts here (ARCH = arm64 | x86_64) -ARCH="$(uname -m)" -DEST="macos/vendor/pg-contrib/$ARCH" -mkdir -p "$DEST/lib" "$DEST/extension" "$DEST/tsearch_data" -cp contrib/unaccent/unaccent.dylib contrib/pg_trgm/pg_trgm.dylib "$DEST/lib/" -cp contrib/unaccent/unaccent.control contrib/unaccent/unaccent--*.sql "$DEST/extension/" -cp contrib/pg_trgm/pg_trgm.control contrib/pg_trgm/pg_trgm--*.sql "$DEST/extension/" -cp contrib/unaccent/unaccent.rules "$DEST/tsearch_data/" -``` - -Each `.dylib` should depend only on `/usr/lib/libSystem.B.dylib` (`otool -L`); -server symbols resolve at load time via `-bundle_loader postgres`. `build.sh` -ad-hoc signs them along with every other nested binary. diff --git a/macos/vendor/redis/README.md b/macos/vendor/redis/README.md deleted file mode 100644 index c29cbb2e..00000000 --- a/macos/vendor/redis/README.md +++ /dev/null @@ -1,39 +0,0 @@ -# Vendored `redis-server` - -The standalone macOS app embeds Redis (RQ broker + pub/sub). The binary is -committed here per architecture and copied into the bundle as-is by -`macos/AudioMuse-AI.spec` — the build never downloads or compiles Redis, so the -bundled binary is byte-identical to what was tested. - -``` -redis/arm64/redis-server # Apple Silicon (M1/M2/M3/M4...) -``` - -Only `arm64` is provided — the macOS build is Apple-Silicon-only. - -## Pinned version - -- **Redis v8.8.0**, arm64, built **from source without TLS** (`make BUILD_TLS=no`). - -## Why built from source (not copied from Homebrew) - -Homebrew's `redis-server` links Homebrew's `openssl@3` -(`/opt/homebrew/opt/openssl@3/lib/lib{ssl,crypto}.3.dylib`), which does not exist -on an end-user Mac without Homebrew, so the bundled app would fail to start -Redis. The embedded instance only ever uses a unix socket (no TLS — see -`taskqueue.build_embedded_redis_argv`), so a no-TLS build is functionally -complete and depends only on `/usr/lib/libSystem.B.dylib` (verify with -`otool -L`). - -## How to regenerate (on an Apple Silicon Mac) - -```bash -curl -fsSL https://download.redis.io/redis-stable.tar.gz | tar xz -cd redis-stable -make -j"$(sysctl -n hw.ncpu)" BUILD_TLS=no -otool -L src/redis-server # must show ONLY /usr/lib/libSystem.B.dylib -cp src/redis-server ../macos/vendor/redis/arm64/redis-server -chmod +x ../macos/vendor/redis/arm64/redis-server -``` - -`build.sh` ad-hoc signs the binary along with every other nested executable. diff --git a/scripts/standalone/assemble_model.py b/scripts/standalone/assemble_model.py new file mode 100644 index 00000000..a2a941b2 --- /dev/null +++ b/scripts/standalone/assemble_model.py @@ -0,0 +1,134 @@ + + +import argparse +import os +import shutil +import subprocess +import sys +import tarfile +import tempfile +from pathlib import Path + +MODEL = Path("model") +DCLAP_REPO = "NeptuneHub/AudioMuse-AI-DCLAP" +_TEN_MB = 10 * 1024 * 1024 + +REQUIRED = [ + "model/musicnn_embedding.onnx", + "model/musicnn_prediction.onnx", + "model/clap_text_model.onnx", + "model/model_epoch_36.onnx", + "model/model_epoch_36.onnx.data", + "model/huggingface/hub/models--roberta-base/snapshots", + "model/silero_vad.onnx", + "model/gte-multilingual-base-int8.onnx", + "model/whisper-small-onnx/encoder_model.onnx", + "model/whisper-small-onnx/decoder_model_merged.onnx", + "model/gte-multilingual-base/tokenizer.json", +] + + +def _env(name): + value = os.environ.get(name) + if not value: + raise SystemExit(f"::error::Required environment variable {name} is not set") + return value + + +def _gh_download(tag, repo, dest, patterns): + cmd = ["gh", "release", "download", tag, "-R", repo, "-D", str(dest), "--clobber"] + for p in patterns: + cmd += ["-p", p] + subprocess.run(cmd, check=True) + + +def _extract(tar_path, dest): + with tarfile.open(tar_path) as tf: + tf.extractall(dest, filter="data") + + +def _trim_hf_cache(): + print("==> Trim HF cache to just the roberta-base tokenizer (~1.4 GB saved)") + hub = MODEL / "huggingface" / "hub" + for name in ("models--bert-base-uncased", "models--facebook--bart-base"): + shutil.rmtree(hub / name, ignore_errors=True) + rb = hub / "models--roberta-base" + if not rb.is_dir(): + return + blobs = rb / "blobs" + if blobs.is_dir(): + for f in blobs.rglob("*"): + if f.is_file() and not f.is_symlink() and f.stat().st_size > _TEN_MB: + f.unlink() + snapshots = rb / "snapshots" + if snapshots.is_dir(): + for f in snapshots.rglob("*"): + if f.name in ("model.safetensors", "pytorch_model.bin"): + f.unlink() + + +def assemble(): + model_release = _env("MODEL_RELEASE") + dclap_release = _env("DCLAP_RELEASE") + repo = _env("GITHUB_REPOSITORY") + MODEL.mkdir(parents=True, exist_ok=True) + + print(f"==> musicnn + CLAP text models (from {model_release})") + _gh_download(model_release, repo, MODEL, + ["musicnn_embedding.onnx", "musicnn_prediction.onnx", "clap_text_model.onnx"]) + + print(f"==> DCLAP audio model (from {dclap_release} in the -DCLAP repo)") + _gh_download(dclap_release, DCLAP_REPO, MODEL, + ["model_epoch_36.onnx", "model_epoch_36.onnx.data"]) + + print("==> HuggingFace cache (roberta/bert/bart) -- HF_HOME points at model/huggingface") + tmp_hf = Path(tempfile.mkdtemp()) + try: + _gh_download(model_release, repo, tmp_hf, ["huggingface_models.tar.gz"]) + (MODEL / "huggingface").mkdir(parents=True, exist_ok=True) + _extract(tmp_hf / "huggingface_models.tar.gz", MODEL / "huggingface") + finally: + shutil.rmtree(tmp_hf, ignore_errors=True) + + _trim_hf_cache() + + print("==> lyrics bundles (whisper / silero / gte)") + tmp = Path(tempfile.mkdtemp()) + try: + bundles = ["lyrics_model_whisper", "lyrics_model_silero_vad", "lyrics_model_gte_vnni"] + _gh_download(model_release, repo, tmp, [f"{b}.tar.gz" for b in bundles]) + for b in bundles: + _extract(tmp / f"{b}.tar.gz", MODEL) + finally: + shutil.rmtree(tmp, ignore_errors=True) + + +def verify(): + missing = [] + for f in REQUIRED: + p = Path(f) + if not p.exists() or (p.is_file() and p.stat().st_size == 0): + missing.append(f) + roberta = MODEL / "huggingface" / "hub" / "models--roberta-base" + if not any(roberta.rglob("tokenizer.json")): + missing.append("roberta-base tokenizer.json (after HF-cache prune)") + if missing: + for f in missing: + print(f"::error::Missing or empty: {f}") + raise SystemExit("::error::Refusing to build an incomplete bundle.") + total = sum(p.stat().st_size for p in MODEL.rglob("*") if p.is_file() and not p.is_symlink()) + print(f"==> Model assembly verified. model/ is {total / 1e9:.1f} GB") + + +def main(): + parser = argparse.ArgumentParser(description="Assemble or verify ./model for the standalone build.") + parser.add_argument("--verify", action="store_true", help="check the assembled model/ is complete") + args = parser.parse_args() + if args.verify: + verify() + else: + assemble() + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/standalone/build.py b/scripts/standalone/build.py new file mode 100644 index 00000000..40a60a4b --- /dev/null +++ b/scripts/standalone/build.py @@ -0,0 +1,104 @@ +import argparse +import os +import platform as _platform +import re +import shutil +import subprocess +import sys +from dataclasses import dataclass +from pathlib import Path + +import config +from platforms import linux as _linux +from platforms import macos as _macos +from platforms import windows as _windows + +ROOT = Path(__file__).resolve().parents[2] +DISPATCH = {"windows": _windows, "macos": _macos, "linux": _linux} + + +@dataclass +class Ctx: + target: str + arch: str + version: str + root: Path + dist_dir: Path + bundle_dir: Path + app_path: Path + use_pgserver: bool + cfg: dict + + +def sanitize_version(raw): + v = raw or "0.0.0" + if v.startswith("v"): + v = v[1:] + v = re.sub(r"[^A-Za-z0-9.+~-]", "-", v) + v = re.sub(r"-{2,}", "-", v).strip("-") + return v or "0.0.0" + + +def _expected_output(ctx): + if ctx.cfg["bundle"]: + return ctx.app_path + name = "AudioMuse-AI.exe" if ctx.target == "windows" else "AudioMuse-AI" + return ctx.bundle_dir / name + + +def main(): + parser = argparse.ArgumentParser(description="Build the AudioMuse-AI standalone bundle.") + parser.add_argument("--platform", required=True, choices=sorted(config.PLATFORMS)) + parser.add_argument("--arch", default=None) + args = parser.parse_args() + + os.chdir(ROOT) + target = args.platform + cfg = config.PLATFORMS[target] + arch = args.arch or config.normalize_arch(_platform.machine(), target) + + version = sanitize_version(config.read_app_version(ROOT)) + + use_pgserver = config.use_pgserver(cfg["use_pgserver"], arch) + + dist_dir = ROOT / "dist" + ctx = Ctx( + target=target, + arch=arch, + version=version, + root=ROOT, + dist_dir=dist_dir, + bundle_dir=dist_dir / "AudioMuse-AI", + app_path=dist_dir / "AudioMuse-AI.app", + use_pgserver=use_pgserver, + cfg=cfg, + ) + module = DISPATCH[target] + + print(f"==> Package version: {version}") + print(f"==> Platform: {target} Architecture: {arch} pgserver: {use_pgserver}") + + print("==> Cleaning previous build") + shutil.rmtree(ROOT / "build", ignore_errors=True) + shutil.rmtree(dist_dir, ignore_errors=True) + + module.prepare(ctx) + + print("==> Running PyInstaller") + env = {**os.environ, "AUDIOMUSE_BUILD_TARGET": target} + subprocess.run([sys.executable, "-m", "PyInstaller", "AudioMuse-AI.spec", "--noconfirm"], + check=True, env=env) + + out = _expected_output(ctx) + if not out.exists(): + raise SystemExit(f"::error::PyInstaller did not produce {out}") + + artifacts = module.package(ctx) + + print("==> Done") + for art in artifacts or []: + print(f" {art}") + + +if __name__ == "__main__": + main() diff --git a/scripts/standalone/config.py b/scripts/standalone/config.py new file mode 100644 index 00000000..26b52db7 --- /dev/null +++ b/scripts/standalone/config.py @@ -0,0 +1,139 @@ +import ast +import os +import sys + + +def read_app_version(root): + """Return APP_VERSION from the app's config.py (leading 'v' stripped). + + Parsed statically with ast -- config.py is never imported or executed, so + there are no import side effects and no dependency on __file__ or the + environment. build.py (deb/rpm version + banner) and the shared spec (macOS + CFBundleShortVersionString) both call this, so every platform stamps the same + manually-maintained version from config.py and no CI/tag value is used. + """ + path = os.path.join(str(root), "config.py") + with open(path, encoding="utf-8") as fh: + tree = ast.parse(fh.read(), filename=path) + for node in tree.body: + if isinstance(node, ast.Assign) and isinstance(node.value, ast.Constant): + for target in node.targets: + if isinstance(target, ast.Name) and target.id == "APP_VERSION": + value = str(node.value.value) + return value[1:] if value.startswith("v") else value + return "0.0.0" + + +PLATFORMS = { + "windows": { + "launcher": "windows/launcher.py", + "vendor_dir": "windows/vendor", + "redis_bin": "redis-server.exe", + "pg_contrib_glob": "*.dll", + "initdb_bin": "initdb.exe", + "use_pgserver": "always", + "console": True, + "exe_icon": "windows/assets/AudioMuse-AI.ico", + "extra_datas": [("windows/assets/AudioMuse-AI.ico", "assets")], + "extra_hiddenimports": ["macos.reverse_log", "pgserver.postgres_server"], + "collect_submodules": ["windows", "pystray", "PIL"], + "excludes_base": ["rumps", "AppKit", "Foundation", "objc"], + "bundle": None, + }, + "macos": { + "launcher": "macos/launcher.py", + "vendor_dir": "macos/vendor", + "redis_bin": "redis-server", + "pg_contrib_glob": "*.dylib", + "initdb_bin": "initdb", + "use_pgserver": "always", + "console": False, + "exe_icon": None, + "extra_datas": [("macos/assets", "assets")], + "extra_hiddenimports": ["numeric_bootstrap", "rumps"], + "collect_submodules": ["macos"], + "excludes_base": [], + "bundle": { + "name": "AudioMuse-AI.app", + "icon": "macos/assets/AudioMuse-AI.icns", + "bundle_identifier": "ai.audiomuse.standalone", + "info_plist": { + "LSUIElement": True, + "NSHighResolutionCapable": True, + "CFBundleName": "AudioMuse-AI", + "CFBundleDisplayName": "AudioMuse-AI", + }, + }, + }, + "linux": { + "launcher": "linux/launcher.py", + "vendor_dir": "linux/vendor", + "redis_bin": "redis-server", + "pg_contrib_glob": "*.so", + "initdb_bin": "initdb", + "use_pgserver": "arch", + "console": True, + "exe_icon": None, + "extra_datas": [], + "extra_hiddenimports": ["macos.control_ipc", "macos.reverse_log"], + "collect_submodules": ["linux"], + "excludes_base": ["rumps", "AppKit", "Foundation", "objc"], + "bundle": None, + }, +} + +_SYS_PLATFORM_TO_TARGET = { + "win32": "windows", + "darwin": "macos", + "linux": "linux", +} + + +def resolve_target(env_value): + """Return the build target, preferring ``AUDIOMUSE_BUILD_TARGET`` over the host. + + ``build.py`` exports ``AUDIOMUSE_BUILD_TARGET`` before invoking PyInstaller, so + the spec reads it from here. A bare ``pyinstaller AudioMuse-AI.spec`` (no + orchestrator) still works for a developer by falling back to the host's + ``sys.platform`` -- PyInstaller cannot cross-compile, so the host OS is always + the correct target in that case. + """ + if env_value: + target = env_value.strip().lower() + if target in PLATFORMS: + return target + raise ValueError( + f"Unknown AUDIOMUSE_BUILD_TARGET {env_value!r}; expected one of {sorted(PLATFORMS)}" + ) + target = _SYS_PLATFORM_TO_TARGET.get(sys.platform) + if target is None: + raise ValueError(f"Unsupported host platform {sys.platform!r} for a native build") + return target + + +def normalize_arch(machine, target): + """Normalize ``platform.machine()`` to the vendor-dir arch name for ``target``. + + Windows reports ``AMD64``/``ARM64`` (and historically ``x86_64``); the vendored + inputs live under ``amd64``/``arm64``, so lowercase and fold ``x86_64`` to + ``amd64``. macOS (``arm64``) and Linux (``x86_64``/``aarch64``) already match + their vendor dirs verbatim. + """ + if target == "windows": + arch = machine.lower() + return "amd64" if arch == "x86_64" else arch + return machine + + +def use_pgserver(policy, arch): + """Resolve the embedded-PostgreSQL sourcing policy to a boolean. + + ``always`` -> the pgserver wheel (Windows/macOS, with a runtime fallback the + spec applies on its own). ``arch`` -> the wheel only where it exists (Linux + x86_64/amd64); aarch64 has no wheel and bundles a from-source tree instead. + """ + if policy == "always": + return True + if policy == "arch": + return arch in ("x86_64", "amd64") + return False diff --git a/scripts/standalone/platforms/__init__.py b/scripts/standalone/platforms/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/scripts/standalone/platforms/_pgserver.py b/scripts/standalone/platforms/_pgserver.py new file mode 100644 index 00000000..f3a4d6e2 --- /dev/null +++ b/scripts/standalone/platforms/_pgserver.py @@ -0,0 +1,84 @@ +import os +import shutil +import subprocess +import tempfile + +_NO_WINDOW = getattr(subprocess, "CREATE_NO_WINDOW", 0) + + +def _merge_tree(src, dst): + for root, _dirs, files in os.walk(src): + rel = os.path.relpath(root, src) + target_root = dst if rel == "." else os.path.join(dst, rel) + os.makedirs(target_root, exist_ok=True) + for name in files: + s = os.path.join(root, name) + d = os.path.join(target_root, name) + if os.path.islink(d) or os.path.exists(d): + os.remove(d) + shutil.copy2(s, d, follow_symlinks=False) + + +def _restore_and_smoke_test(ctx, pgserver): + pg_pkg = os.path.dirname(os.path.abspath(pgserver.__file__)) + pg_site = os.path.dirname(pg_pkg) + src_pginstall = os.path.join(pg_pkg, "pginstall") + dst_pginstall = os.path.join(str(ctx.bundle_dir), "_internal", "pgserver", "pginstall") + + if not os.path.isdir(os.path.join(src_pginstall, "bin")) or not os.path.isdir(dst_pginstall): + raise SystemExit( + f"::error::Cannot locate pgserver pginstall to restore " + f"(src={src_pginstall} dst={dst_pginstall})" + ) + + _merge_tree(src_pginstall, dst_pginstall) + src_libs = os.path.join(pg_site, "pgserver.libs") + dst_libs = os.path.join(str(ctx.bundle_dir), "_internal", "pgserver.libs") + if os.path.isdir(src_libs) and os.path.isdir(dst_libs): + _merge_tree(src_libs, dst_libs) + print(f"==> Restored complete unstripped pgserver tree into {dst_pginstall}") + + initdb = os.path.join(dst_pginstall, "bin", ctx.cfg["initdb_bin"]) + win_flags = {"creationflags": _NO_WINDOW} if _NO_WINDOW else {} + tmp = tempfile.mkdtemp() + try: + proc = subprocess.run( + [initdb, "-D", os.path.join(tmp, "d"), "--auth=trust", "--encoding=utf8", "-U", "postgres"], + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + stdin=subprocess.DEVNULL, + text=True, + **win_flags, + ) + if proc.returncode != 0: + print("::error::Bundled initdb failed after restore:") + print(proc.stdout) + raise SystemExit(1) + version = subprocess.run( + [initdb, "--version"], + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + text=True, + **win_flags, + ).stdout.strip() + print(f"==> Verified bundled initdb creates a cluster ({version})") + finally: + shutil.rmtree(tmp, ignore_errors=True) + + +def verify_pgserver_bundle(ctx, strict=True): + try: + import pgserver + except Exception: + print("==> pgserver not importable; skipping wheel restore (from-source tree assumed)") + return + + try: + _restore_and_smoke_test(ctx, pgserver) + except (SystemExit, Exception) as exc: + if strict: + raise + print( + f"::warning::pgserver bundle check did not pass; continuing " + f"(Windows best-effort, the bundle still runs initdb at first launch): {exc!r}" + ) diff --git a/scripts/standalone/platforms/linux.py b/scripts/standalone/platforms/linux.py new file mode 100644 index 00000000..b078c277 --- /dev/null +++ b/scripts/standalone/platforms/linux.py @@ -0,0 +1,124 @@ +import shutil +import subprocess + +from ._pgserver import verify_pgserver_bundle + +_NFPM_ARCH = {"x86_64": "amd64", "aarch64": "arm64"} +_ICON_SIZES = (512, 256, 128, 64, 48, 32) + + +def _present(path): + return path.exists() and path.stat().st_size > 0 + + +def prepare(ctx): + arch = ctx.arch + vendor = ctx.root / "linux" / "vendor" + if ctx.use_pgserver: + required = [ + vendor / "redis" / arch / "redis-server", + vendor / "pg-contrib" / arch / "lib" / "unaccent.so", + vendor / "pg-contrib" / arch / "lib" / "pg_trgm.so", + vendor / "pg-contrib" / arch / "extension" / "unaccent.control", + vendor / "pg-contrib" / arch / "extension" / "pg_trgm.control", + vendor / "pg-contrib" / arch / "tsearch_data" / "unaccent.rules", + ] + else: + required = [ + vendor / "redis" / arch / "redis-server", + vendor / "postgres" / arch / "bin" / "postgres", + vendor / "postgres" / arch / "bin" / "initdb", + vendor / "postgres" / arch / "bin" / "pg_ctl", + ] + missing = [str(p) for p in required if not _present(p)] + if not ctx.use_pgserver: + pgtree = vendor / "postgres" / arch + for name in ("unaccent.so", "pg_trgm.so", "unaccent.control", "pg_trgm.control"): + if not any(pgtree.rglob(name)): + missing.append(f"contrib artifact in {pgtree}: {name}") + if missing: + for m in missing: + print(f"::error::Missing vendored file: {m}") + raise SystemExit("Vendored inputs missing (see linux/vendor/*/README.md).") + (vendor / "redis" / arch / "redis-server").chmod(0o755) + + +def _restore_aarch64_exec_bits(ctx): + pgbin = next( + (p for p in ctx.bundle_dir.rglob("bin") if p.is_dir() and p.parent.name == "pgsql"), + None, + ) + if pgbin is None: + raise SystemExit( + "::error::Expected bundled PostgreSQL (pgsql/bin) in the bundle (aarch64 build)" + ) + for f in pgbin.rglob("*"): + if f.is_file(): + f.chmod(f.stat().st_mode | 0o111) + print(f"==> Restored +x on bundled PostgreSQL binaries ({pgbin})") + + +def _stage(ctx): + stage = ctx.dist_dir / "_pkg" + shutil.rmtree(stage, ignore_errors=True) + (stage / "opt").mkdir(parents=True) + (stage / "usr" / "share" / "applications").mkdir(parents=True) + (stage / "usr" / "lib" / "systemd" / "user").mkdir(parents=True) + subprocess.run(["cp", "-a", str(ctx.bundle_dir), str(stage / "opt" / "AudioMuse-AI")], check=True) + + pkg = ctx.root / "linux" / "packaging" + shutil.copy2(pkg / "AudioMuse-AI.desktop", stage / "usr" / "share" / "applications" / "AudioMuse-AI.desktop") + shutil.copy2(pkg / "AudioMuse-AI-stop.desktop", stage / "usr" / "share" / "applications" / "AudioMuse-AI-stop.desktop") + shutil.copy2(pkg / "audiomuse-ai.service", stage / "usr" / "lib" / "systemd" / "user" / "audiomuse-ai.service") + + for size in _ICON_SIZES: + src = pkg / "icons" / f"audiomuse-ai_{size}.png" + if not _present(src): + raise SystemExit(f"::error::Missing square icon source: {src}") + dst = stage / "usr" / "share" / "icons" / "hicolor" / f"{size}x{size}" / "apps" + dst.mkdir(parents=True, exist_ok=True) + shutil.copy2(src, dst / "audiomuse-ai.png") + return stage + + +def package(ctx): + if ctx.use_pgserver: + verify_pgserver_bundle(ctx) + else: + _restore_aarch64_exec_bits(ctx) + + _stage(ctx) + nfpm_arch = _NFPM_ARCH[ctx.arch] + + print("==> Generating nfpm config") + template = (ctx.root / "linux" / "packaging" / "nfpm.yaml.in").read_text() + content = ( + template.replace("@VERSION@", ctx.version) + .replace("@ARCH@", nfpm_arch) + .replace("@STAGE@", "dist/_pkg") + ) + (ctx.dist_dir / "nfpm.yaml").write_text(content) + + print(f"==> Building .deb and .rpm with nfpm (arch={nfpm_arch})") + pkg = ctx.dist_dir / "pkg" + pkg.mkdir(parents=True, exist_ok=True) + for packager in ("deb", "rpm"): + subprocess.run( + ["nfpm", "package", "--config", str(ctx.dist_dir / "nfpm.yaml"), + "--packager", packager, "--target", str(pkg) + "/"], + check=True, + cwd=str(ctx.root), + ) + + debs = sorted(pkg.glob("*.deb")) + rpms = sorted(pkg.glob("*.rpm")) + if not debs or not rpms: + raise SystemExit("::error::nfpm did not produce the expected .deb/.rpm packages.") + deb = debs[0] + rpm = rpms[0] + out_deb = ctx.dist_dir / f"AudioMuse-AI-{ctx.arch}-linux.deb" + out_rpm = ctx.dist_dir / f"AudioMuse-AI-{ctx.arch}-linux.rpm" + shutil.copy2(deb, out_deb) + shutil.copy2(rpm, out_rpm) + print(f"==> Done:\n {out_deb}\n {out_rpm}") + return [out_deb, out_rpm] diff --git a/scripts/standalone/platforms/macos.py b/scripts/standalone/platforms/macos.py new file mode 100644 index 00000000..ed402751 --- /dev/null +++ b/scripts/standalone/platforms/macos.py @@ -0,0 +1,65 @@ +import os +import subprocess + +_SIGN_SUFFIXES = (".dylib", ".so") +_SIGN_NAMES = {"redis-server", "postgres", "initdb", "pg_ctl", "psql", "pg_isready"} + +_README = """This AudioMuse-AI app is not signed to avoid Apple recurrent subscription cost. To have it working you need to: +- Move AudioMuse-AI.app in /Applications +- Open a terminal and run this command to authorize: +xattr -dr com.apple.quarantine /Applications/AudioMuse-AI.app + +After this you can just open it like any other application. +""" + + +def prepare(ctx): + print("==> Generating icons from screenshot/audiomuseai.png") + subprocess.run(["bash", "macos/make_icns.sh"], check=True, cwd=str(ctx.root)) + + +def _sign_nested(app, entitlements): + print("==> Ad-hoc signing nested binaries") + contents = os.path.join(str(app), "Contents") + for root, _dirs, files in os.walk(contents): + for name in files: + if name.endswith(_SIGN_SUFFIXES) or name in _SIGN_NAMES: + subprocess.run( + ["codesign", "--force", "--timestamp=none", "--sign", "-", + "--entitlements", entitlements, os.path.join(root, name)], + stderr=subprocess.DEVNULL, + ) + + +def package(ctx): + app = ctx.app_path + entitlements = str(ctx.root / "macos" / "entitlements.plist") + + _sign_nested(app, entitlements) + + print("==> Ad-hoc signing the bundle") + subprocess.run( + ["codesign", "--force", "--deep", "--timestamp=none", "--sign", "-", + "--entitlements", entitlements, str(app)], + check=True, + ) + + print("==> Verifying signature (rejection by spctl is expected for an unsigned app)") + subprocess.run(["codesign", "--verify", "--verbose", str(app)]) + subprocess.run(["spctl", "-a", "-vv", str(app)]) + + out = ctx.dist_dir / f"AudioMuse-AI-{ctx.arch}-macos.zip" + print(f"==> Packaging {out.name} (AudioMuse-AI.app + readme.md)") + stage = ctx.dist_dir / "_pkg" + subprocess.run(["rm", "-rf", str(stage)], check=True) + stage.mkdir(parents=True) + staged_app = stage / "AudioMuse-AI.app" + if subprocess.run(["cp", "-cR", str(app), str(staged_app)], stderr=subprocess.DEVNULL).returncode != 0: + subprocess.run(["ditto", str(app), str(staged_app)], check=True) + (stage / "readme.md").write_text(_README) + if out.exists(): + out.unlink() + subprocess.run(["ditto", "-c", "-k", str(stage), str(out)], check=True) + subprocess.run(["rm", "-rf", str(stage)], check=True) + print(f"==> Done: {out} (expands to AudioMuse-AI.app + readme.md)") + return [out] diff --git a/scripts/standalone/platforms/windows.py b/scripts/standalone/platforms/windows.py new file mode 100644 index 00000000..c20c4e73 --- /dev/null +++ b/scripts/standalone/platforms/windows.py @@ -0,0 +1,41 @@ +import zipfile + +from ._pgserver import verify_pgserver_bundle + + +def prepare(ctx): + arch = ctx.arch + pg_contrib = ctx.root / "windows" / "vendor" / "pg-contrib" / arch + required = [ + ctx.root / "windows" / "vendor" / "redis" / arch / "redis-server.exe", + pg_contrib / "lib" / "unaccent.dll", + pg_contrib / "lib" / "pg_trgm.dll", + pg_contrib / "extension" / "unaccent.control", + pg_contrib / "extension" / "pg_trgm.control", + pg_contrib / "tsearch_data" / "unaccent.rules", + ] + missing = [str(p) for p in required if not p.exists()] + if missing: + for m in missing: + print(f"[ERROR] Missing vendored file: {m}") + raise SystemExit("Vendored inputs missing (see windows/vendor/README.md).") + + +def _zip_dir(src_dir, out_path): + base = src_dir.parent + with zipfile.ZipFile(out_path, "w", zipfile.ZIP_DEFLATED, allowZip64=True) as zf: + for path in sorted(src_dir.rglob("*")): + if path.is_file(): + zf.write(path, path.relative_to(base).as_posix()) + + +def package(ctx): + if ctx.use_pgserver: + verify_pgserver_bundle(ctx, strict=False) + out = ctx.dist_dir / f"AudioMuse-AI-{ctx.arch}-windows.zip" + if out.exists(): + out.unlink() + print(f"==> Packaging {out.name}") + _zip_dir(ctx.bundle_dir, out) + print(f"==> ZIP: {out}") + return [out] diff --git a/windows/AudioMuse-AI.spec b/windows/AudioMuse-AI.spec deleted file mode 100644 index 02a8ed2b..00000000 --- a/windows/AudioMuse-AI.spec +++ /dev/null @@ -1,133 +0,0 @@ -# -*- mode: python ; coding: utf-8 -*- -"""PyInstaller spec for the standalone Windows build. - -Run from the repo root: ``pyinstaller windows/AudioMuse-AI.spec --noconfirm``. -Produces a one-dir bundle at ``dist/AudioMuse-AI/`` (the executable plus an -``_internal`` tree with Python, the libraries, the models and the embedded -PostgreSQL/Redis). ``windows/build.bat`` then packages that tree into a zip -archive. - -Builds for the architecture of the running Python (CI builds on ``windows-latest``, -which is x86_64/amd64). -""" - -import glob -import os -import platform - -from PyInstaller.utils.hooks import collect_data_files, collect_dynamic_libs, collect_submodules - -arch = platform.machine().lower() # 'amd64' on Windows, normalize to what the build expects -if arch == 'x86_64': - arch = 'amd64' -# pgserver ships a Windows wheel; if not available, the fallback embedded_pg is used. -USE_PGSERVER = True # will be checked at runtime by db_backend - -# ``SPECPATH`` is the directory containing this spec (``/windows``); the repo -# root is its parent. -ROOT = os.path.dirname(SPECPATH) - -datas = [ - (os.path.join(ROOT, 'templates'), 'templates'), - (os.path.join(ROOT, 'static'), 'static'), - (os.path.join(ROOT, 'model'), 'model'), - # Root-level data file the app loads relative to config.py's __file__ - (os.path.join(ROOT, 'mood_centroids_real_080_clap.json'), '.'), - # Tray-app icon, loaded at runtime via paths.tray_icon() - (os.path.join(ROOT, 'windows/assets/AudioMuse-AI.ico'), 'assets'), -] -if USE_PGSERVER: - try: - datas += collect_data_files('pgserver') - except Exception: - USE_PGSERVER = False -if not USE_PGSERVER: - # Bundle the entire PostgreSQL install as opaque data under ``pgsql/``. - datas += [(os.path.join(ROOT, 'windows/vendor/postgres', arch), 'pgsql')] -datas += collect_data_files('librosa') -datas += collect_data_files('resampy') -datas += collect_data_files('transformers', include_py_files=False) -datas += collect_data_files('flasgger') -datas += collect_data_files('wn') -datas += collect_data_files('langdetect') - -binaries = [ - (os.path.join(ROOT, f'windows/vendor/redis/{arch}/redis-server.exe'), '.'), -] -binaries += collect_dynamic_libs('av') -binaries += collect_dynamic_libs('voyager') -binaries += collect_dynamic_libs('psycopg2') - -if USE_PGSERVER: - _pg_contrib = os.path.join(ROOT, 'windows/vendor/pg-contrib', arch) - _pg_dst = 'pgserver/pginstall' - for _f in glob.glob(os.path.join(_pg_contrib, 'extension', '*')): - datas.append((_f, f'{_pg_dst}/share/postgresql/extension')) - for _f in glob.glob(os.path.join(_pg_contrib, 'tsearch_data', '*')): - datas.append((_f, f'{_pg_dst}/share/postgresql/tsearch_data')) - for _f in glob.glob(os.path.join(_pg_contrib, 'lib', '*.dll')): - binaries.append((_f, f'{_pg_dst}/lib/postgresql')) - -hiddenimports = [ - 'app', - 'rq_worker', - 'rq_worker_high_priority', - 'rq_janitor', - 'restart_listener', - 'waitress', - 'flasgger', - # Platform-agnostic helpers reused by windows.supervisor from the macos - # package (no GUI deps). - 'macos.reverse_log', -] -hiddenimports += collect_submodules('windows') -hiddenimports += collect_submodules('tasks') -hiddenimports += collect_submodules('lyrics') -hiddenimports += collect_submodules('sklearn') -hiddenimports += collect_submodules('pystray') -hiddenimports += collect_submodules('PIL') - -# rumps/AppKit are macOS-only. -excludes = ['rumps', 'AppKit', 'Foundation', 'objc'] -if not USE_PGSERVER: - excludes.append('pgserver') - -a = Analysis( - [os.path.join(ROOT, 'windows/launcher.py')], - pathex=[ROOT], - binaries=binaries, - datas=datas, - hiddenimports=hiddenimports, - hookspath=[os.path.join(ROOT, 'macos/hooks')], # hook-tasks.py is platform-agnostic - hooksconfig={}, - runtime_hooks=[], - excludes=excludes, - noarchive=False, -) - -pyz = PYZ(a.pure) - -# strip=False matches Linux/macOS: stripping can corrupt manylinux/patchelf-modified -# binaries, and on Windows stripping .pyd files can break them. -exe = EXE( - pyz, - a.scripts, - [], - exclude_binaries=True, - name='AudioMuse-AI', - debug=False, - bootloader_ignore_signals=False, - strip=False, - upx=False, - console=True, # Windows: console app so users see output and can Ctrl+C - icon=os.path.join(ROOT, 'windows/assets/AudioMuse-AI.ico'), -) - -coll = COLLECT( - exe, - a.binaries, - a.datas, - strip=False, - upx=False, - name='AudioMuse-AI', -) diff --git a/windows/README.md b/windows/README.md deleted file mode 100644 index c3fed446..00000000 --- a/windows/README.md +++ /dev/null @@ -1,121 +0,0 @@ -# AudioMuse-AI — Standalone Windows App - -Windows counterpart of the [`macos/`](../macos) and [`linux/`](../linux) builds. -Packages the entire AudioMuse-AI stack (Python, ONNX models, embedded PostgreSQL -via pgserver, embedded Redis, the Flask web UI, the RQ workers) into a single -self-contained folder, distributed as a zip — no Docker, no external database, -no manual setup. - -> **x86_64 only** initially. ARM64 Windows support may be added later once -> onnxruntime/pgserver/redis-server Windows arm64 builds are validated. - -## Quick start (developer build) - -**Prerequisites:** -* Windows 10/11 (x86_64) -* Python 3.12 -* Visual Studio Build Tools (for compiling pg-contrib; or use pre-built vendor binaries) - -**Steps:** -```powershell -# 1. Create and activate a virtual environment -python -m venv .venv-windows -.venv-windows\Scripts\activate - -# 2. Install Python dependencies -pip install -r requirements\windows.txt - -# 3. Build or download vendor binaries (Redis + PostgreSQL contrib) -windows\vendor\build-redis.bat -# For pg-contrib, either cross-compile on Linux (see vendor README) -# or use pre-built binaries committed to the repo. - -# 4. Assemble ./model (same models as Docker/macOS/Linux) -# In CI this is done by the workflow. For local dev: -# - Download models from GitHub releases (see build-windows.yml for URLs) - -# 5. Build the app bundle -set PKG_VERSION=0.0.0 -windows\build.bat - -# 6. Output -# dist\AudioMuse-AI\ — one-dir PyInstaller bundle -# dist\AudioMuse-AI-amd64-windows.zip — ZIP archive (the shipped artifact) -``` - -## What the build produces - -* `dist/AudioMuse-AI/` — runnable folder. Double-click `AudioMuse-AI.exe` to launch the tray app, or run from a terminal. -* `dist/AudioMuse-AI-amd64-windows.zip` — the shipped artifact: a portable zip of the one-dir bundle. Unzip anywhere and run `AudioMuse-AI.exe`. - -## Runtime layout - -The unzipped bundle (run it from wherever you extract it): -``` -AudioMuse-AI\ -├── AudioMuse-AI.exe # Launcher (tray app / start/stop/status/open) -└── _internal\ # PyInstaller bundle - ├── python3.dll - ├── model\ # ONNX models + HuggingFace cache - ├── templates\ - ├── static\ - ├── redis-server.exe # Embedded Redis - └── pgserver\ # Embedded PostgreSQL (pgserver wheel) -``` - -Writable data lives in `%LOCALAPPDATA%\AudioMuse-AI\`: -``` -C:\Users\\AppData\Local\AudioMuse-AI\ -├── pgdata\ # PostgreSQL cluster -├── redis\ # Redis working directory -├── temp_audio\ # Transcoding scratch -├── numba_cache\ -├── backup\ -├── logs\ -│ └── audiomuse.log # Newest lines first (same as macOS) -├── supervisor.lock # Single-instance mutex -└── supervisor_pids.json -``` - -## Tray app - -Launching with no arguments (double-click, or the Start Menu / desktop shortcut) -opens a notification-area (system tray) icon — the Windows counterpart of the -macOS menu-bar agent. Right-click the icon for the menu: - -* **Status** — Running / Starting… / Stopped -* **Open in Browser** — open the web UI (also the left-click action) -* **Start** / **Stop** — boot or shut down the embedded stack -* **Open Log** — open `audiomuse.log` in the default editor -* **Quit** — stop everything and exit - -The console window is hidden automatically when launched by double-click (it stays -visible when you run a command from an existing terminal). - -## CLI commands - -``` -AudioMuse-AI.exe # Open the tray app (default) -AudioMuse-AI.exe tray # Same as above -AudioMuse-AI.exe start # Run the supervisor in the foreground console (logs to the terminal) -AudioMuse-AI.exe stop # Gracefully shut down a running instance -AudioMuse-AI.exe status # Print running/stopped -AudioMuse-AI.exe open # Open web UI (auto-starts if stopped) -``` - -## Platform notes - -* **No Unix sockets** — Windows uses TCP on 127.0.0.1 for Redis (6379), PostgreSQL (5432), and the control server (8001). The `restart_manager.py` shared code uses `AUDIOMUSE_PLATFORM=macos` (same as Linux) to select the socket-based restart path; `windows/control_server.py` provides the same JSON-line protocol over TCP. -* **No `flock`** — single-instance enforcement uses a Windows named mutex (`CreateMutexW`). -* **Tray app via `pystray`** (not `rumps`/`AppKit`) — the Windows counterpart of the macOS menu-bar agent: a notification-area icon with Start / Stop / Open Log / Open in Browser / Quit. The exe stays a console app (so the CLI subcommands and `CTRL_BREAK_EVENT` shutdown keep working); the console window is just hidden when launched by double-click. -* **`CTRL_BREAK_EVENT`** instead of `SIGTERM` for child process termination. -* **`waitress`** (not `gunicorn`) serves the Flask app — same as macOS and Linux builds. - -## Shared-code impact - -**Zero.** The Windows build follows the same pattern as Linux: - -* Reports `AUDIOMUSE_PLATFORM=macos` so `restart_manager.py` uses the control-server path (no change). -* Uses the macOS hooks (`macos/hooks/hook-tasks.py`), `macos.reverse_log`, which are platform-agnostic. -* `database.py` is called as-is via `windows/db_backend.py`; the fallback `windows/embedded_pg.py` only activates when pgserver isn't available. -* All path overrides go through `windows/env.py` → `config` environment variables. diff --git a/windows/assets/README.md b/windows/assets/README.md deleted file mode 100644 index 1db1cc96..00000000 --- a/windows/assets/README.md +++ /dev/null @@ -1,10 +0,0 @@ -# Windows assets directory - -Place the application icon here (``AudioMuse-AI.ico``) for the PyInstaller build. - -Generate the .ico from the project's main icon: -1. Use ``screenshot/audiomuseai.png`` as the source -2. Convert to .ico with multiple resolutions (16, 32, 48, 256) -3. Place as ``windows/assets/AudioMuse-AI.ico`` - -The macOS build's ``make_icns.sh`` can serve as reference for the conversion. diff --git a/windows/build.bat b/windows/build.bat deleted file mode 100644 index e9b3e818..00000000 --- a/windows/build.bat +++ /dev/null @@ -1,62 +0,0 @@ -@echo off -setlocal enabledelayedexpansion -REM ============================================================================ -REM Build the standalone Windows bundle with PyInstaller, then package it as a -REM zip archive. -REM -REM Run from the repo root, inside the build venv: -REM .venv-windows\Scripts\activate -REM set PKG_VERSION=1.0.0 -REM windows\build.bat -REM -REM Prerequisites (the CI workflow installs these): -REM * Python 3.12 + deps from requirements/windows.txt (incl. pyinstaller, pgserver) -REM * the vendored redis-server.exe + pg-contrib for this arch -REM (windows\vendor\...; built by windows\vendor\build-redis.bat and -REM windows\vendor\pg-contrib\build-pg-contrib.bat) -REM ============================================================================ - -if "%PKG_VERSION%"=="" set PKG_VERSION=0.0.0 -REM Strip leading v (v1.2.3 -> 1.2.3) and sanitize. -set "VER=%PKG_VERSION:v=%" -echo ==^> Package version: %VER% - -REM Detect architecture -set "ARCH=amd64" -if "%PROCESSOR_ARCHITECTURE%"=="ARM64" set "ARCH=arm64" -echo ==^> Architecture: %ARCH% - -echo ==^> Cleaning previous build -if exist build rmdir /s /q build -if exist dist rmdir /s /q dist - -echo ==^> Verifying vendored native build inputs are present -set MISSING=0 -if not exist "windows\vendor\redis\%ARCH%\redis-server.exe" ( - echo [ERROR] Missing vendored file: windows\vendor\redis\%ARCH%\redis-server.exe - set MISSING=1 -) -if not exist "windows\vendor\pg-contrib\%ARCH%\lib\unaccent.dll" ( - echo [ERROR] Missing vendored file: windows\vendor\pg-contrib\%ARCH%\lib\unaccent.dll - set MISSING=1 -) -if "%MISSING%"=="1" ( - echo Vendored inputs missing ^(see windows\vendor\README.md^). - exit /b 1 -) - -echo ==^> Running PyInstaller -pyinstaller windows\AudioMuse-AI.spec --noconfirm - -set "BUNDLE=dist\AudioMuse-AI" -if not exist "%BUNDLE%\AudioMuse-AI.exe" ( - echo [ERROR] PyInstaller did not produce %BUNDLE%\AudioMuse-AI.exe - exit /b 1 -) - -echo ==^> Packaging zip -set "ZIP=dist\AudioMuse-AI-%ARCH%-windows.zip" -powershell -Command "Compress-Archive -Path '%BUNDLE%' -DestinationPath '%ZIP%' -Force" - -echo ==^> Done -echo ZIP: %ZIP% diff --git a/windows/vendor/README.md b/windows/vendor/README.md deleted file mode 100644 index a8d31f7b..00000000 --- a/windows/vendor/README.md +++ /dev/null @@ -1,77 +0,0 @@ -# Windows vendor binaries - -The standalone Windows app embeds two native services: - -* **Redis** — the RQ broker and pub/sub bus -* **PostgreSQL contrib modules** — `unaccent` and `pg_trgm` extensions - -These are NOT regenerated at build time; they are committed (or built by CI) so -the bundle is reproducible. - -## Directory layout - -``` -windows/vendor/ -├── README.md -├── redis/ -│ └── amd64/ -│ └── redis-server.exe # Redis 7.x for Windows (Microsoft Archive) -├── pg-contrib/ -│ └── amd64/ -│ ├── lib/ -│ │ ├── unaccent.dll # compiled against pgserver's PostgreSQL 16.2 -│ │ └── pg_trgm.dll -│ ├── extension/ -│ │ ├── unaccent.control -│ │ └── pg_trgm.control -│ └── tsearch_data/ -│ └── unaccent.rules -└── postgres/ # fallback: full PostgreSQL tree (if pgserver unavailable) - └── amd64/ - ├── bin/ # postgres.exe, initdb.exe, pg_ctl.exe, etc. - ├── lib/ # .dll files - └── share/ # extension .sql and .control files -``` - -## Building the vendor binaries - -### redis-server.exe - -Redis does not officially support Windows. The recommended approach is to use -the **Microsoft Archive Redis for Windows** build: - -1. Download the latest release from https://github.com/microsoftarchive/redis/releases -2. Extract `redis-server.exe` to `windows/vendor/redis/amd64/` - -Alternatively, build from source with MSYS2/MinGW or use the Windows Subsystem for Linux. - -**Version target:** Redis 7.x (matching what the container deployments use). - -### PostgreSQL contrib modules (unaccent, pg_trgm) - -These must be compiled against the EXACT PostgreSQL version that pgserver bundles -(16.2). The .dll files are ABI-specific to this PG minor. - -**Option A — cross-compile on Linux (recommended for CI):** -```bash -# On an x86_64 Linux runner with mingw-w64 installed: -bash windows/vendor/pg-contrib/build-pg-contrib-cross.sh -``` - -**Option B — build on Windows:** -```powershell -# Requires Visual Studio Build Tools + PostgreSQL 16.2 dev headers -.\windows\vendor\pg-contrib\build-pg-contrib.ps1 -``` - -### Full PostgreSQL tree (fallback, only if pgserver has no Windows wheel) - -If pgserver does not provide a Windows wheel, the `windows/embedded_pg.py` -fallback requires a full relocatable PostgreSQL installation. - -**Build steps:** -1. Download PostgreSQL 16.2 source from https://www.postgresql.org/ftp/source/ -2. Compile with MSVC or MinGW, targeting a relocatable layout -3. Place the resulting `bin/`, `lib/`, `share/` trees under `windows/vendor/postgres/amd64/` - -This is the same approach used by the Linux aarch64 build.