diff --git a/.dockerignore b/.dockerignore index 9cc988e1..142f2573 100644 --- a/.dockerignore +++ b/.dockerignore @@ -25,9 +25,7 @@ student_clap/ # (vendored redis/pg binaries, assets, nfpm config, runtime launchers), the shared # PyInstaller spec + build tooling, and build artifacts. None of it is used by the # Linux container, so keep it out of the image. -macos/ -linux/ -windows/ +native-build/ AudioMuse-AI.spec scripts/standalone/ build/ @@ -67,7 +65,5 @@ model/ *.bin *.h5 *.pb -CLAMP3/weights_*.pth -CLAMP3/*.npy .cache/ *.tar.gz diff --git a/.github/workflows/build-linux.yml b/.github/workflows/build-linux.yml index 901a3b0b..0582a59c 100644 --- a/.github/workflows/build-linux.yml +++ b/.github/workflows/build-linux.yml @@ -7,13 +7,13 @@ # Mirrors build-macos.yml. The package bundles EVERYTHING (Python runtime, # onnxruntime, PyAV/ffmpeg, the ONNX models, an embedded PostgreSQL via pgserver # and an embedded Redis), so an installed package needs no Docker and no -# separately-installed database/broker. See linux/README.md for why Postgres and +# separately-installed database/broker. See native-build/linux/README.md for why Postgres and # Redis are bundled rather than declared as system dependencies (pgvector + # unaccent/pg_trgm against an exact PG minor, plus the per-distro package-name / # version skew, make a portable single .deb/.rpm "hard dependency" infeasible). # # The small native build inputs (redis-server, the unaccent/pg_trgm contrib -# modules) are built from source in this workflow (linux/vendor/*) rather than +# modules) are built from source in this workflow (native-build/linux/vendor/*) rather than # committed, then baked into the bundle by the shared AudioMuse-AI.spec. # # The ~5 GB of models are NOT in git. This workflow assembles ./model from the @@ -59,7 +59,7 @@ jobs: # * x86_64 -> pgserver wheel (+ vendored unaccent/pg_trgm grafted in). # * aarch64 -> a from-source PostgreSQL (server + the two contrib modules) # built here and bundled wholesale; managed by - # linux/embedded_pg.py at runtime. + # native-build/linux/embedded_pg.py at runtime. strategy: fail-fast: false matrix: @@ -114,8 +114,8 @@ jobs: - name: Validate the desktop entries run: | set -euo pipefail - desktop-file-validate linux/packaging/AudioMuse-AI.desktop - desktop-file-validate linux/packaging/AudioMuse-AI-stop.desktop + desktop-file-validate native-build/linux/packaging/AudioMuse-AI.desktop + desktop-file-validate native-build/linux/packaging/AudioMuse-AI-stop.desktop echo "Desktop entries are valid." - name: Install Python dependencies @@ -127,7 +127,7 @@ jobs: pip install -r requirements/linux.txt - name: Build vendored redis-server - run: bash linux/vendor/build-redis.sh + run: bash native-build/linux/vendor/build-redis.sh # x86_64: compile unaccent/pg_trgm against the installed pgserver's PG. - name: Build vendored PostgreSQL contrib (x86_64, against pgserver) @@ -135,13 +135,13 @@ jobs: run: | set -euo pipefail source .venv-linux/bin/activate # pgserver must be importable - bash linux/vendor/pg-contrib/build-pg-contrib.sh + bash native-build/linux/vendor/pg-contrib/build-pg-contrib.sh # aarch64: no pgserver wheel -> build a relocatable PostgreSQL (server + # unaccent/pg_trgm) from source and bundle the whole tree. - name: Build vendored PostgreSQL from source (aarch64) if: matrix.arch == 'aarch64' - run: bash linux/vendor/postgres/build-postgres.sh + run: bash native-build/linux/vendor/postgres/build-postgres.sh - name: Assemble ./model (mirrors the Dockerfile/macOS models stage) env: diff --git a/.github/workflows/build-macos.yml b/.github/workflows/build-macos.yml index 4afc6c04..44348857 100644 --- a/.github/workflows/build-macos.yml +++ b/.github/workflows/build-macos.yml @@ -6,16 +6,17 @@ # reliably universal2, so we build natively on an arm64 runner (macos-14). # # Reproducibility: the small, exact-version native build inputs are committed in -# git (macos/vendor/redis/arm64/redis-server and macos/vendor/pg-contrib/arm64/*), +# git (native-build/macos/vendor/redis/arm64/redis-server and +# native-build/macos/vendor/pg-contrib/arm64/*), # NOT regenerated at build time, so the bundle's embedded Redis and the Postgres # unaccent/pg_trgm contrib extensions are byte-identical to the tested local build. # # The app is ad-hoc signed only (no Apple Developer account); end users clear -# quarantine with `xattr -dr ...` after download (see macos/README.md). +# quarantine with `xattr -dr ...` after download (see native-build/macos/README.md). # # The ~5 GB of models are NOT in git. This workflow assembles ./model from the # GitHub releases the Dockerfile uses, INCLUDING the HuggingFace cache -# (huggingface_models.tar.gz) — macos/env.py points HF_HOME at model/huggingface +# (huggingface_models.tar.gz) — native-build/macos/env.py points HF_HOME at model/huggingface # for the CLAP RoBERTa tokenizer, so the bundle needs it. The HF cache is then # trimmed to just that tokenizer (bert/bart and the roberta weights are unused by # the app, ~1.4 GB) so the release zip stays under GitHub's 2 GB asset limit. @@ -86,20 +87,20 @@ jobs: # These are committed in git (not regenerated here) so the bundle is # reproducible. If a fresh checkout is missing them, fail loudly. required=( - macos/vendor/redis/arm64/redis-server - macos/vendor/pg-contrib/arm64/lib/unaccent.dylib - macos/vendor/pg-contrib/arm64/lib/pg_trgm.dylib - macos/vendor/pg-contrib/arm64/extension/unaccent.control - macos/vendor/pg-contrib/arm64/extension/pg_trgm.control - macos/vendor/pg-contrib/arm64/tsearch_data/unaccent.rules + native-build/macos/vendor/redis/arm64/redis-server + native-build/macos/vendor/pg-contrib/arm64/lib/unaccent.dylib + native-build/macos/vendor/pg-contrib/arm64/lib/pg_trgm.dylib + native-build/macos/vendor/pg-contrib/arm64/extension/unaccent.control + native-build/macos/vendor/pg-contrib/arm64/extension/pg_trgm.control + native-build/macos/vendor/pg-contrib/arm64/tsearch_data/unaccent.rules ) missing=0 for f in "${required[@]}"; do if [ ! -s "$f" ]; then echo "::error::Missing committed vendor file: $f"; missing=1; fi done - [ "$missing" -eq 0 ] || { echo "::error::macos/vendor/ must be committed (see macos/vendor/pg-contrib/README.md)."; exit 1; } - chmod +x macos/vendor/redis/arm64/redis-server - file macos/vendor/redis/arm64/redis-server + [ "$missing" -eq 0 ] || { echo "::error::native-build/macos/vendor/ must be committed (see native-build/macos/vendor/pg-contrib/README.md)."; exit 1; } + chmod +x native-build/macos/vendor/redis/arm64/redis-server + file native-build/macos/vendor/redis/arm64/redis-server - name: Assemble ./model (mirrors the Dockerfile models stage) env: diff --git a/.github/workflows/build-windows.yml b/.github/workflows/build-windows.yml index 983afcdb..a8cd58f1 100644 --- a/.github/workflows/build-windows.yml +++ b/.github/workflows/build-windows.yml @@ -81,12 +81,12 @@ jobs: - name: Build vendored redis-server.exe shell: cmd - run: windows\vendor\build-redis.bat + run: native-build\windows\vendor\build-redis.bat - name: Verify vendored PostgreSQL contrib is present shell: powershell run: | - $dest = "windows\vendor\pg-contrib\$env:ARCH" + $dest = "native-build\windows\vendor\pg-contrib\$env:ARCH" $required = @("lib\unaccent.dll", "lib\pg_trgm.dll", "extension\unaccent.control", "extension\pg_trgm.control", "extension\unaccent--1.1.sql", "extension\pg_trgm--1.3.sql", "tsearch_data\unaccent.rules") $bad = @() foreach ($f in $required) { @@ -94,7 +94,7 @@ jobs: if (-not (Test-Path $p) -or (Get-Item $p).Length -eq 0) { $bad += $f } } if ($bad.Count -ne 0) { - Write-Error "Missing or empty vendored pg-contrib files: $($bad -join ', '). Regenerate with windows/vendor/pg-contrib/build-pg-contrib-cross.sh -- never ship empty placeholders (they silently disable unaccent/pg_trgm search)." + Write-Error "Missing or empty vendored pg-contrib files: $($bad -join ', '). Regenerate with native-build/windows/vendor/pg-contrib/build-pg-contrib-cross.sh -- never ship empty placeholders (they silently disable unaccent/pg_trgm search)." exit 1 } Write-Host "Vendored pg-contrib present and non-empty." diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 50120002..c3c9dde0 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -143,22 +143,22 @@ jobs: - name: Run integration tests run: | # Run the MusiCNN integration test - pytest test/test_analysis_integration.py -s -v --tb=short + pytest test/integration/test_analysis_integration.py -s -v --tb=short # Run the CLAP integration test - pytest test/test_clap_analysis_integration.py -s -v --tb=short + pytest test/integration/test_clap_analysis_integration.py -s -v --tb=short - name: Run provider migration integration test env: AUDIOMUSE_TEST_DATABASE_URL: postgresql://postgres:postgres@localhost:5432/audiomuse_migration_test run: | - pytest test/test_provider_migration_integration.py -s -v --tb=short + pytest test/integration/test_provider_migration_integration.py -s -v --tb=short - name: Run app endpoint and auth integration tests env: AUDIOMUSE_TEST_DATABASE_URL: postgresql://postgres:postgres@localhost:5432/audiomuse_migration_test run: | - pytest test/test_app_endpoints_integration.py test/test_auth_users_integration.py -m integration -s -v --tb=short + pytest test/integration/test_app_endpoints_integration.py test/integration/test_auth_users_integration.py -m integration -s -v --tb=short - name: Run lyrics integration test run: | @@ -166,7 +166,7 @@ jobs: # test enters RECORD mode automatically, writes the file and exits # successfully. The next step then commits it back to main so # subsequent runs pin against it (cosine similarity >= 0.99). - pytest test/test_lyrics_analysis_integration.py -s -v --tb=short + pytest test/integration/test_lyrics_analysis_integration.py -s -v --tb=short - name: Commit recorded lyrics expected vectors # Push back the auto-recorded vectors on: diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 34dda73d..8313cfa1 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -25,4 +25,4 @@ jobs: - name: Run unit tests run: | - pytest tests/unit/ -v --tb=short + pytest test/unit/ -v --tb=short diff --git a/.gitignore b/.gitignore index 2baed74a..2708a402 100644 --- a/.gitignore +++ b/.gitignore @@ -12,11 +12,11 @@ venv/ .venv-windows/ env/ -# Linux native build inputs built fresh in CI (not committed; see linux/vendor/README.md) -/linux/vendor/redis/ -/linux/vendor/pg-contrib/*/ -/linux/vendor/postgres/*/ -/windows/vendor/redis/ +# Linux native build inputs built fresh in CI (not committed; see native-build/linux/vendor/README.md) +/native-build/linux/vendor/redis/ +/native-build/linux/vendor/pg-contrib/*/ +/native-build/linux/vendor/postgres/*/ +/native-build/windows/vendor/redis/ # Local environment variables # IMPORTANT: Never commit your .env file with secrets! @@ -46,16 +46,6 @@ env/ htmlcov/ nul -# Large model files in query folder -/query/*.pt -/query/*.onnx - - -# CLAMP3 weight files (large model files) -/CLAMP3/weights_*.pth -/CLAMP3/*.pth - - # Downloaded ONNX models (from run_worker_macos.sh or similar) /model/ @@ -82,4 +72,4 @@ analysis.md .claude/settings.local.json .claude/settings.local.json lyrics_model_gte_vnni.tar.gz -windows/vendor/redis/amd64/redis-server.exe +native-build/windows/vendor/redis/amd64/redis-server.exe diff --git a/.sonarcloud.properties b/.sonarcloud.properties new file mode 100644 index 00000000..47ed46e1 --- /dev/null +++ b/.sonarcloud.properties @@ -0,0 +1,10 @@ +# Vendored third-party code (PostgreSQL contrib extensions bundled per platform) +# is not ours to analyze. +sonar.exclusions=native-build/*/vendor/** + +# Copy-paste detection exclusions: +# - native-build/ holds three intentionally parallel per-platform trees +# (linux/macos/windows supervisor, launcher, env, paths); their similarity +# is by design, not accidental duplication. +# - test/ duplication is repeated setup/fixture boilerplate across tests. +sonar.cpd.exclusions=native-build/**,test/** diff --git a/AudioMuse-AI.spec b/AudioMuse-AI.spec index 1a71c328..860c9ba9 100644 --- a/AudioMuse-AI.spec +++ b/AudioMuse-AI.spec @@ -3,10 +3,13 @@ import glob import importlib.util import os import platform +import sys from PyInstaller.utils.hooks import collect_data_files, collect_dynamic_libs, collect_submodules ROOT = SPECPATH +NATIVE = os.path.join(ROOT, "native-build") +sys.path.insert(0, NATIVE) _cfg_path = os.path.join(ROOT, "scripts", "standalone", "config.py") _cfg_spec = importlib.util.spec_from_file_location("_amai_build_config", _cfg_path) @@ -80,11 +83,11 @@ if not USE_PGSERVER: a = Analysis( [os.path.join(ROOT, cfg["launcher"])], - pathex=[ROOT], + pathex=[ROOT, NATIVE], binaries=binaries, datas=datas, hiddenimports=hiddenimports, - hookspath=[os.path.join(ROOT, "macos/hooks")], + hookspath=[os.path.join(NATIVE, "macos", "hooks")], hooksconfig={}, runtime_hooks=[], excludes=excludes, diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index a5a05a7c..14e695c2 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -28,7 +28,7 @@ The following table details the most important paths in the repository, their pu | :---- | :---- | | app.py, app_*.py | The main entry point for the Flask web application. It handles the initialization of the Flask app, database connections, and the registration of API routes and blueprints. | | tasks/ | **The Core Logic Hub.** This is where the most intensive computations occur. Each API or async task then point to an specific implementation in this directory| -| tasks/mediaserver.py | In this fail the generic method to interact with the mediaservers are specialized to call the specific one | +| tasks/mediaserver/ | In this package the generic methods to interact with the mediaservers (`__init__.py`) dispatch to the specific backend (`jellyfin.py`, `navidrome.py`, `emby.py`, `lyrion.py`) | | tasks/ai/ | All AI / MCP code. `tasks/ai/api.py` is the provider dispatcher (called via `tasks.ai.api.call_with_tools()`), with backends in `tasks/ai/providers/{openai,gemini,mistral,ollama}.py`. Prompts are in `tasks/ai/prompts.py`. MCP tool schemas + dispatcher in `tasks/ai/tools.py`; tool bodies in `tasks/ai/tool_impl.py`. Two-stage planner (intent classifier + plan validation + execution) in `tasks/ai/planner.py`. Vocabulary normalization helpers in `tasks/ai/vocab.py`. | | config.py | Contains the application's default, non-sensitive configuration parameters. These values serve as fallbacks and can be easily overridden by environment variables, providing a flexible and secure configuration system. | | Authentication | Configured in `config.py` by `AUTH_ENABLED`, `AUDIOMUSE_USER`, `AUDIOMUSE_PASSWORD`, `API_TOKEN`, and `JWT_SECRET`. Enforcement happens in `app.py` and `app_helepr.py` functionality | diff --git a/app.py b/app.py index f1722c98..d316579d 100644 --- a/app.py +++ b/app.py @@ -10,7 +10,7 @@ # RQ imports from rq.job import Job, JobStatus from rq.exceptions import NoSuchJobError -from tasks.setup_manager import SetupManager +from tasks.setup_manager import setup_manager # Redis client from redis import Redis @@ -29,7 +29,6 @@ # The Flask instance lives in `flask_app` so RQ task modules can import it # without creating a circular import back into this file. from flask_app import app -setup_manager = SetupManager() # Import helper functions from app_helper import ( @@ -760,11 +759,6 @@ def listen_for_index_reloads(): # --- Import and Register Blueprints --- # This is the original, working structure. -# Import tasks modules to ensure they're available to RQ workers -import tasks.clustering -import tasks.analysis - - from app_chat import chat_bp from app_clustering import clustering_bp from app_analysis import analysis_bp diff --git a/app_clustering.py b/app_clustering.py index 604fad94..8e4eb3ed 100644 --- a/app_clustering.py +++ b/app_clustering.py @@ -21,7 +21,7 @@ def clustering_task_failure_handler(job, connection, type, value, tb): """A failure handler for the main clustering task, executed by the worker.""" - from app import app + from flask_app import app from app_helper import save_task_status, TASK_STATUS_FAILURE with app.app_context(): task_id = getattr(job, 'id', None) or getattr(job, 'get_id', lambda: None)() @@ -247,7 +247,7 @@ def start_clustering_endpoint(): """ # Local imports to prevent circular dependency at startup from app_helper import rq_queue_high, get_active_main_task - from app_helper import clean_up_previous_main_tasks, save_task_status, TASK_STATUS_PENDING, TASK_STATUS_FAILURE + from app_helper import clean_up_previous_main_tasks, save_task_status, TASK_STATUS_PENDING # Check for any existing active main task to prevent parallel batch runs active_task = get_active_main_task() diff --git a/app_cron.py b/app_cron.py index c68f8a90..b4b41c93 100644 --- a/app_cron.py +++ b/app_cron.py @@ -1,6 +1,8 @@ from flask import Blueprint, render_template, jsonify, request from psycopg2.extras import DictCursor -from app_helper import get_db, rq_queue_high, save_task_status, TASK_STATUS_PENDING +from database import get_db +from taskqueue import rq_queue_high +from app_helper import save_task_status, TASK_STATUS_PENDING import uuid, time, logging from config import ( TOP_N_MOODS, @@ -292,7 +294,7 @@ def run_due_cron_jobs(): f"(playlist_id={playlist_id}, tracks={len(track_ids)}, job_id={job_id})" ) except NotImplementedError: - # MPD or unsupported backend: keep the legacy date-suffixed behavior. + # Unsupported backend: keep the legacy date-suffixed behavior. legacy_name = f"Sonic Fingerprint (Cron {time.strftime('%Y-%m-%d')})" playlist_id = create_playlist_from_ids(legacy_name, track_ids) logger.info( diff --git a/app_dashboard.py b/app_dashboard.py index 32983e6b..d9f4273f 100644 --- a/app_dashboard.py +++ b/app_dashboard.py @@ -14,7 +14,8 @@ from flask import Blueprint, render_template, jsonify from psycopg2.extras import DictCursor -from app_helper import get_db, redis_conn +from database import get_db +from taskqueue import redis_conn from tz_helper import LOCAL_TZ_FMT, UTC_NOW_SQL, to_local_str logger = logging.getLogger(__name__) diff --git a/app_helper.py b/app_helper.py index a054df36..3651850f 100644 --- a/app_helper.py +++ b/app_helper.py @@ -10,7 +10,6 @@ import psycopg2 from psycopg2.extras import DictCursor import numpy as np -from flask import g from database import get_db, close_db from taskqueue import ( @@ -1419,7 +1418,7 @@ def build_and_store_map_projection(index_name='main_map'): """ # Import local projection helpers to avoid circular imports try: - from tasks.song_alchemy import _project_with_umap, _project_to_2d + from tasks.alchemy_projections import _project_with_umap, _project_to_2d except Exception: _project_with_umap = None _project_to_2d = None @@ -1540,7 +1539,7 @@ def build_and_store_artist_projection(index_name='artist_map'): Returns True on success. """ from tasks.artist_gmm_manager import load_artist_index_for_querying - from tasks.song_alchemy import _project_with_umap, _project_to_2d + from tasks.alchemy_projections import _project_with_umap, _project_to_2d # Always reload artist GMM params from database (force reload to ensure fresh data) load_artist_index_for_querying(force_reload=True) @@ -1768,25 +1767,3 @@ def cancel_job_and_children_recursive(job_id, task_type_from_db=None, reason="Ta logger.error(f"Failed to insert REVOKED recap row for {job_id}: {e_save}") return cancelled_count - - -# --- Auth / user-management helpers --- -# All auth logic (setup/auth/admin barriers, user CRUD, password hashing, -# JWT handling, the Flask routes) lives in ``app_auth``. The re-exports -# below keep the legacy ``from app_helper import ...`` paths working. -from app_auth import ( # noqa: E402 (intentional late import to avoid cycles) - USER_ROLE_USER, - USER_ROLE_ADMIN, - check_setup_needed, - check_auth_needed, - check_admin_needed, - is_admin_path, - list_additional_users, - count_admin_users, - get_additional_user_by_id, - create_additional_user, - delete_additional_user_safe, - verify_additional_user, - upsert_admin_user, - seed_admin_from_env, -) diff --git a/app_helper_artist.py b/app_helper_artist.py index bf35fa02..f285c944 100644 --- a/app_helper_artist.py +++ b/app_helper_artist.py @@ -5,7 +5,7 @@ """ import logging -from app_helper import get_db +from database import get_db from tasks.memory_utils import sanitize_string_for_db logger = logging.getLogger(__name__) diff --git a/app_map.py b/app_map.py index 80acf713..7b8160a9 100644 --- a/app_map.py +++ b/app_map.py @@ -6,16 +6,16 @@ import numpy as np import gzip -from app_helper import get_db, load_map_projection +from database import get_db +from app_helper import load_map_projection -# Try to reuse projection helpers from song_alchemy +# Try to reuse the shared projection helpers try: - from tasks.song_alchemy import _project_with_umap, _project_to_2d, _project_aligned_add_sub, _project_with_discriminant + from tasks.alchemy_projections import _project_with_umap, _project_to_2d, _project_with_discriminant except Exception: # Fallbacks will be used if import fails _project_with_umap = None _project_to_2d = None - _project_aligned_add_sub = None _project_with_discriminant = None logger = logging.getLogger(__name__) diff --git a/app_provider_migration.py b/app_provider_migration.py index 1eb24897..b4942d56 100644 --- a/app_provider_migration.py +++ b/app_provider_migration.py @@ -21,8 +21,10 @@ # App-level singletons (DB connection, Redis, RQ queues). Importing here keeps # the blueprint file self-contained — the rest of the app doesn't need to hand # anything in. -from app_helper import get_db, redis_conn, rq_queue_high, validate_outbound_url -from tasks.mediaserver_helper import detect_path_format as _detect_path_format +from database import get_db +from taskqueue import redis_conn, rq_queue_high +from app_helper import validate_outbound_url +from tasks.mediaserver.helper import detect_path_format as _detect_path_format logger = logging.getLogger(__name__) @@ -59,14 +61,14 @@ def __getattr__(self, name): # Supported target providers (what the tool knows how to talk to) # --------------------------------------------------------------------------- -_SUPPORTED_TARGETS = frozenset({'jellyfin', 'navidrome', 'emby', 'lyrion', 'mpd'}) +_SUPPORTED_TARGETS = frozenset({'jellyfin', 'navidrome', 'emby', 'lyrion'}) # --------------------------------------------------------------------------- # SSRF guard for the user-supplied media-server URL. Delegates to the shared # ``app_helper.validate_outbound_url`` (allows LAN/loopback, blocks non-HTTP(S) -# schemes and link-local/cloud-metadata). MPD targets carry no URL, so a missing -# url is allowed and left to the downstream probe. +# schemes and link-local/cloud-metadata). A missing url is allowed and left to +# the downstream probe. # --------------------------------------------------------------------------- def _validate_probe_url(creds): @@ -114,8 +116,7 @@ def _current_provider_creds(): """Build a creds dict from ``config`` for the currently active provider. Returns ``(provider_type, creds_dict)`` or ``(None, {})`` when the - provider isn't one we can re-probe (e.g. MPD — its paths come from the - filesystem directly and don't need refreshing). + provider isn't one we can re-probe. """ import config as cfg t = (getattr(cfg, 'MEDIASERVER_TYPE', '') or '').lower() diff --git a/app_setup.py b/app_setup.py index 6eae2957..347ab4cc 100644 --- a/app_setup.py +++ b/app_setup.py @@ -2,8 +2,10 @@ import types from flask import request, jsonify, render_template, make_response, after_this_request import config -from app import app, setup_manager -from app_helper import check_setup_needed, validate_outbound_url +from flask_app import app +from tasks.setup_manager import setup_manager +from app_auth import check_setup_needed +from app_helper import validate_outbound_url import restart_manager import tasks.mediaserver as mediaserver from error import error_manager @@ -74,10 +76,6 @@ 'OTHER_FEATURE_PREDOMINANCE_THRESHOLD_FOR_PURITY', 'PROBE_TOP_PLAYED_LIMIT', 'MOOD_CENTROIDS_FILE', - 'MPD_HOST', - 'MPD_MUSIC_DIRECTORY', - 'MPD_PASSWORD', - 'MPD_PORT', 'OTHER_FEATURE_LABELS', 'STRATIFIED_GENRES', 'TEMPO_MAX_BPM', @@ -219,7 +217,7 @@ def _get_allowed_setup_keys(): def _has_admin_user(): """Return True if at least one admin exists in audiomuse_users.""" try: - from app_helper import count_admin_users + from app_auth import count_admin_users return count_admin_users() > 0 except Exception as exc: app.logger.error( @@ -426,7 +424,8 @@ def setup_api(): # If auth will remain enabled we need an admin after the save. That # admin must either already exist in audiomuse_users or be provided # via the form (new_admin_user + new_admin_password). - from app_helper import count_admin_users, upsert_admin_user, get_db + from app_auth import count_admin_users, upsert_admin_user + from database import get_db auth_will_be_enabled = not auth_being_disabled if isinstance(simulated.AUTH_ENABLED, str): auth_will_be_enabled = simulated.AUTH_ENABLED.strip().lower() == 'true' @@ -710,7 +709,7 @@ def setup_lyrics_api_analyze(): ctx = None import time as _time _t0 = _time.monotonic() - with urllib.request.urlopen(req, timeout=10, context=ctx) as resp: + with urllib.request.urlopen(req, timeout=60, context=ctx) as resp: raw_bytes = resp.read(512 * 1024) elapsed_ms = (_time.monotonic() - _t0) * 1000 raw_text = raw_bytes.decode('utf-8', errors='replace') diff --git a/app_sync.py b/app_sync.py index ea53b845..83694769 100644 --- a/app_sync.py +++ b/app_sync.py @@ -20,7 +20,8 @@ from flasgger import swag_from import config -from app_helper import get_db, load_map_projection +from database import get_db +from app_helper import load_map_projection logger = logging.getLogger(__name__) @@ -57,7 +58,7 @@ 'Three modes: `?fields=index` returns a lightweight {id, fp} manifest ' '(<=1000/page) for client-side change detection; `?ids=a,b,c` returns full ' 'payloads for a specific id set (<=500); default returns the full library ' - 'page by page (<=500). Not supported for the `mpd` media server (501).' + 'page by page (<=500).' ), 'parameters': [ {'name': 'fields', 'in': 'query', 'required': False, @@ -79,13 +80,9 @@ 'responses': { '200': {'description': 'A page of the manifest or the full payload.'}, '500': {'description': 'Internal server error.'}, - '501': {'description': 'Media server type not supported (e.g. `mpd`).'}, }, }) def sync_endpoint(): - if config.MEDIASERVER_TYPE == 'mpd': - return jsonify({"error": "mpd is not yet supported by the mobile sync endpoint"}), 501 - manifest_mode = request.args.get('fields') == 'index' page = max(1, request.args.get('page', 1, type=int)) max_limit = _MAX_MANIFEST_LIMIT if manifest_mode else _MAX_PAYLOAD_LIMIT diff --git a/app_voyager.py b/app_voyager.py index 283ab8c1..04253483 100644 --- a/app_voyager.py +++ b/app_voyager.py @@ -2,6 +2,7 @@ from flask import Blueprint, jsonify, request, render_template import logging import json +import threading import numpy as np # Import the new config option @@ -18,9 +19,10 @@ logger = logging.getLogger(__name__) -# --- Load mood centroids at module level --- _MOOD_CENTROIDS_DATA = {} # mood_name -> list of centroid dicts (with vectors) _MOOD_CENTROIDS_META = {} # mood_name -> list of {cluster_id, top_tags (top 3)} for API +_mood_centroids_loaded = False +_mood_centroids_lock = threading.Lock() def _load_mood_centroids_for_similarity(): try: @@ -45,7 +47,22 @@ def _load_mood_centroids_for_similarity(): except Exception as e: logger.warning(f"Could not load mood centroids from {MOOD_CENTROIDS_FILE}: {e}") -_load_mood_centroids_for_similarity() +def _ensure_mood_centroids_loaded(): + """Parse the mood-centroids JSON on first use instead of at import. + + The file is ~1MB; loading it lazily keeps module import (and therefore + web/worker startup) free of the parse cost. A single attempt is made, + matching the old import-time behavior where a failed load left the + dicts empty without retrying. + """ + global _mood_centroids_loaded + if _mood_centroids_loaded: + return + with _mood_centroids_lock: + if _mood_centroids_loaded: + return + _load_mood_centroids_for_similarity() + _mood_centroids_loaded = True # Create a Blueprint for Voyager (similarity) related routes voyager_bp = Blueprint('voyager_bp', __name__, template_folder='../templates') @@ -190,6 +207,7 @@ def get_mood_centroids_endpoint(): 200: description: Dictionary of mood names to lists of centroid metadata. """ + _ensure_mood_centroids_loaded() mood_filter = request.args.get('mood', '', type=str).strip().lower() if mood_filter: if mood_filter not in _MOOD_CENTROIDS_META: @@ -300,6 +318,7 @@ def get_similar_tracks_endpoint(): # --- Mood centroid mode: use centroid vector instead of a song --- if mood_param and centroid_index_param is not None: + _ensure_mood_centroids_loaded() if mood_param not in _MOOD_CENTROIDS_DATA: return jsonify({"error": f"Unknown mood '{mood_param}'. Available: {list(_MOOD_CENTROIDS_DATA.keys())}"}), 400 centroids = _MOOD_CENTROIDS_DATA[mood_param] diff --git a/app_waveform.py b/app_waveform.py index 8215355d..eb94d345 100644 --- a/app_waveform.py +++ b/app_waveform.py @@ -15,7 +15,7 @@ logger.warning("librosa not available. Waveform generation will not work. Install with: pip install librosa") from config import MEDIASERVER_TYPE -from app_helper import get_db +from database import get_db logger = logging.getLogger(__name__) @@ -212,7 +212,7 @@ def get_waveform_endpoint(): # This ensures we have all the metadata needed for proper file extension detection if MEDIASERVER_TYPE == "navidrome": # Import Navidrome-specific function to get full song details - from tasks.mediaserver_navidrome import _navidrome_request + from tasks.mediaserver.navidrome import _navidrome_request song_response = _navidrome_request("getSong", {"id": item_id}) if song_response and "song" in song_response: item = song_response["song"] @@ -240,7 +240,6 @@ def get_waveform_endpoint(): item = { 'Id': item_id, # Jellyfin/Emby format 'id': item_id, # Navidrome/Lyrion format - 'file': item_id, # MPD format (uses file path as ID) 'Name': title, 'Path': '' # Will be fetched by download_track if needed } diff --git a/config.py b/config.py index 3cbaedb8..e3350f4c 100644 --- a/config.py +++ b/config.py @@ -2,7 +2,7 @@ import os # --- Media Server Type --- -MEDIASERVER_TYPE = os.environ.get("MEDIASERVER_TYPE", "jellyfin").lower() # Possible values: jellyfin, navidrome, lyrion, mpd, emby +MEDIASERVER_TYPE = os.environ.get("MEDIASERVER_TYPE", "jellyfin").lower() # Possible values: jellyfin, navidrome, lyrion, emby # --- Jellyfin and DB Constants (Read from Environment Variables first) --- @@ -98,16 +98,8 @@ def _compute_headers(): 'LYRICS_INSTRUMENTAL_AXIS_FILL', } -# --- MPD (Music Player Daemon) Constants --- -# These are used only if MEDIASERVER_TYPE is "mpd". -MPD_HOST = os.environ.get("MPD_HOST", "localhost") -MPD_PORT = int(os.environ.get("MPD_PORT", "6600")) -MPD_PASSWORD = os.environ.get("MPD_PASSWORD", "") # Optional password, leave empty if none -MPD_MUSIC_DIRECTORY = os.environ.get("MPD_MUSIC_DIRECTORY", "/var/lib/mpd/music") # Path to MPD's music directory for file access - - # --- General Constants (Read from Environment Variables where applicable) --- -APP_VERSION = "v2.1.5" +APP_VERSION = "v2.2.0" MAX_DISTANCE = float(os.environ.get("MAX_DISTANCE", "0.5")) MAX_SONGS_PER_CLUSTER = int(os.environ.get("MAX_SONGS_PER_CLUSTER", "0")) MAX_SONGS_PER_ARTIST = int(os.getenv("MAX_SONGS_PER_ARTIST", "3")) # Max songs per artist in similarity results and clustering @@ -667,36 +659,41 @@ def _compute_headers(): # Default is True to preserve the current secure behavior. AUTH_ENABLED = os.environ.get("AUTH_ENABLED", "True").lower() == "true" -try: - from tasks.setup_manager import SetupManager - _setup_manager = SetupManager() - worker_mode = os.environ.get('AUDIOMUSE_ROLE', '').lower() == 'worker' - if worker_mode: - if _setup_manager.config_table_exists(): - _overrides = _setup_manager.get_raw_overrides(ensure_table=False) +def _apply_db_overrides(): + global HEADERS, refresh_config + try: + from tasks.setup_manager import SetupManager + _setup_manager = SetupManager() + worker_mode = os.environ.get('AUDIOMUSE_ROLE', '').lower() == 'worker' + if worker_mode: + if _setup_manager.config_table_exists(): + _overrides = _setup_manager.get_raw_overrides(ensure_table=False) + else: + _overrides = {} else: - _overrides = {} - else: - _setup_manager.ensure_table() - _overrides = _setup_manager.get_raw_overrides() - _excluded_override_keys = globals().get('SETUP_BOOTSTRAP_EXCLUDED_KEYS', set()) - for _key, _value in _overrides.items(): - # Skip any keys that are explicitly excluded from overrides (Redis and Postgres) - if _key in _excluded_override_keys: - continue - # Read the value from the db and override the variable - if _key in globals(): - globals()[_key] = _setup_manager.cast_value(globals()[_key], _value) - - HEADERS = _compute_headers() - - def refresh_config(): - """Reload the config module from the current database and environment.""" - import importlib - import sys - importlib.reload(sys.modules[__name__]) -except Exception as _exc: - import logging - logging.getLogger(__name__).warning(f"Could not load config overrides from DB: {_exc}") - def refresh_config(): - pass + _setup_manager.ensure_table() + _overrides = _setup_manager.get_raw_overrides() + _excluded_override_keys = globals().get('SETUP_BOOTSTRAP_EXCLUDED_KEYS', set()) + for _key, _value in _overrides.items(): + # Skip any keys that are explicitly excluded from overrides (Redis and Postgres) + if _key in _excluded_override_keys: + continue + # Read the value from the db and override the variable + if _key in globals(): + globals()[_key] = _setup_manager.cast_value(globals()[_key], _value) + + HEADERS = _compute_headers() + + def refresh_config(): + """Reload the config module from the current database and environment.""" + import importlib + import sys + importlib.reload(sys.modules[__name__]) + except Exception as _exc: + import logging + logging.getLogger(__name__).warning(f"Could not load config overrides from DB: {_exc}") + def refresh_config(): + pass + + +_apply_db_overrides() diff --git a/database.py b/database.py index e0764d30..cfb96182 100644 --- a/database.py +++ b/database.py @@ -55,7 +55,7 @@ def start_embedded(data_dir): ``embedded``. The data directory must live outside the read-only app bundle and its path must not contain spaces (pgserver doubles it as the unix-socket dir, which ``postgres`` receives via ``pg_ctl -o '-k '`` and re-splits on - whitespace); see ``macos/paths.py::app_support_dir``. Initializes the cluster + whitespace); see ``native-build/macos/paths.py::app_support_dir``. Initializes the cluster on first run, idempotent afterwards. """ global _embedded_server diff --git a/docs/ALGORITHM.md b/docs/ALGORITHM.md index 925fdaef..1d2395a2 100644 --- a/docs/ALGORITHM.md +++ b/docs/ALGORITHM.md @@ -86,7 +86,7 @@ Core components and responsibilities: - ONNX Models & Audio Stack: Analysis uses ONNX Runtime to run embedding and prediction models. Audio loading uses `librosa` with a `pydub`/ffmpeg fallback for resilient decoding. The Docker image pre-fetches ONNX model files and pins runtime libs to ensure consistent behavior across environments. -- Media Server Adapters: `mediaserver.py` provides adapters for Jellyfin, Navidrome, Emby, etc., enabling playlist creation and reading play-history for the Sonic Fingerprint feature. +- Media Server Adapters: the `tasks/mediaserver/` package provides adapters for Jellyfin, Navidrome, Emby, etc., enabling playlist creation and reading play-history for the Sonic Fingerprint feature. Deployment considerations (informed by `Dockerfile`): @@ -266,13 +266,12 @@ The Song Analysis functionality is configured by the following environment varia #### **Media Server** -* MEDIASERVER\_TYPE: **(Required)** Specifies the media server to connect to. (e.g., jellyfin, navidrome, emby, lyrion, mpd). +* MEDIASERVER\_TYPE: **(Required)** Specifies the media server to connect to. (e.g., jellyfin, navidrome, emby, lyrion). * MUSIC\_LIBRARIES: (Optional) A comma-separated list of library names to scan. If empty, all music libraries are scanned. * JELLYFIN\_URL, JELLYFIN\_USER\_ID, JELLYFIN\_TOKEN: Credentials for Jellyfin (if MEDIASERVER\_TYPE="jellyfin"). * EMBY\_URL, EMBY\_USER\_ID, EMBY\_TOKEN: Credentials for Emby (if MEDIASERVER\_TYPE="emby"). * NAVIDROME\_URL, NAVIDROME\_USER, NAVIDROME\_PASSWORD: Credentials for Navidrome (if MEDIASERVER\_TYPE="navidrome"). -* LYRION\_URL: Credentials for Lyrion (if MEDIASERVER\_TYPE="lyrion"). -* MPD\_HOST, MPD\_PORT, MPD\_PASSWORD, MPD\_MUSIC\_DIRECTORY: Credentials for MPD (if MEDIASERVER\_TYPE="mpd"). +* LYRION\_URL: Credentials for Lyrion (if MEDIASERVER\_TYPE="lyrion"). #### **Task & Performance Tuning** @@ -624,7 +623,7 @@ This feature primarily interacts with the pre-built Voyager index for fast simil #### **Stage 4: Playlist Creation** 1. **Route:** Clicking "Create Playlist" (similarity.html) sends a POST request to /api/create\_playlist (app\_voyager.py). The payload contains the desired playlist\_name and the list of track\_ids (seed song \+ similar songs). -2. **Backend Logic:** The endpoint calls create\_playlist\_from\_ids (voyager\_manager.py), which in turn calls create\_instant\_playlist (mediaserver.py). This function uses the configured MEDIASERVER\_TYPE and credentials to interact with the media server's API and create the actual playlist. +2. **Backend Logic:** The endpoint calls create\_playlist\_from\_ids (voyager\_manager.py), which in turn calls create\_instant\_playlist (tasks/mediaserver/). This function uses the configured MEDIASERVER\_TYPE and credentials to interact with the media server's API and create the actual playlist. ### **3.3. Environment Variable Configuration** @@ -1009,11 +1008,11 @@ This feature combines media server interaction for user history with vector anal #### **Stage 2: Fingerprint Generation (generate\_sonic\_fingerprint in sonic\_fingerprint\_manager.py)** -1. **Fetch Top Songs:** Calls get\_top\_played\_songs (from mediaserver.py), passing the user\_creds. This function interacts with the media server API (based on MEDIASERVER\_TYPE) to retrieve the user's top SONIC\_FINGERPRINT\_TOP\_N\_SONGS most played tracks. +1. **Fetch Top Songs:** Calls get\_top\_played\_songs (from tasks/mediaserver/), passing the user\_creds. This function interacts with the media server API (based on MEDIASERVER\_TYPE) to retrieve the user's top SONIC\_FINGERPRINT\_TOP\_N\_SONGS most played tracks. 2. **Fetch Embeddings:** Retrieves the embedding vectors (embedding\_vector) for these top songs from the application's PostgreSQL database (embedding table) using get\_tracks\_by\_ids. 3. **Calculate Recency Weights:** * Iterates through the top songs (for which embeddings were found). - * For each song, calls get\_last\_played\_time (from mediaserver.py), passing user\_creds, to get the timestamp of the last play. + * For each song, calls get\_last\_played\_time (from tasks/mediaserver/), passing user\_creds, to get the timestamp of the last play. * Calculates days\_since\_played. * Applies an **exponential decay function** (weight \= exp(-decay\_rate \* days\_since\_played)) to calculate a weight. The half\_life (set to 30 days) determines how quickly the weight decreases for older plays. Songs without a valid last played time receive a fixed lower weight. 4. **Weighted Average:** Calculates the weighted average of the embedding vectors (average\_vector). This vector represents the user's "sonic fingerprint". @@ -1120,7 +1119,7 @@ Stage 5: Execute Query & Post-process Results Stage 6: Optional Playlist Creation on Media Server -1. The frontend posts the playlist name and `item_ids` to an endpoint that maps internal `item_id`s to media-server-specific IDs and creates the playlist using the configured media server adapter (Jellyfin/Emby/Navidrome). These adapter functions live in `mediaserver.py`. +1. The frontend posts the playlist name and `item_ids` to an endpoint that maps internal `item_id`s to media-server-specific IDs and creates the playlist using the configured media server adapter (Jellyfin/Emby/Navidrome). These adapter functions live in `tasks/mediaserver/`. 2. Return the media server response (success, playlist id, or error) to the frontend and display it in the UI. Safety & Fallbacks diff --git a/linux/__init__.py b/native-build/linux/__init__.py similarity index 84% rename from linux/__init__.py rename to native-build/linux/__init__.py index a0208290..62de9913 100644 --- a/linux/__init__.py +++ b/native-build/linux/__init__.py @@ -12,10 +12,10 @@ package (``macos.control_ipc.ControlServer`` and ``macos.reverse_log.NewestFirstFileHandler``) instead of duplicating them. Both are pure-stdlib and contain nothing macOS-specific. -* The child env (``linux/env.py``) reports ``AUDIOMUSE_PLATFORM=macos`` on +* The child env (``native-build/linux/env.py``) reports ``AUDIOMUSE_PLATFORM=macos`` on purpose: that value is the *only* platform-keyed branch in the shared ``restart_manager.py`` and it selects the unix-socket control-server path (which this package's supervisor implements identically). We are not allowed to touch shared code to add a separate ``linux`` value, so we ride the - existing standalone-mode branch. See ``linux/env.py`` for the full rationale. + existing standalone-mode branch. See ``native-build/linux/env.py`` for the full rationale. """ diff --git a/linux/db_backend.py b/native-build/linux/db_backend.py similarity index 100% rename from linux/db_backend.py rename to native-build/linux/db_backend.py diff --git a/linux/embedded_pg.py b/native-build/linux/embedded_pg.py similarity index 98% rename from linux/embedded_pg.py rename to native-build/linux/embedded_pg.py index 91097aa4..05819da6 100644 --- a/linux/embedded_pg.py +++ b/native-build/linux/embedded_pg.py @@ -12,7 +12,7 @@ The bundled server is relocatable: PostgreSQL derives its support-file paths from the running executable's location, so the tree works wherever the package is installed (``/opt/AudioMuse-AI/_internal/pgsql``). The unix socket lives in -the data dir itself (which ``linux/paths.py`` guarantees is space-free), exactly +the data dir itself (which ``native-build/linux/paths.py`` guarantees is space-free), exactly like the pgserver/macOS setup. """ diff --git a/linux/env.py b/native-build/linux/env.py similarity index 97% rename from linux/env.py rename to native-build/linux/env.py index 2e0e822d..823cd947 100644 --- a/linux/env.py +++ b/native-build/linux/env.py @@ -1,6 +1,6 @@ """Build the environment handed to each supervised child process (Linux build). -Linux counterpart of ``macos/env.py``. The standalone-mode overrides are +Linux counterpart of ``native-build/macos/env.py``. The standalone-mode overrides are centralized here so every child imports ``config`` already pointed at the embedded services and the bundled models. Keys mirror the env-driven settings in ``config.py`` (``DATABASE_URL``, ``REDIS_URL``, ``TEMP_DIR``, the ``*_MODEL_PATH`` @@ -19,7 +19,7 @@ flow work on the native Linux build with **zero shared-code changes**. The value is an internal "standalone embedded supervisor" signal, not a real OS check. -Note the macOS-only workarounds from ``macos/env.py`` are intentionally omitted: +Note the macOS-only workarounds from ``native-build/macos/env.py`` are intentionally omitted: ``OBJC_DISABLE_INITIALIZE_FORK_SAFETY`` and the ``LC_NUMERIC=C`` pin both address Apple-framework behavior that does not exist on Linux (and Linux is the platform ``config.py``/the container already target, so its defaults are correct here). diff --git a/linux/launcher.py b/native-build/linux/launcher.py similarity index 99% rename from linux/launcher.py rename to native-build/linux/launcher.py index e774ac2b..94de9175 100644 --- a/linux/launcher.py +++ b/native-build/linux/launcher.py @@ -1,6 +1,6 @@ """Standalone Linux entry point (the PyInstaller entry script). -Linux counterpart of ``macos/launcher.py``. There is no native menu-bar agent +Linux counterpart of ``native-build/macos/launcher.py``. There is no native menu-bar agent here (rumps/AppKit are macOS-only); instead the frozen binary is a small multi-call launcher: diff --git a/linux/packaging/AudioMuse-AI-stop.desktop b/native-build/linux/packaging/AudioMuse-AI-stop.desktop similarity index 100% rename from linux/packaging/AudioMuse-AI-stop.desktop rename to native-build/linux/packaging/AudioMuse-AI-stop.desktop diff --git a/linux/packaging/AudioMuse-AI.desktop b/native-build/linux/packaging/AudioMuse-AI.desktop similarity index 100% rename from linux/packaging/AudioMuse-AI.desktop rename to native-build/linux/packaging/AudioMuse-AI.desktop diff --git a/linux/packaging/audiomuse-ai.service b/native-build/linux/packaging/audiomuse-ai.service similarity index 100% rename from linux/packaging/audiomuse-ai.service rename to native-build/linux/packaging/audiomuse-ai.service diff --git a/linux/packaging/icons/audiomuse-ai_128.png b/native-build/linux/packaging/icons/audiomuse-ai_128.png similarity index 100% rename from linux/packaging/icons/audiomuse-ai_128.png rename to native-build/linux/packaging/icons/audiomuse-ai_128.png diff --git a/linux/packaging/icons/audiomuse-ai_256.png b/native-build/linux/packaging/icons/audiomuse-ai_256.png similarity index 100% rename from linux/packaging/icons/audiomuse-ai_256.png rename to native-build/linux/packaging/icons/audiomuse-ai_256.png diff --git a/linux/packaging/icons/audiomuse-ai_32.png b/native-build/linux/packaging/icons/audiomuse-ai_32.png similarity index 100% rename from linux/packaging/icons/audiomuse-ai_32.png rename to native-build/linux/packaging/icons/audiomuse-ai_32.png diff --git a/linux/packaging/icons/audiomuse-ai_48.png b/native-build/linux/packaging/icons/audiomuse-ai_48.png similarity index 100% rename from linux/packaging/icons/audiomuse-ai_48.png rename to native-build/linux/packaging/icons/audiomuse-ai_48.png diff --git a/linux/packaging/icons/audiomuse-ai_512.png b/native-build/linux/packaging/icons/audiomuse-ai_512.png similarity index 100% rename from linux/packaging/icons/audiomuse-ai_512.png rename to native-build/linux/packaging/icons/audiomuse-ai_512.png diff --git a/linux/packaging/icons/audiomuse-ai_64.png b/native-build/linux/packaging/icons/audiomuse-ai_64.png similarity index 100% rename from linux/packaging/icons/audiomuse-ai_64.png rename to native-build/linux/packaging/icons/audiomuse-ai_64.png diff --git a/linux/packaging/nfpm.yaml.in b/native-build/linux/packaging/nfpm.yaml.in similarity index 96% rename from linux/packaging/nfpm.yaml.in rename to native-build/linux/packaging/nfpm.yaml.in index 8bfa6820..87a79180 100644 --- a/linux/packaging/nfpm.yaml.in +++ b/native-build/linux/packaging/nfpm.yaml.in @@ -76,5 +76,5 @@ contents: type: symlink scripts: - postinstall: linux/packaging/postinstall.sh - postremove: linux/packaging/postremove.sh + postinstall: native-build/linux/packaging/postinstall.sh + postremove: native-build/linux/packaging/postremove.sh diff --git a/linux/packaging/postinstall.sh b/native-build/linux/packaging/postinstall.sh similarity index 100% rename from linux/packaging/postinstall.sh rename to native-build/linux/packaging/postinstall.sh diff --git a/linux/packaging/postremove.sh b/native-build/linux/packaging/postremove.sh similarity index 100% rename from linux/packaging/postremove.sh rename to native-build/linux/packaging/postremove.sh diff --git a/linux/paths.py b/native-build/linux/paths.py similarity index 93% rename from linux/paths.py rename to native-build/linux/paths.py index f7fe4aba..664661a3 100644 --- a/linux/paths.py +++ b/native-build/linux/paths.py @@ -1,6 +1,6 @@ """Filesystem locations for the standalone Linux build. -Mirrors ``macos/paths.py`` but follows the XDG Base Directory spec instead of +Mirrors ``native-build/macos/paths.py`` but follows the XDG Base Directory spec instead of ``~/Library``. Read-only resources (ONNX models, Flask templates/static, the bundled @@ -36,7 +36,7 @@ def resource_root(): def _repo_root(): - return os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + return os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) def _ensure(path): @@ -143,7 +143,7 @@ def redis_binary(): def _uses_pgserver(): """x86_64 uses the pgserver wheel; other arches (aarch64) use the from-source PostgreSQL bundled under ``pgsql/`` (pgserver has no arm64 - wheel). Mirrors ``linux/db_backend.py``'s selector.""" + wheel). Mirrors ``native-build/linux/db_backend.py``'s selector.""" return platform.machine() in ("x86_64", "amd64") @@ -153,8 +153,8 @@ def pg_install_dir(): * x86_64: the pgserver wheel's ``pginstall`` tree (bundled at ``pgserver/pginstall`` when frozen; resolved from the package in dev). * aarch64: the from-source server bundled at ``pgsql/`` (built by - ``linux/vendor/postgres/build-postgres.sh`` into - ``linux/vendor/postgres/`` in dev). + ``native-build/linux/vendor/postgres/build-postgres.sh`` into + ``native-build/linux/vendor/postgres/`` in dev). """ if _uses_pgserver(): if getattr(sys, "frozen", False): diff --git a/linux/supervisor.py b/native-build/linux/supervisor.py similarity index 99% rename from linux/supervisor.py rename to native-build/linux/supervisor.py index 350bd2db..d3c5c027 100644 --- a/linux/supervisor.py +++ b/native-build/linux/supervisor.py @@ -1,6 +1,6 @@ """Process supervisor for the standalone Linux app. -Linux counterpart of ``macos/supervisor.py`` -- the logic is platform-agnostic +Linux counterpart of ``native-build/macos/supervisor.py`` -- the logic is platform-agnostic (stdlib ``subprocess``/``signal``/``os.killpg`` + ``psutil``), so this is a near copy that swaps ``macos`` path/env helpers for the ``linux`` ones and reuses the two genuinely platform-neutral helpers from the macOS package diff --git a/linux/vendor/build-redis.sh b/native-build/linux/vendor/build-redis.sh similarity index 91% rename from linux/vendor/build-redis.sh rename to native-build/linux/vendor/build-redis.sh index 64d8c7b3..0ded4efc 100755 --- a/linux/vendor/build-redis.sh +++ b/native-build/linux/vendor/build-redis.sh @@ -1,7 +1,7 @@ #!/usr/bin/env bash # # Build a self-contained ``redis-server`` for the native Linux bundle and drop it -# at ``linux/vendor/redis//redis-server`` (where the PyInstaller spec +# at ``native-build/linux/vendor/redis//redis-server`` (where the PyInstaller spec # expects it). Run from the repo root on the target architecture (the CI # workflow runs it on an x86_64 and an aarch64 runner). # @@ -17,7 +17,7 @@ set -euo pipefail REDIS_VERSION="${REDIS_VERSION:-7.4.2}" ARCH="$(uname -m)" # x86_64 | aarch64 -DEST="linux/vendor/redis/${ARCH}" +DEST="native-build/linux/vendor/redis/${ARCH}" echo "==> Building redis-server ${REDIS_VERSION} for ${ARCH}" work="$(mktemp -d)" diff --git a/linux/vendor/pg-contrib/build-pg-contrib.sh b/native-build/linux/vendor/pg-contrib/build-pg-contrib.sh similarity index 91% rename from linux/vendor/pg-contrib/build-pg-contrib.sh rename to native-build/linux/vendor/pg-contrib/build-pg-contrib.sh index a2251fd9..39bda980 100755 --- a/linux/vendor/pg-contrib/build-pg-contrib.sh +++ b/native-build/linux/vendor/pg-contrib/build-pg-contrib.sh @@ -2,12 +2,12 @@ # # Compile the PostgreSQL ``unaccent`` and ``pg_trgm`` contrib extensions against # the EXACT PostgreSQL version that ``pgserver`` bundles, and drop the artifacts -# under ``linux/vendor/pg-contrib//`` where the PyInstaller spec grafts +# under ``native-build/linux/vendor/pg-contrib//`` where the PyInstaller spec grafts # them into the bundled ``pgserver/pginstall`` tree. # # Run from the repo root, inside the build venv (so ``pgserver`` is importable): # source .venv-linux/bin/activate -# bash linux/vendor/pg-contrib/build-pg-contrib.sh +# bash native-build/linux/vendor/pg-contrib/build-pg-contrib.sh # # Why this is necessary: ``pgserver`` ships a MINIMAL PostgreSQL (only plpgsql + # pgvector), but the AudioMuse-AI schema (app_helper.py::init_db) runs @@ -19,7 +19,7 @@ set -euo pipefail ARCH="$(uname -m)" # x86_64 | aarch64 -DEST="linux/vendor/pg-contrib/${ARCH}" +DEST="native-build/linux/vendor/pg-contrib/${ARCH}" # pgserver's own pg_config (in the active venv). PGC="$(python -c 'import pgserver,os;print(os.path.join(os.path.dirname(pgserver.__file__),"pginstall","bin","pg_config"))')" diff --git a/linux/vendor/postgres/build-postgres.sh b/native-build/linux/vendor/postgres/build-postgres.sh similarity index 93% rename from linux/vendor/postgres/build-postgres.sh rename to native-build/linux/vendor/postgres/build-postgres.sh index 1bd837da..ecc3ffce 100755 --- a/linux/vendor/postgres/build-postgres.sh +++ b/native-build/linux/vendor/postgres/build-postgres.sh @@ -2,13 +2,13 @@ # # Build a relocatable PostgreSQL (server + client tools + the ``unaccent`` and # ``pg_trgm`` contrib extensions) from source and install it under -# ``linux/vendor/postgres//`` where the PyInstaller spec bundles it as +# ``native-build/linux/vendor/postgres//`` where the PyInstaller spec bundles it as # ``pgsql/``. Used for the **aarch64** Linux build, where ``pgserver`` has no # wheel. # # Run from the repo root on the target architecture (the CI workflow runs it on # the aarch64 runner): -# bash linux/vendor/postgres/build-postgres.sh +# bash native-build/linux/vendor/postgres/build-postgres.sh # # Why from source (not a prebuilt binary like zonky's embedded-postgres): # * we must compile the unaccent/pg_trgm contrib modules, which need the @@ -22,7 +22,7 @@ set -euo pipefail PG_VERSION="${PG_VERSION:-16.9}" ARCH="$(uname -m)" # aarch64 -PREFIX="$(pwd)/linux/vendor/postgres/${ARCH}" +PREFIX="$(pwd)/native-build/linux/vendor/postgres/${ARCH}" echo "==> Building PostgreSQL ${PG_VERSION} (${ARCH}) -> ${PREFIX}" rm -rf "$PREFIX" diff --git a/macos/__init__.py b/native-build/macos/__init__.py similarity index 100% rename from macos/__init__.py rename to native-build/macos/__init__.py diff --git a/macos/assets/AudioMuse-AI.icns b/native-build/macos/assets/AudioMuse-AI.icns similarity index 100% rename from macos/assets/AudioMuse-AI.icns rename to native-build/macos/assets/AudioMuse-AI.icns diff --git a/macos/assets/menubar-icon.png b/native-build/macos/assets/menubar-icon.png similarity index 100% rename from macos/assets/menubar-icon.png rename to native-build/macos/assets/menubar-icon.png diff --git a/macos/control_ipc.py b/native-build/macos/control_ipc.py similarity index 100% rename from macos/control_ipc.py rename to native-build/macos/control_ipc.py diff --git a/macos/entitlements.plist b/native-build/macos/entitlements.plist similarity index 100% rename from macos/entitlements.plist rename to native-build/macos/entitlements.plist diff --git a/macos/env.py b/native-build/macos/env.py similarity index 98% rename from macos/env.py rename to native-build/macos/env.py index 1ddb3d16..0941dada 100644 --- a/macos/env.py +++ b/native-build/macos/env.py @@ -35,7 +35,7 @@ def build_child_env(role, database_url, redis_url): # tasks stuck pending. Pinning the *numeric* locale to C makes strtold # deterministic with no transition to race against. Kept to LC_NUMERIC only # so LC_CTYPE/UTF-8 (accented file paths) is untouched. The matching - # in-process pin lives in macos/launcher.py (numeric_bootstrap.py); this + # in-process pin lives in native-build/macos/launcher.py (numeric_bootstrap.py); this # is all macOS-only and does not affect the Linux/Docker images. "LC_NUMERIC": "C", "APP_DATA_DIR": paths.app_support_dir(), diff --git a/macos/hooks/hook-tasks.py b/native-build/macos/hooks/hook-tasks.py similarity index 100% rename from macos/hooks/hook-tasks.py rename to native-build/macos/hooks/hook-tasks.py diff --git a/macos/launcher.py b/native-build/macos/launcher.py similarity index 100% rename from macos/launcher.py rename to native-build/macos/launcher.py diff --git a/macos/make_icns.sh b/native-build/macos/make_icns.sh similarity index 94% rename from macos/make_icns.sh rename to native-build/macos/make_icns.sh index 2f8cee3c..598c7800 100644 --- a/macos/make_icns.sh +++ b/native-build/macos/make_icns.sh @@ -2,7 +2,7 @@ set -euo pipefail SRC="screenshot/audiomuseai.png" -OUT_DIR="macos/assets" +OUT_DIR="native-build/macos/assets" WORK="$(mktemp -d)/AudioMuse-AI.iconset" mkdir -p "$WORK" "$OUT_DIR" diff --git a/macos/paths.py b/native-build/macos/paths.py similarity index 97% rename from macos/paths.py rename to native-build/macos/paths.py index a9f42dde..2c58ab6b 100644 --- a/macos/paths.py +++ b/native-build/macos/paths.py @@ -25,7 +25,7 @@ def resource_root(): def _repo_root(): - return os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + return os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) def _ensure(path): diff --git a/macos/reverse_log.py b/native-build/macos/reverse_log.py similarity index 100% rename from macos/reverse_log.py rename to native-build/macos/reverse_log.py diff --git a/macos/supervisor.py b/native-build/macos/supervisor.py similarity index 99% rename from macos/supervisor.py rename to native-build/macos/supervisor.py index 0636e3a9..f9ddd827 100644 --- a/macos/supervisor.py +++ b/native-build/macos/supervisor.py @@ -63,7 +63,7 @@ def _setup_logging(self): if not log.handlers: # Newest line on top so opening the log shows the latest activity # first. Bounded by line count (~40k) instead of a byte cap; see - # macos/reverse_log.py for why it's not a literal per-line prepend. + # native-build/macos/reverse_log.py for why it's not a literal per-line prepend. handler = NewestFirstFileHandler(paths.log_file()) handler.setFormatter(logging.Formatter("%(asctime)s %(message)s")) log.addHandler(handler) diff --git a/macos/vendor/pg-contrib/arm64/extension/pg_trgm--1.0--1.1.sql b/native-build/macos/vendor/pg-contrib/arm64/extension/pg_trgm--1.0--1.1.sql similarity index 100% rename from macos/vendor/pg-contrib/arm64/extension/pg_trgm--1.0--1.1.sql rename to native-build/macos/vendor/pg-contrib/arm64/extension/pg_trgm--1.0--1.1.sql diff --git a/macos/vendor/pg-contrib/arm64/extension/pg_trgm--1.1--1.2.sql b/native-build/macos/vendor/pg-contrib/arm64/extension/pg_trgm--1.1--1.2.sql similarity index 100% rename from macos/vendor/pg-contrib/arm64/extension/pg_trgm--1.1--1.2.sql rename to native-build/macos/vendor/pg-contrib/arm64/extension/pg_trgm--1.1--1.2.sql diff --git a/macos/vendor/pg-contrib/arm64/extension/pg_trgm--1.2--1.3.sql b/native-build/macos/vendor/pg-contrib/arm64/extension/pg_trgm--1.2--1.3.sql similarity index 100% rename from macos/vendor/pg-contrib/arm64/extension/pg_trgm--1.2--1.3.sql rename to native-build/macos/vendor/pg-contrib/arm64/extension/pg_trgm--1.2--1.3.sql diff --git a/macos/vendor/pg-contrib/arm64/extension/pg_trgm--1.3--1.4.sql b/native-build/macos/vendor/pg-contrib/arm64/extension/pg_trgm--1.3--1.4.sql similarity index 100% rename from macos/vendor/pg-contrib/arm64/extension/pg_trgm--1.3--1.4.sql rename to native-build/macos/vendor/pg-contrib/arm64/extension/pg_trgm--1.3--1.4.sql diff --git a/macos/vendor/pg-contrib/arm64/extension/pg_trgm--1.3.sql b/native-build/macos/vendor/pg-contrib/arm64/extension/pg_trgm--1.3.sql similarity index 100% rename from macos/vendor/pg-contrib/arm64/extension/pg_trgm--1.3.sql rename to native-build/macos/vendor/pg-contrib/arm64/extension/pg_trgm--1.3.sql diff --git a/macos/vendor/pg-contrib/arm64/extension/pg_trgm--1.4--1.5.sql b/native-build/macos/vendor/pg-contrib/arm64/extension/pg_trgm--1.4--1.5.sql similarity index 100% rename from macos/vendor/pg-contrib/arm64/extension/pg_trgm--1.4--1.5.sql rename to native-build/macos/vendor/pg-contrib/arm64/extension/pg_trgm--1.4--1.5.sql diff --git a/macos/vendor/pg-contrib/arm64/extension/pg_trgm--1.5--1.6.sql b/native-build/macos/vendor/pg-contrib/arm64/extension/pg_trgm--1.5--1.6.sql similarity index 100% rename from macos/vendor/pg-contrib/arm64/extension/pg_trgm--1.5--1.6.sql rename to native-build/macos/vendor/pg-contrib/arm64/extension/pg_trgm--1.5--1.6.sql diff --git a/macos/vendor/pg-contrib/arm64/extension/pg_trgm.control b/native-build/macos/vendor/pg-contrib/arm64/extension/pg_trgm.control similarity index 100% rename from macos/vendor/pg-contrib/arm64/extension/pg_trgm.control rename to native-build/macos/vendor/pg-contrib/arm64/extension/pg_trgm.control diff --git a/macos/vendor/pg-contrib/arm64/extension/unaccent--1.0--1.1.sql b/native-build/macos/vendor/pg-contrib/arm64/extension/unaccent--1.0--1.1.sql similarity index 100% rename from macos/vendor/pg-contrib/arm64/extension/unaccent--1.0--1.1.sql rename to native-build/macos/vendor/pg-contrib/arm64/extension/unaccent--1.0--1.1.sql diff --git a/macos/vendor/pg-contrib/arm64/extension/unaccent--1.1.sql b/native-build/macos/vendor/pg-contrib/arm64/extension/unaccent--1.1.sql similarity index 100% rename from macos/vendor/pg-contrib/arm64/extension/unaccent--1.1.sql rename to native-build/macos/vendor/pg-contrib/arm64/extension/unaccent--1.1.sql diff --git a/macos/vendor/pg-contrib/arm64/extension/unaccent.control b/native-build/macos/vendor/pg-contrib/arm64/extension/unaccent.control similarity index 100% rename from macos/vendor/pg-contrib/arm64/extension/unaccent.control rename to native-build/macos/vendor/pg-contrib/arm64/extension/unaccent.control diff --git a/macos/vendor/pg-contrib/arm64/lib/pg_trgm.dylib b/native-build/macos/vendor/pg-contrib/arm64/lib/pg_trgm.dylib similarity index 100% rename from macos/vendor/pg-contrib/arm64/lib/pg_trgm.dylib rename to native-build/macos/vendor/pg-contrib/arm64/lib/pg_trgm.dylib diff --git a/macos/vendor/pg-contrib/arm64/lib/unaccent.dylib b/native-build/macos/vendor/pg-contrib/arm64/lib/unaccent.dylib similarity index 100% rename from macos/vendor/pg-contrib/arm64/lib/unaccent.dylib rename to native-build/macos/vendor/pg-contrib/arm64/lib/unaccent.dylib diff --git a/macos/vendor/pg-contrib/arm64/tsearch_data/unaccent.rules b/native-build/macos/vendor/pg-contrib/arm64/tsearch_data/unaccent.rules similarity index 100% rename from macos/vendor/pg-contrib/arm64/tsearch_data/unaccent.rules rename to native-build/macos/vendor/pg-contrib/arm64/tsearch_data/unaccent.rules diff --git a/macos/vendor/redis/arm64/redis-server b/native-build/macos/vendor/redis/arm64/redis-server similarity index 100% rename from macos/vendor/redis/arm64/redis-server rename to native-build/macos/vendor/redis/arm64/redis-server diff --git a/native-build/windows/__init__.py b/native-build/windows/__init__.py new file mode 100644 index 00000000..ec50a564 --- /dev/null +++ b/native-build/windows/__init__.py @@ -0,0 +1,4 @@ +"""Windows standalone build package. + +See native-build/windows/README.md for build instructions. +""" diff --git a/windows/assets/AudioMuse-AI.ico b/native-build/windows/assets/AudioMuse-AI.ico similarity index 100% rename from windows/assets/AudioMuse-AI.ico rename to native-build/windows/assets/AudioMuse-AI.ico diff --git a/windows/control_server.py b/native-build/windows/control_server.py similarity index 95% rename from windows/control_server.py rename to native-build/windows/control_server.py index 734e328a..eac8a0b1 100644 --- a/windows/control_server.py +++ b/native-build/windows/control_server.py @@ -1,6 +1,6 @@ """TCP control server for the standalone Windows supervisor. -Replaces the Unix-domain-socket ``macos/control_ipc.py`` on Windows, where +Replaces the Unix-domain-socket ``native-build/macos/control_ipc.py`` on Windows, where ``AF_UNIX`` is not available. The protocol is identical (JSON line → response), just the transport is TCP on localhost. @@ -72,7 +72,7 @@ def _handle(self, conn, addr): conn.sendall(b"HTTP/1.1 200 OK\r\nContent-Type: text/plain\r\n\r\nstopping") return - # JSON control protocol (same as macos/control_ipc.py). + # JSON control protocol (same as native-build/macos/control_ipc.py). request = json.loads(text) ok = bool(self._dispatch(request.get("action", ""), request.get("services", []))) conn.sendall(b"ok" if ok else b"error") diff --git a/windows/db_backend.py b/native-build/windows/db_backend.py similarity index 100% rename from windows/db_backend.py rename to native-build/windows/db_backend.py diff --git a/windows/download_models.bat b/native-build/windows/download_models.bat similarity index 100% rename from windows/download_models.bat rename to native-build/windows/download_models.bat diff --git a/windows/embedded_pg.py b/native-build/windows/embedded_pg.py similarity index 100% rename from windows/embedded_pg.py rename to native-build/windows/embedded_pg.py diff --git a/windows/env.py b/native-build/windows/env.py similarity index 97% rename from windows/env.py rename to native-build/windows/env.py index 7e304d13..af17ea7d 100644 --- a/windows/env.py +++ b/native-build/windows/env.py @@ -1,6 +1,6 @@ """Build the environment handed to each supervised child process (Windows build). -Windows counterpart of ``linux/env.py``. The standalone-mode overrides are +Windows counterpart of ``native-build/linux/env.py``. The standalone-mode overrides are centralized here so every child imports ``config`` already pointed at the embedded services and the bundled models. diff --git a/windows/launcher.py b/native-build/windows/launcher.py similarity index 99% rename from windows/launcher.py rename to native-build/windows/launcher.py index 1b7b4797..0203027f 100644 --- a/windows/launcher.py +++ b/native-build/windows/launcher.py @@ -1,6 +1,6 @@ """Standalone Windows entry point (the PyInstaller entry script). -Windows counterpart of ``linux/launcher.py``. There is no native menu-bar agent +Windows counterpart of ``native-build/linux/launcher.py``. There is no native menu-bar agent here (rumps/AppKit are macOS-only); instead the frozen binary is a small multi-call launcher: diff --git a/windows/paths.py b/native-build/windows/paths.py similarity index 97% rename from windows/paths.py rename to native-build/windows/paths.py index 702853f4..0e8de69e 100644 --- a/windows/paths.py +++ b/native-build/windows/paths.py @@ -1,6 +1,6 @@ """Filesystem locations for the standalone Windows build. -Mirrors ``linux/paths.py`` but uses Windows conventions (``%LOCALAPPDATA%``) +Mirrors ``native-build/linux/paths.py`` but uses Windows conventions (``%LOCALAPPDATA%``) instead of XDG directories. Read-only resources (ONNX models, Flask templates/static, the bundled @@ -37,7 +37,7 @@ def resource_root(): def _repo_root(): - return os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + return os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) def tray_icon(): diff --git a/windows/supervisor.py b/native-build/windows/supervisor.py similarity index 99% rename from windows/supervisor.py rename to native-build/windows/supervisor.py index 81819d08..498012c0 100644 --- a/windows/supervisor.py +++ b/native-build/windows/supervisor.py @@ -1,6 +1,6 @@ """Process supervisor for the standalone Windows app. -Windows counterpart of ``linux/supervisor.py`` -- the logic is platform-agnostic +Windows counterpart of ``native-build/linux/supervisor.py`` -- the logic is platform-agnostic (stdlib ``subprocess``/``signal``/``os.kill`` + ``psutil``), so this is a near copy that swaps ``linux`` path/env helpers for the ``windows`` ones and adapts the control server to use a TCP socket instead of a Unix socket (Windows has no diff --git a/windows/vendor/build-redis.bat b/native-build/windows/vendor/build-redis.bat similarity index 97% rename from windows/vendor/build-redis.bat rename to native-build/windows/vendor/build-redis.bat index f5f107eb..e50f525a 100644 --- a/windows/vendor/build-redis.bat +++ b/native-build/windows/vendor/build-redis.bat @@ -11,7 +11,7 @@ setlocal enabledelayedexpansion set "ARCH=amd64" if "%PROCESSOR_ARCHITECTURE%"=="ARM64" set "ARCH=arm64" -set "DEST=windows\vendor\redis\%ARCH%" +set "DEST=native-build\windows\vendor\redis\%ARCH%" set "REDIS_VERSION=5.0.14.1" set "ZIP=Redis-x64-%REDIS_VERSION%.zip" set "URL=https://github.com/tporadowski/redis/releases/download/v%REDIS_VERSION%/%ZIP%" diff --git a/windows/vendor/pg-contrib/amd64/extension/pg_trgm--1.0--1.1.sql b/native-build/windows/vendor/pg-contrib/amd64/extension/pg_trgm--1.0--1.1.sql similarity index 100% rename from windows/vendor/pg-contrib/amd64/extension/pg_trgm--1.0--1.1.sql rename to native-build/windows/vendor/pg-contrib/amd64/extension/pg_trgm--1.0--1.1.sql diff --git a/windows/vendor/pg-contrib/amd64/extension/pg_trgm--1.1--1.2.sql b/native-build/windows/vendor/pg-contrib/amd64/extension/pg_trgm--1.1--1.2.sql similarity index 100% rename from windows/vendor/pg-contrib/amd64/extension/pg_trgm--1.1--1.2.sql rename to native-build/windows/vendor/pg-contrib/amd64/extension/pg_trgm--1.1--1.2.sql diff --git a/windows/vendor/pg-contrib/amd64/extension/pg_trgm--1.2--1.3.sql b/native-build/windows/vendor/pg-contrib/amd64/extension/pg_trgm--1.2--1.3.sql similarity index 100% rename from windows/vendor/pg-contrib/amd64/extension/pg_trgm--1.2--1.3.sql rename to native-build/windows/vendor/pg-contrib/amd64/extension/pg_trgm--1.2--1.3.sql diff --git a/windows/vendor/pg-contrib/amd64/extension/pg_trgm--1.3--1.4.sql b/native-build/windows/vendor/pg-contrib/amd64/extension/pg_trgm--1.3--1.4.sql similarity index 100% rename from windows/vendor/pg-contrib/amd64/extension/pg_trgm--1.3--1.4.sql rename to native-build/windows/vendor/pg-contrib/amd64/extension/pg_trgm--1.3--1.4.sql diff --git a/windows/vendor/pg-contrib/amd64/extension/pg_trgm--1.3.sql b/native-build/windows/vendor/pg-contrib/amd64/extension/pg_trgm--1.3.sql similarity index 100% rename from windows/vendor/pg-contrib/amd64/extension/pg_trgm--1.3.sql rename to native-build/windows/vendor/pg-contrib/amd64/extension/pg_trgm--1.3.sql diff --git a/windows/vendor/pg-contrib/amd64/extension/pg_trgm--1.4--1.5.sql b/native-build/windows/vendor/pg-contrib/amd64/extension/pg_trgm--1.4--1.5.sql similarity index 100% rename from windows/vendor/pg-contrib/amd64/extension/pg_trgm--1.4--1.5.sql rename to native-build/windows/vendor/pg-contrib/amd64/extension/pg_trgm--1.4--1.5.sql diff --git a/windows/vendor/pg-contrib/amd64/extension/pg_trgm--1.5--1.6.sql b/native-build/windows/vendor/pg-contrib/amd64/extension/pg_trgm--1.5--1.6.sql similarity index 100% rename from windows/vendor/pg-contrib/amd64/extension/pg_trgm--1.5--1.6.sql rename to native-build/windows/vendor/pg-contrib/amd64/extension/pg_trgm--1.5--1.6.sql diff --git a/windows/vendor/pg-contrib/amd64/extension/pg_trgm.control b/native-build/windows/vendor/pg-contrib/amd64/extension/pg_trgm.control similarity index 100% rename from windows/vendor/pg-contrib/amd64/extension/pg_trgm.control rename to native-build/windows/vendor/pg-contrib/amd64/extension/pg_trgm.control diff --git a/windows/vendor/pg-contrib/amd64/extension/unaccent--1.0--1.1.sql b/native-build/windows/vendor/pg-contrib/amd64/extension/unaccent--1.0--1.1.sql similarity index 100% rename from windows/vendor/pg-contrib/amd64/extension/unaccent--1.0--1.1.sql rename to native-build/windows/vendor/pg-contrib/amd64/extension/unaccent--1.0--1.1.sql diff --git a/windows/vendor/pg-contrib/amd64/extension/unaccent--1.1.sql b/native-build/windows/vendor/pg-contrib/amd64/extension/unaccent--1.1.sql similarity index 100% rename from windows/vendor/pg-contrib/amd64/extension/unaccent--1.1.sql rename to native-build/windows/vendor/pg-contrib/amd64/extension/unaccent--1.1.sql diff --git a/windows/vendor/pg-contrib/amd64/extension/unaccent.control b/native-build/windows/vendor/pg-contrib/amd64/extension/unaccent.control similarity index 100% rename from windows/vendor/pg-contrib/amd64/extension/unaccent.control rename to native-build/windows/vendor/pg-contrib/amd64/extension/unaccent.control diff --git a/windows/vendor/pg-contrib/amd64/lib/pg_trgm.dll b/native-build/windows/vendor/pg-contrib/amd64/lib/pg_trgm.dll similarity index 100% rename from windows/vendor/pg-contrib/amd64/lib/pg_trgm.dll rename to native-build/windows/vendor/pg-contrib/amd64/lib/pg_trgm.dll diff --git a/windows/vendor/pg-contrib/amd64/lib/unaccent.dll b/native-build/windows/vendor/pg-contrib/amd64/lib/unaccent.dll similarity index 100% rename from windows/vendor/pg-contrib/amd64/lib/unaccent.dll rename to native-build/windows/vendor/pg-contrib/amd64/lib/unaccent.dll diff --git a/windows/vendor/pg-contrib/amd64/tsearch_data/unaccent.rules b/native-build/windows/vendor/pg-contrib/amd64/tsearch_data/unaccent.rules similarity index 100% rename from windows/vendor/pg-contrib/amd64/tsearch_data/unaccent.rules rename to native-build/windows/vendor/pg-contrib/amd64/tsearch_data/unaccent.rules diff --git a/windows/vendor/pg-contrib/build-pg-contrib-cross.sh b/native-build/windows/vendor/pg-contrib/build-pg-contrib-cross.sh similarity index 90% rename from windows/vendor/pg-contrib/build-pg-contrib-cross.sh rename to native-build/windows/vendor/pg-contrib/build-pg-contrib-cross.sh index 0fc7e6fd..651e8c25 100644 --- a/windows/vendor/pg-contrib/build-pg-contrib-cross.sh +++ b/native-build/windows/vendor/pg-contrib/build-pg-contrib-cross.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -# Regenerate windows/vendor/pg-contrib// : run on Linux with gcc-mingw-w64-x86-64 + python3 + curl. +# Regenerate native-build/windows/vendor/pg-contrib// : run on Linux with gcc-mingw-w64-x86-64 + python3 + curl. set -euo pipefail ARCH="amd64" @@ -7,8 +7,8 @@ PG_VERSION="16.2" PGSERVER_VERSION="0.1.4" SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -REPO_ROOT="$(cd "${SCRIPT_DIR}/../../.." && pwd)" -DEST="${DEST:-${REPO_ROOT}/windows/vendor/pg-contrib/${ARCH}}" +REPO_ROOT="$(cd "${SCRIPT_DIR}/../../../.." && pwd)" +DEST="${DEST:-${REPO_ROOT}/native-build/windows/vendor/pg-contrib/${ARCH}}" command -v x86_64-w64-mingw32-gcc >/dev/null || { echo "Need gcc-mingw-w64-x86-64 (apt install gcc-mingw-w64-x86-64)"; exit 1; } diff --git a/numeric_bootstrap.py b/numeric_bootstrap.py index 6db8461c..6ccb5020 100644 --- a/numeric_bootstrap.py +++ b/numeric_bootstrap.py @@ -15,9 +15,9 @@ locale transition there is to race against. It is kept narrow -- only the numeric category, so ``LC_CTYPE``/UTF-8 (accented file paths) is untouched. -This module is imported ONLY by the macOS launcher (``macos/launcher.py``); it is +This module is imported ONLY by the macOS launcher (``native-build/macos/launcher.py``); it is never referenced by the Linux/Docker worker entrypoints, so the container images -are entirely unaffected. The matching env-level pin lives in ``macos/env.py``. +are entirely unaffected. The matching env-level pin lives in ``native-build/macos/env.py``. """ import os diff --git a/query/CLAMP3/clamp3_search_demo.py b/query/CLAMP3/clamp3_search_demo.py deleted file mode 100644 index 1d59fcf8..00000000 --- a/query/CLAMP3/clamp3_search_demo.py +++ /dev/null @@ -1,557 +0,0 @@ -#!/usr/bin/env python3 - -# source .venv/bin/activate -# pip install -r requirements.txt -""" -CLAMP3 Audio Search Demo -Standalone script for analyzing audio files and searching with natural language queries. -Uses CLAMP3 model (SAAS version) for audio-text retrieval. -""" - -print("Starting CLAMP3 demo script...") -print("Importing dependencies...") - -import os -import sys -import numpy as np -from pathlib import Path -import torch -import torch.nn as nn -from transformers import AutoTokenizer, AutoModel, BertConfig, Wav2Vec2FeatureExtractor -from tqdm import tqdm -import librosa -import warnings -warnings.filterwarnings('ignore') - -print("✓ All imports successful!") - - -# ============================================================================= -# CLAMP3 Configuration (matching the SAAS model) -# ============================================================================= - -TEXT_MODEL_NAME = "FacebookAI/xlm-roberta-base" -CLAMP3_HIDDEN_SIZE = 768 -AUDIO_HIDDEN_SIZE = 768 -AUDIO_NUM_LAYERS = 12 -MAX_AUDIO_LENGTH = 128 -MAX_TEXT_LENGTH = 128 - - -# ============================================================================= -# Simplified CLaMP3 Model (inference only, no M3/symbolic encoder needed) -# ============================================================================= - -class CLaMP3AudioEncoder(nn.Module): - """Simplified CLAMP3 model for audio-text retrieval.""" - - def __init__(self, weights_path): - super().__init__() - - # Load text model (XLM-RoBERTa) - print("Loading text encoder (XLM-RoBERTa)...") - print(" (This may take a few minutes on first run - downloading from HuggingFace)") - self.text_model = AutoModel.from_pretrained(TEXT_MODEL_NAME) - self.text_proj = nn.Linear(self.text_model.config.hidden_size, CLAMP3_HIDDEN_SIZE) - - # Load audio model (BERT-style transformer for MERT features) - print("Loading audio encoder...") - audio_config = BertConfig( - vocab_size=1, - hidden_size=AUDIO_HIDDEN_SIZE, - num_hidden_layers=AUDIO_NUM_LAYERS, - num_attention_heads=AUDIO_HIDDEN_SIZE // 64, - intermediate_size=AUDIO_HIDDEN_SIZE * 4, - max_position_embeddings=MAX_AUDIO_LENGTH - ) - from transformers import BertModel - self.audio_model = BertModel(audio_config) - self.audio_proj = nn.Linear(audio_config.hidden_size, CLAMP3_HIDDEN_SIZE) - - # Load checkpoint - print(f"Loading CLAMP3 weights from: {weights_path}") - checkpoint = torch.load(weights_path, map_location='cpu') - - # Load state dict - state_dict = checkpoint.get('model', checkpoint) - - # Filter and load only audio and text components - text_model_dict = {k.replace('text_model.', ''): v for k, v in state_dict.items() if k.startswith('text_model.')} - text_proj_dict = {k.replace('text_proj.', ''): v for k, v in state_dict.items() if k.startswith('text_proj.')} - audio_model_dict = {k.replace('audio_model.', ''): v for k, v in state_dict.items() if k.startswith('audio_model.')} - audio_proj_dict = {k.replace('audio_proj.', ''): v for k, v in state_dict.items() if k.startswith('audio_proj.')} - - print(f" - Text model params: {len(text_model_dict)}") - print(f" - Text proj params: {len(text_proj_dict)}") - print(f" - Audio model params: {len(audio_model_dict)}") - print(f" - Audio proj params: {len(audio_proj_dict)}") - - if len(audio_model_dict) == 0 or len(text_model_dict) == 0: - print(" ⚠️ WARNING: Model weights appear to be missing or incorrectly formatted!") - print(" Available keys in checkpoint:", list(state_dict.keys())[:10]) - - self.text_model.load_state_dict(text_model_dict, strict=False) - self.text_proj.load_state_dict(text_proj_dict) - self.audio_model.load_state_dict(audio_model_dict) - self.audio_proj.load_state_dict(audio_proj_dict) - - self.eval() - print("✓ CLAMP3 model loaded successfully") - - def avg_pooling(self, features, masks): - """Average pooling with mask.""" - masks = masks.unsqueeze(-1) # (batch, seq, 1) - features = features * masks - return features.sum(dim=1) / masks.sum(dim=1) - - def get_text_embedding(self, text_inputs, text_masks): - """Get text embedding from text.""" - with torch.no_grad(): - text_features = self.text_model(text_inputs, attention_mask=text_masks)['last_hidden_state'] - text_features = self.avg_pooling(text_features, text_masks) - text_features = self.text_proj(text_features) - # Normalize - text_features = text_features / text_features.norm(dim=-1, keepdim=True) - return text_features - - def get_audio_embedding(self, audio_inputs, audio_masks): - """Get audio embedding from MERT features.""" - with torch.no_grad(): - audio_features = self.audio_model(inputs_embeds=audio_inputs, attention_mask=audio_masks)['last_hidden_state'] - audio_features = self.avg_pooling(audio_features, audio_masks) - audio_features = self.audio_proj(audio_features) - # Normalize - audio_features = audio_features / audio_features.norm(dim=-1, keepdim=True) - return audio_features - - -# ============================================================================= -# MERT Feature Extractor (with on-the-fly extraction) -# ============================================================================= - -class MERTFeatureExtractor: - """MERT feature extractor for audio preprocessing.""" - - def __init__(self, model_name='m-a-p/MERT-v1-95M', device='cpu', - window_size=10, overlap_percent=50): - """Initialize MERT model for feature extraction. - - Args: - model_name: HuggingFace model name - device: cpu or cuda - window_size: Window size in seconds (default: 10) - overlap_percent: Overlap percentage 0-100 (default: 50) - """ - self.device = torch.device(device) - self.target_sr = 24000 - self.window_size = window_size # seconds - self.overlap_percent = overlap_percent - - print(f"Loading MERT model: {model_name}") - print(f" Window: {window_size}s with {overlap_percent}% overlap") - print(" (This may take a few minutes on first run - downloading ~400MB)") - from transformers import AutoModel as HFAutoModel - self.model = HFAutoModel.from_pretrained(model_name, trust_remote_code=True) - self.model = self.model.to(self.device) - self.model.eval() - - # Load MERT processor (Wav2Vec2 feature extractor) for proper preprocessing - self.processor = Wav2Vec2FeatureExtractor( - feature_size=1, - sampling_rate=self.target_sr, - padding_value=0.0, - return_attention_mask=True, - do_normalize=True, - ) - print("✓ MERT model loaded") - - def load_audio(self, audio_path): - """Load audio file and resample to target sample rate.""" - try: - # Use librosa for MP3 compatibility (no torchcodec required) - waveform, sr = librosa.load(audio_path, sr=self.target_sr, mono=True) - - # Convert to torch tensor - waveform = torch.from_numpy(waveform).float() - - return waveform - except Exception as e: - print(f"Error loading audio {audio_path}: {e}") - return None - - def extract_features(self, audio_path): - """Extract MERT features from audio file. - - Returns: (num_windows, hidden_dim) array - one vector per window - """ - # Load audio - waveform = self.load_audio(audio_path) - if waveform is None: - return None - - # Get audio duration for debugging - duration_sec = len(waveform) / self.target_sr - - # Process through Wav2Vec2FeatureExtractor for normalization - processed_wav = self.processor(waveform.numpy(), return_tensors="pt", - sampling_rate=self.target_sr).input_values[0] - processed_wav = processed_wav.to(self.device) - - # Calculate stride based on overlap - window_samples = int(self.target_sr * self.window_size) - stride_samples = int(window_samples * (1 - self.overlap_percent / 100)) - - all_features = [] - - print(f" Audio: {duration_sec:.1f}s | Window: {self.window_size}s | Overlap: {self.overlap_percent}%", end=" ") - - with torch.no_grad(): - for start in range(0, processed_wav.shape[-1] - window_samples + 1, stride_samples): - chunk = processed_wav[start:start + window_samples] - chunk = chunk.unsqueeze(0) # Add batch dimension - - # Get all layers from MERT - outputs = self.model(chunk, output_hidden_states=True) - hidden_states = outputs.hidden_states # Tuple of (1, seq_len, hidden) per layer - - # Stack layers: (num_layers, 1, seq_len, hidden) - hidden_states = torch.stack(hidden_states) - - # Average across TIME and LAYERS to get ONE vector per window - features = hidden_states.mean(dim=2) # Average time: (num_layers, 1, hidden) - features = features.mean(dim=0) # Average layers: (1, hidden) - - all_features.append(features.squeeze(0).cpu()) - - if len(all_features) == 0: - return None - - # Stack: (num_windows, hidden_dim) - all_features = torch.stack(all_features) - print(f"→ {len(all_features)} windows") - - return all_features.numpy() - - -def extract_mert_features_simple(audio_path, mert_extractor=None): - """ - Load pre-extracted MERT features (.npy files) or extract on-the-fly. - - Returns: (num_windows, hidden_dim) - multiple windows per song (NOT averaged) - """ - npy_path = Path(audio_path).with_suffix('.npy') - - # Try to load pre-extracted features first - if npy_path.exists(): - features = np.load(npy_path) - features = torch.tensor(features).float() - if features.ndim == 3: - features = features.squeeze(0) - if features.ndim == 1: - features = features.unsqueeze(0) - return features - - # Extract features on-the-fly if MERT extractor is provided - if mert_extractor is not None: - features = mert_extractor.extract_features(audio_path) - if features is not None: - # Save multi-window features (NOT averaged) - npy_path.parent.mkdir(parents=True, exist_ok=True) - np.save(npy_path, features) - return torch.tensor(features).float() - - -# ============================================================================= -# CLAMP3 Search System -# ============================================================================= - -class CLAMP3Searcher: - """CLAMP3-based audio search system.""" - - def __init__(self, model_path, device='cpu', extract_mert_on_fly=True): - """Initialize CLAMP3 model.""" - self.device = torch.device(device) - self.model = CLaMP3AudioEncoder(model_path).to(self.device) - self.tokenizer = AutoTokenizer.from_pretrained(TEXT_MODEL_NAME) - - # Initialize MERT extractor if needed - self.mert_extractor = None - if extract_mert_on_fly: - print("\nInitializing MERT feature extractor...") - self.mert_extractor = MERTFeatureExtractor(device=device) - - # Cache for audio embeddings - self.audio_embeddings = [] - self.audio_files = [] - - def get_audio_embedding(self, audio_features): - """Get CLAMP3 embedding for audio (from MERT features). - - Args: - audio_features: (num_windows, hidden_dim) tensor from MERT - - Returns: - Single embedding vector for the whole audio - """ - if audio_features is None: - return None - - # Add zero vectors at start and end (as in training) - zero_vec = torch.zeros((1, audio_features.size(-1))) - audio_features = torch.cat((zero_vec, audio_features, zero_vec), 0) - - total_length = audio_features.size(0) - - # If features fit in MAX_AUDIO_LENGTH, process at once - if total_length <= MAX_AUDIO_LENGTH: - # Pad to MAX_AUDIO_LENGTH - pad_len = MAX_AUDIO_LENGTH - total_length - if pad_len > 0: - pad = torch.zeros((pad_len, audio_features.size(-1))) - audio_features_padded = torch.cat((audio_features, pad), 0) - else: - audio_features_padded = audio_features - - # Create mask for actual content - mask = torch.zeros(MAX_AUDIO_LENGTH) - mask[:total_length] = 1 - - # Get embedding from CLAMP3 - audio_features_padded = audio_features_padded.unsqueeze(0).to(self.device) - mask = mask.unsqueeze(0).to(self.device) - - embedding = self.model.get_audio_embedding(audio_features_padded, mask) - return embedding.cpu().numpy().flatten() - - else: - # Song too long: split into chunks of MAX_AUDIO_LENGTH, get embeddings, average - embeddings_list = [] - - for start_idx in range(0, total_length, MAX_AUDIO_LENGTH): - end_idx = min(start_idx + MAX_AUDIO_LENGTH, total_length) - chunk = audio_features[start_idx:end_idx] - - # Pad chunk - pad_len = MAX_AUDIO_LENGTH - chunk.size(0) - if pad_len > 0: - pad = torch.zeros((pad_len, chunk.size(-1))) - chunk = torch.cat((chunk, pad), 0) - - # Create mask - mask = torch.zeros(MAX_AUDIO_LENGTH) - mask[:end_idx - start_idx] = 1 - - # Get embedding - chunk = chunk.unsqueeze(0).to(self.device) - mask = mask.unsqueeze(0).to(self.device) - - embedding = self.model.get_audio_embedding(chunk, mask) - embeddings_list.append(embedding) - - # Average all chunk embeddings - final_embedding = torch.stack(embeddings_list).mean(dim=0) - return final_embedding.cpu().numpy().flatten() - - def get_text_embedding(self, query_text): - """Get CLAMP3 embedding for text query.""" - # Tokenize - inputs = self.tokenizer(query_text, return_tensors='pt', max_length=MAX_TEXT_LENGTH, - truncation=True, padding='max_length') - - text_inputs = inputs['input_ids'].to(self.device) - text_masks = inputs['attention_mask'].to(self.device) - - # Get embedding - embedding = self.model.get_text_embedding(text_inputs, text_masks) - return embedding.cpu().numpy().flatten() - - def analyze_folder(self, folder_path): - """Analyze all audio files in a folder.""" - folder = Path(folder_path) - - print(f"\nAnalyzing audio files in: {folder}") - print("=" * 70) - - # Look for both pre-extracted .npy files and raw audio files - npy_files = list(folder.glob('*.npy')) - audio_exts = ['.mp3', '.wav', '.flac', '.ogg', '.m4a'] - raw_audio = [] - for ext in audio_exts: - raw_audio.extend(folder.glob(f'*{ext}')) - - # Prioritize .npy files if they exist - if npy_files: - print(f"Found {len(npy_files)} pre-extracted MERT .npy file(s)") - audio_files = npy_files - process_as_npy = True - elif raw_audio: - if self.mert_extractor is None: - print(f"⚠️ Found {len(raw_audio)} audio file(s) but MERT extractor not initialized") - print(f" Re-run with extract_mert_on_fly=True to enable on-the-fly extraction") - return - print(f"Found {len(raw_audio)} audio file(s) - will extract MERT features on-the-fly") - audio_files = raw_audio - process_as_npy = False - else: - print(f"No audio files found in {folder}") - return - - print(f"Processing {len(audio_files)} file(s)...") - print() - - # Analyze each file - for audio_file in tqdm(sorted(audio_files), desc="Extracting embeddings"): - if process_as_npy: - # Load pre-extracted features - features = np.load(audio_file) - features = torch.tensor(features).float() - if features.ndim == 3: - features = features.squeeze(0) - else: - # Extract MERT features on-the-fly - features = extract_mert_features_simple(audio_file, self.mert_extractor) - - if features is not None: - embedding = self.get_audio_embedding(features) - if embedding is not None: - self.audio_embeddings.append(embedding) - self.audio_files.append(audio_file) - - print() - print(f"✓ Analyzed {len(self.audio_embeddings)} audio files successfully") - - def search(self, query_text, top_k=5): - """Search audio files using a text query.""" - if not self.audio_embeddings: - print("No audio files analyzed yet!") - return [] - - print(f"\n{'=' * 70}") - print(f"Query: \"{query_text}\"") - print(f"{'=' * 70}") - - # Get text embedding - text_embedding = self.get_text_embedding(query_text) - - # Compute similarities with all audio files - embeddings_matrix = np.vstack(self.audio_embeddings) - similarities = embeddings_matrix @ text_embedding - - # Show score statistics for better understanding - print(f"Score stats: min={similarities.min():.4f}, max={similarities.max():.4f}, " - f"mean={similarities.mean():.4f}, std={similarities.std():.4f}") - - # Get top-k results - top_indices = np.argsort(similarities)[::-1][:top_k] - - # Print results - results = [] - for rank, idx in enumerate(top_indices, 1): - similarity = similarities[idx] - audio_file = self.audio_files[idx] - - # Calculate normalized score (0-100 scale for easier interpretation) - score_range = similarities.max() - similarities.min() - normalized_score = ((similarity - similarities.min()) / score_range * 100) if score_range > 0 else 50 - - result = { - 'rank': rank, - 'file': audio_file.name, - 'path': str(audio_file), - 'similarity': float(similarity), - 'normalized_score': float(normalized_score) - } - results.append(result) - - print(f"{rank}. {audio_file.stem}") - print(f" Similarity: {similarity:.4f} | Normalized: {normalized_score:.1f}/100") - - return results - - -def main(): - """Main function.""" - # Paths - script_dir = Path(__file__).parent - model_path = script_dir.parent.parent / "CLAMP3" / "weights_clamp3_saas_h_size_768_t_model_FacebookAI_xlm-roberta-base_t_length_128_a_size_768_a_layers_12_a_length_128_s_size_768_s_layers_12_p_size_64_p_length_512.pth" - - # Check for MERT features folder - mert_folder = script_dir.parent.parent / "test" / "songs_mert" - audio_folder = script_dir.parent.parent / "test" / "songs" - - # Check if model exists - if not model_path.exists(): - print(f"ERROR: Model not found at {model_path}") - print(f"\nPlease download the CLAMP3 SAAS model from:") - print(f"https://huggingface.co/sander-wood/clamp3/blob/main/weights_clamp3_saas_h_size_768_t_model_FacebookAI_xlm-roberta-base_t_length_128_a_size_768_a_layers_12_a_length_128_s_size_768_s_layers_12_p_size_64_p_length_512.pth") - sys.exit(1) - - print("=" * 70) - print("CLAMP3 AUDIO SEARCH DEMO") - print("=" * 70) - print() - print("This demo uses the CLAMP3 SAAS model for audio-text retrieval.") - print() - print("Requirements:") - print(" 1. CLAMP3 SAAS weights (already found)") - print(" 2. MERT-preprocessed audio features (.npy files)") - print() - print("=" * 70) - print() - - # Detect CUDA - device = 'cuda' if torch.cuda.is_available() else 'cpu' - print(f"Using device: {device}") - print() - - # Initialize searcher (with on-the-fly MERT extraction enabled) - searcher = CLAMP3Searcher(str(model_path), device=device, extract_mert_on_fly=True) - - # Check if MERT features exist, otherwise process raw audio - if mert_folder.exists(): - print(f"✓ Found MERT features folder: {mert_folder}") - searcher.analyze_folder(mert_folder) - else: - print(f"⚠️ MERT features folder not found: {mert_folder}") - print(f" Will extract MERT features on-the-fly from audio files in: {audio_folder}") - - if audio_folder.exists(): - searcher.analyze_folder(audio_folder) - else: - print(f"⚠️ Audio folder not found: {audio_folder}") - sys.exit(1) - - # Search queries - queries = [ - "relax ukulele song", - "calm piano song", - "Hard rock song", - "slash bass song", - "Classic music", - "happy pop song", - "Jazz song", - "POP song with Female Vocalist" - ] - - if len(searcher.audio_embeddings) > 0: - print("\n" + "=" * 70) - print("SEARCHING WITH TEXT QUERIES") - print("=" * 70) - - all_results = {} - for query in queries: - results = searcher.search(query, top_k=3) - all_results[query] = results - print() # Add spacing between queries - - # Summary - print("\n" + "=" * 70) - print("SEARCH COMPLETE") - print("=" * 70) - print(f"Analyzed: {len(searcher.audio_files)} audio files") - print(f"Queries: {len(queries)}") - print() - - -if __name__ == "__main__": - main() diff --git a/query/CLAMP3/requirements.txt b/query/CLAMP3/requirements.txt deleted file mode 100644 index 47e9cd1b..00000000 --- a/query/CLAMP3/requirements.txt +++ /dev/null @@ -1,6 +0,0 @@ -numpy -torch -transformers -tqdm -accelerate -librosa diff --git a/query/throwaway_lyrics_search.py b/query/throwaway_lyrics_search.py deleted file mode 100644 index c720372f..00000000 --- a/query/throwaway_lyrics_search.py +++ /dev/null @@ -1,96 +0,0 @@ -"""Throwaway sanity check for the gte-multilingual lyrics embedding. - -Invents 5 short songs in 5 languages (3 about love, 2 about war), embeds them -with lyrics/gte_onnx.py, then ranks them by cosine similarity against the query -word "love". Multilingual love songs should float to the top regardless of -language; war songs should sink. - -Run from the repo root with the venv active: - source .venv/bin/activate - python throwaway_lyrics_search.py -""" - -from __future__ import annotations - -import logging -import os - -os.environ.setdefault('LYRICS_GTE_ONNX_PATH', 'model/gte-multilingual-base-int8.onnx') -os.environ.setdefault('LYRICS_GTE_TOKENIZER_DIR', 'model/gte-multilingual-base') - -logging.basicConfig(level=logging.INFO, format='%(levelname)s %(name)s: %(message)s') -log = logging.getLogger('lyrics-search-demo') - -import numpy as np - -from lyrics.gte_onnx import embed_text - -SONGS = { - 'Italian - Love': ( - "Ti amo come il sole ama il mattino, " - "il tuo nome è una carezza sul mio cuore. " - "Resta con me per sempre, amore mio, " - "le tue mani sono la mia casa." - ), - 'English - Love': ( - "I love you more than words can ever say, " - "your gentle heart is the home where I belong. " - "Hold me close and never let me go, " - "our love is a tender, endless song." - ), - 'Spanish - Love': ( - "Te amo con toda mi alma y mi corazón, " - "tu mirada es la luz de mi mañana. " - "Eres mi amor, mi vida y mi canción, " - "quédate conmigo cada semana." - ), - 'France - War': ( - "Les canons grondent sur le champ de bataille, " - "les soldats tombent sous la fumée et le feu. " - "La guerre dévore les villes en ruine, " - "le sang coule sur la terre de Dieu." - ), - 'Chinese - War': ( - "战争的硝烟笼罩着大地，" - "士兵在炮火中倒下流血。" - "城市化为废墟，钢铁与火焰， " - "战鼓敲响，敌人在前线厮杀。" - ), -} - -QUERIES = ['love', 'war', 'dog', 'amore', 'amor', '爱', 'guerra', '战争'] - - -def _search(query: str, song_vecs: dict) -> None: - qvec = embed_text(query) - ranked = sorted( - ((name, float(np.dot(qvec, vec))) for name, vec in song_vecs.items()), - key=lambda kv: kv[1], - reverse=True, - ) - log.info('=' * 56) - log.info('SEARCH RESULTS for query %r (cosine, higher = closer):', query) - log.info('%-6s %-18s %-8s %s', 'rank', 'song', 'score', 'theme') - log.info('-' * 56) - for i, (name, score) in enumerate(ranked, start=1): - theme = 'LOVE' if name.endswith('Love') else 'WAR' - log.info('%-6d %-18s %.4f %s', i, name, score, theme) - log.info('=' * 56) - - -def main() -> None: - log.info('Embedding %d songs...', len(SONGS)) - song_vecs = {} - for name, lyric in SONGS.items(): - vec = embed_text(lyric) - song_vecs[name] = vec - log.info('embedded %-16s dim=%s norm=%.4f', name, vec.shape[0], - float(np.linalg.norm(vec))) - - for query in QUERIES: - log.info('Embedding query word: %r', query) - _search(query, song_vecs) - - -if __name__ == '__main__': - main() diff --git a/requirements/common-noavx2.txt b/requirements/common-noavx2.txt index 951b315a..971150ea 100644 --- a/requirements/common-noavx2.txt +++ b/requirements/common-noavx2.txt @@ -19,7 +19,6 @@ google-genai==1.57.0 mistralai>=1.11.1,<2.0.0 umap-learn av==13.1.0 -python-mpd2 psutil PyJWT==2.12.1 argon2-cffi==25.1.0 diff --git a/requirements/common.txt b/requirements/common.txt index 4b8263ce..4675430e 100644 --- a/requirements/common.txt +++ b/requirements/common.txt @@ -19,7 +19,6 @@ google-genai==1.57.0 mistralai>=1.11.1,<2.0.0 umap-learn==0.5.12 av==13.1.0 -python-mpd2==3.1.1 psutil==7.2.2 onnx==1.20.0 resampy==0.4.3 diff --git a/requirements/linux.txt b/requirements/linux.txt index e59de990..96df0050 100644 --- a/requirements/linux.txt +++ b/requirements/linux.txt @@ -7,11 +7,11 @@ onnxruntime==1.19.2 # pgserver ships a prebuilt PostgreSQL; 0.1.4 bundles PostgreSQL 16.2. Pinned # because contrib modules are NOT ABI-stable across PG minors -- the vendored -# unaccent/pg_trgm in linux/vendor/pg-contrib/ are compiled against 16.2 and a +# unaccent/pg_trgm in native-build/linux/vendor/pg-contrib/ are compiled against 16.2 and a # pgserver bump to a newer PG would break them at load time (see that README). # # x86_64 ONLY: pgserver publishes no Linux/aarch64 wheel, so on arm64 we bundle a -# from-source PostgreSQL instead (linux/vendor/postgres/build-postgres.sh) and +# from-source PostgreSQL instead (native-build/linux/vendor/postgres/build-postgres.sh) and # this requirement is skipped. The marker makes `pip install -r` succeed on both. pgserver==0.1.4; platform_machine == "x86_64" diff --git a/requirements/macos.txt b/requirements/macos.txt index c49ab72a..88f2092c 100644 --- a/requirements/macos.txt +++ b/requirements/macos.txt @@ -7,7 +7,7 @@ onnxruntime==1.19.2 # pgserver ships a prebuilt PostgreSQL; 0.1.4 bundles PostgreSQL 16.2. Pinned # because contrib modules are NOT ABI-stable across PG minors -- the vendored -# unaccent/pg_trgm in macos/vendor/pg-contrib/ are compiled against 16.2 and a +# unaccent/pg_trgm in native-build/macos/vendor/pg-contrib/ are compiled against 16.2 and a # pgserver bump to a newer PG would break them at load time (see that README). pgserver==0.1.4 rumps diff --git a/requirements/windows.txt b/requirements/windows.txt index 2ed1e027..fd5b1dbb 100644 --- a/requirements/windows.txt +++ b/requirements/windows.txt @@ -6,7 +6,7 @@ onnxruntime==1.19.2 # pgserver ships a prebuilt PostgreSQL; 0.1.4 bundles PostgreSQL 16.2. Pinned # because contrib modules are NOT ABI-stable across PG minors -- the vendored -# unaccent/pg_trgm in windows/vendor/pg-contrib/ are compiled against 16.2. +# unaccent/pg_trgm in native-build/windows/vendor/pg-contrib/ are compiled against 16.2. pgserver==0.1.4 # Served by waitress (same WSGI server the macOS/Linux builds use); not in common.txt. diff --git a/rq_janitor.py b/rq_janitor.py index 299212f7..411558ff 100644 --- a/rq_janitor.py +++ b/rq_janitor.py @@ -7,7 +7,7 @@ try: # We need the queue objects to get their registries - from app_helper import redis_conn, rq_queue_high, rq_queue_default + from app_helper import rq_queue_high, rq_queue_default from app_logging import configure_logging except ImportError as e: print(f"Error importing from app.py: {e}") diff --git a/scripts/standalone/config.py b/scripts/standalone/config.py index 26b52db7..f6347947 100644 --- a/scripts/standalone/config.py +++ b/scripts/standalone/config.py @@ -26,36 +26,36 @@ def read_app_version(root): PLATFORMS = { "windows": { - "launcher": "windows/launcher.py", - "vendor_dir": "windows/vendor", + "launcher": "native-build/windows/launcher.py", + "vendor_dir": "native-build/windows/vendor", "redis_bin": "redis-server.exe", "pg_contrib_glob": "*.dll", "initdb_bin": "initdb.exe", "use_pgserver": "always", "console": True, - "exe_icon": "windows/assets/AudioMuse-AI.ico", - "extra_datas": [("windows/assets/AudioMuse-AI.ico", "assets")], + "exe_icon": "native-build/windows/assets/AudioMuse-AI.ico", + "extra_datas": [("native-build/windows/assets/AudioMuse-AI.ico", "assets")], "extra_hiddenimports": ["macos.reverse_log", "pgserver.postgres_server"], "collect_submodules": ["windows", "pystray", "PIL"], "excludes_base": ["rumps", "AppKit", "Foundation", "objc"], "bundle": None, }, "macos": { - "launcher": "macos/launcher.py", - "vendor_dir": "macos/vendor", + "launcher": "native-build/macos/launcher.py", + "vendor_dir": "native-build/macos/vendor", "redis_bin": "redis-server", "pg_contrib_glob": "*.dylib", "initdb_bin": "initdb", "use_pgserver": "always", "console": False, "exe_icon": None, - "extra_datas": [("macos/assets", "assets")], + "extra_datas": [("native-build/macos/assets", "assets")], "extra_hiddenimports": ["numeric_bootstrap", "rumps"], "collect_submodules": ["macos"], "excludes_base": [], "bundle": { "name": "AudioMuse-AI.app", - "icon": "macos/assets/AudioMuse-AI.icns", + "icon": "native-build/macos/assets/AudioMuse-AI.icns", "bundle_identifier": "ai.audiomuse.standalone", "info_plist": { "LSUIElement": True, @@ -66,8 +66,8 @@ def read_app_version(root): }, }, "linux": { - "launcher": "linux/launcher.py", - "vendor_dir": "linux/vendor", + "launcher": "native-build/linux/launcher.py", + "vendor_dir": "native-build/linux/vendor", "redis_bin": "redis-server", "pg_contrib_glob": "*.so", "initdb_bin": "initdb", diff --git a/scripts/standalone/platforms/linux.py b/scripts/standalone/platforms/linux.py index b078c277..8fcedec8 100644 --- a/scripts/standalone/platforms/linux.py +++ b/scripts/standalone/platforms/linux.py @@ -13,7 +13,7 @@ def _present(path): def prepare(ctx): arch = ctx.arch - vendor = ctx.root / "linux" / "vendor" + vendor = ctx.root / "native-build" / "linux" / "vendor" if ctx.use_pgserver: required = [ vendor / "redis" / arch / "redis-server", @@ -39,7 +39,7 @@ def prepare(ctx): if missing: for m in missing: print(f"::error::Missing vendored file: {m}") - raise SystemExit("Vendored inputs missing (see linux/vendor/*/README.md).") + raise SystemExit("Vendored inputs missing (see native-build/linux/vendor/*/README.md).") (vendor / "redis" / arch / "redis-server").chmod(0o755) @@ -66,7 +66,7 @@ def _stage(ctx): (stage / "usr" / "lib" / "systemd" / "user").mkdir(parents=True) subprocess.run(["cp", "-a", str(ctx.bundle_dir), str(stage / "opt" / "AudioMuse-AI")], check=True) - pkg = ctx.root / "linux" / "packaging" + pkg = ctx.root / "native-build" / "linux" / "packaging" shutil.copy2(pkg / "AudioMuse-AI.desktop", stage / "usr" / "share" / "applications" / "AudioMuse-AI.desktop") shutil.copy2(pkg / "AudioMuse-AI-stop.desktop", stage / "usr" / "share" / "applications" / "AudioMuse-AI-stop.desktop") shutil.copy2(pkg / "audiomuse-ai.service", stage / "usr" / "lib" / "systemd" / "user" / "audiomuse-ai.service") @@ -91,7 +91,7 @@ def package(ctx): nfpm_arch = _NFPM_ARCH[ctx.arch] print("==> Generating nfpm config") - template = (ctx.root / "linux" / "packaging" / "nfpm.yaml.in").read_text() + template = (ctx.root / "native-build" / "linux" / "packaging" / "nfpm.yaml.in").read_text() content = ( template.replace("@VERSION@", ctx.version) .replace("@ARCH@", nfpm_arch) diff --git a/scripts/standalone/platforms/macos.py b/scripts/standalone/platforms/macos.py index c7df7737..6356ab44 100644 --- a/scripts/standalone/platforms/macos.py +++ b/scripts/standalone/platforms/macos.py @@ -15,7 +15,7 @@ def prepare(ctx): print("==> Generating icons from screenshot/audiomuseai.png") - subprocess.run(["bash", "macos/make_icns.sh"], check=True, cwd=str(ctx.root)) + subprocess.run(["bash", "native-build/macos/make_icns.sh"], check=True, cwd=str(ctx.root)) def _sign_nested(app, entitlements): @@ -33,7 +33,7 @@ def _sign_nested(app, entitlements): def package(ctx): app = ctx.app_path - entitlements = str(ctx.root / "macos" / "entitlements.plist") + entitlements = str(ctx.root / "native-build" / "macos" / "entitlements.plist") _sign_nested(app, entitlements) diff --git a/scripts/standalone/platforms/windows.py b/scripts/standalone/platforms/windows.py index 5ee66db8..11efa49b 100644 --- a/scripts/standalone/platforms/windows.py +++ b/scripts/standalone/platforms/windows.py @@ -3,9 +3,9 @@ def prepare(ctx): arch = ctx.arch - pg_contrib = ctx.root / "windows" / "vendor" / "pg-contrib" / arch + pg_contrib = ctx.root / "native-build" / "windows" / "vendor" / "pg-contrib" / arch required = [ - ctx.root / "windows" / "vendor" / "redis" / arch / "redis-server.exe", + ctx.root / "native-build" / "windows" / "vendor" / "redis" / arch / "redis-server.exe", pg_contrib / "lib" / "unaccent.dll", pg_contrib / "lib" / "pg_trgm.dll", pg_contrib / "extension" / "unaccent.control", @@ -16,7 +16,7 @@ def prepare(ctx): if missing: for m in missing: print(f"[ERROR] Missing vendored file: {m}") - raise SystemExit("Vendored inputs missing (see windows/vendor/README.md).") + raise SystemExit("Vendored inputs missing (see native-build/windows/vendor/README.md).") def package(ctx): diff --git a/static/setup.js b/static/setup.js index 03c259a3..81694869 100644 --- a/static/setup.js +++ b/static/setup.js @@ -694,7 +694,7 @@ function updateMusicLibrariesHint() { function collectMusicLibrariesValue() { // Returns the MUSIC_LIBRARIES value to store, or null to skip writing. if (!currentLibraryCheckboxes.length && !currentNoRestrictionCheckbox) { - // Section isn't rendered (MPD or provider doesn't support it, or the + // Section isn't rendered (provider doesn't support it, or the // fetch failed). Don't touch MUSIC_LIBRARIES. return null; } diff --git a/tasks/alchemy_projections.py b/tasks/alchemy_projections.py new file mode 100644 index 00000000..096f78c9 --- /dev/null +++ b/tasks/alchemy_projections.py @@ -0,0 +1,218 @@ +"""2D projection helpers shared by song_alchemy, app_map, and app_helper. + +These functions are pure numpy/sklearn math with no dependency on the +voyager index, the media server layer, or the database, so consumers that +only need a projection (e.g. the music map) can import this module without +pulling the full song_alchemy import chain. +""" + +import logging +from typing import List, Tuple + +import numpy as np + +try: + # sklearn is already a dependency; import lazily for environments where it's present + from sklearn.decomposition import PCA + from sklearn.linear_model import LogisticRegression +except Exception: + PCA = None + LogisticRegression = None + +logger = logging.getLogger(__name__) + + +def _project_to_2d(vectors: List[np.ndarray]) -> List[Tuple[float, float]]: + """Simple PCA via SVD to project a list of vectors to 2D. + Returns a list of (x, y) tuples in the same order as input vectors. + If there are fewer than 2 vectors, returns zeros for all. + """ + if not vectors: + return [] + mat = np.vstack(vectors) + # Center + mean = np.mean(mat, axis=0) + mat_c = mat - mean + # SVD + try: + _, _, vh = np.linalg.svd(mat_c, full_matrices=False) + except Exception: + # Fallback: return zeros + return [(0.0, 0.0) for _ in vectors] + # Take first two principal components + pcs = vh[:2] + proj = mat_c.dot(pcs.T) + # Normalize projection for nicer plotting + if proj.size == 0: + return [(0.0, 0.0) for _ in vectors] + # Normalize preserving aspect ratio: use a single global scale so x/y units are comparable + # center at zero + proj_centered = proj - proj.mean(axis=0) + max_abs = np.max(np.abs(proj_centered)) + if max_abs == 0: + return [(0.0, 0.0) for _ in vectors] + scaled = proj_centered / max_abs + # clamp to [-1,1] for safety + scaled = np.clip(scaled, -1.0, 1.0) + return [(float(x), float(y)) for x, y in scaled] + + +def _project_aligned_add_sub(vectors: List[np.ndarray], add_centroid: np.ndarray, subtract_centroid: np.ndarray) -> List[Tuple[float, float]]: + """Project vectors to 2D where the x-axis is aligned with the vector + from add_centroid -> subtract_centroid. The y-axis is the leading + orthogonal component (first PC of residuals). + This emphasizes separation along the add-vs-subtract direction. + """ + if not vectors: + return [] + # Convert list to matrix and center relative to add_centroid + mat = np.vstack(vectors) + rel = mat - add_centroid + axis = subtract_centroid - add_centroid + axis_norm = np.linalg.norm(axis) + if axis_norm == 0: + # Fallback to PCA if centroids coincide + return _project_to_2d(vectors) + axis_u = axis / axis_norm + + # Compute x coordinates as projection on axis + x_coords = rel.dot(axis_u) + + # Remove axis component to get residuals for y-axis computation + proj_on_axis = np.outer(x_coords, axis_u) + residuals = rel - proj_on_axis + + # Find leading direction in residuals via SVD + try: + # If residuals are all near-zero, SVD will still succeed but produce small values + _, _, vh = np.linalg.svd(residuals, full_matrices=False) + y_u = vh[0] + except Exception: + y_u = None + + if y_u is None or np.linalg.norm(y_u) == 0: + # Create an arbitrary orthogonal vector to axis_u + # pick an index where axis_u has smallest absolute value + idx = int(np.argmin(np.abs(axis_u))) + e = np.zeros_like(axis_u) + e[idx] = 1.0 + y_u = e - np.dot(e, axis_u) * axis_u + norm_y = np.linalg.norm(y_u) + if norm_y == 0: + # fallback + return _project_to_2d(vectors) + y_u = y_u / norm_y + else: + # ensure orthogonal to axis_u (numerical stability) + y_u = y_u - np.dot(y_u, axis_u) * axis_u + y_u_norm = np.linalg.norm(y_u) + if y_u_norm == 0: + return _project_to_2d(vectors) + y_u = y_u / y_u_norm + + y_coords = residuals.dot(y_u) + + coords = np.vstack([x_coords, y_coords]).T + # Center and scale uniformly so x and y share same units + coords_centered = coords - coords.mean(axis=0) + max_abs = np.max(np.abs(coords_centered)) + if max_abs == 0: + return [(0.0, 0.0) for _ in vectors] + scaled = coords_centered / max_abs + scaled = np.clip(scaled, -1.0, 1.0) + return [(float(x), float(y)) for x, y in scaled] + + +def _project_with_umap(vectors: List[np.ndarray], n_components: int = 2) -> List[Tuple[float, float]]: + """Try to project using UMAP if available. Raises ImportError if umap is not installed.""" + import umap + if not vectors: + return [] + mat = np.vstack(vectors) + reducer = umap.UMAP(n_components=n_components, random_state=None, n_jobs=-1) + embedding = reducer.fit_transform(mat) + # Center and scale uniformly so x and y share same units + emb_centered = embedding - embedding.mean(axis=0) + max_abs = np.max(np.abs(emb_centered)) + if max_abs == 0: + return [(0.0, 0.0) for _ in vectors] + scaled = emb_centered / max_abs + scaled = np.clip(scaled, -1.0, 1.0) + return [(float(x), float(y)) for x, y in scaled] + + +def _project_with_discriminant(add_vectors: List[np.ndarray], sub_vectors: List[np.ndarray], all_vectors: List[np.ndarray]) -> List[Tuple[float, float]]: + """Compute a discriminant direction separating add and sub using PCA+LogisticRegression. + Returns 2D coords for all_vectors projected onto (discriminant axis, residual axis). + Falls back (raises) if sklearn not available or insufficient samples. + """ + if LogisticRegression is None or PCA is None: + raise RuntimeError('sklearn not available') + # Need at least one sample in each class + if not add_vectors or not sub_vectors: + raise RuntimeError('Insufficient classes for discriminant') + + X_train = np.vstack([np.vstack(add_vectors), np.vstack(sub_vectors)]) + y_train = np.array([1] * len(add_vectors) + [0] * len(sub_vectors)) + + n_samples, n_features = X_train.shape + # Reduce dimensionality so training is stable (components <= n_samples-1) + max_components = min(32, n_samples - 1, n_features) + if max_components < 1: + raise RuntimeError('Not enough samples for discriminant PCA') + + pca = PCA(n_components=max_components, random_state=42) + x_pca = pca.fit_transform(X_train) + + # Fit logistic regression with regularization for robustness + try: + clf = LogisticRegression(l1_ratio=0, C=1.0, solver='saga', max_iter=1000) + clf.fit(x_pca, y_train) + except Exception: + # Fallback with less regularization if solver fails + clf = LogisticRegression(l1_ratio=0, C=0.1, solver='saga', max_iter=1000) + clf.fit(x_pca, y_train) + + # direction in PCA space + coef = clf.coef_.ravel() + norm = np.linalg.norm(coef) + if norm == 0: + raise RuntimeError('Discriminant produced zero vector') + dir_pca = coef / norm + + # Project all vectors into PCA space then onto discriminant for x coords + all_mat = np.vstack(all_vectors) + all_pca = pca.transform(all_mat) + x_coords = all_pca.dot(dir_pca) + + # Residuals in PCA space + proj_on_dir = np.outer(x_coords, dir_pca) + residuals = all_pca - proj_on_dir + # y direction: leading PC of residuals + try: + _, _, vh = np.linalg.svd(residuals, full_matrices=False) + y_u = vh[0] + except Exception: + y_u = None + + if y_u is None or np.linalg.norm(y_u) == 0: + # fallback: arbitrary orthogonal + idx = int(np.argmin(np.abs(dir_pca))) + e = np.zeros_like(dir_pca) + e[idx] = 1.0 + y_u = e - np.dot(e, dir_pca) * dir_pca + y_u = y_u / (np.linalg.norm(y_u) or 1.0) + else: + y_u = y_u - np.dot(y_u, dir_pca) * dir_pca + y_u = y_u / (np.linalg.norm(y_u) or 1.0) + + y_coords = residuals.dot(y_u) + + coords = np.vstack([x_coords, y_coords]).T + coords_centered = coords - coords.mean(axis=0) + max_abs = np.max(np.abs(coords_centered)) + if max_abs == 0: + return [(0.0, 0.0) for _ in all_vectors] + scaled = coords_centered / max_abs + scaled = np.clip(scaled, -1.0, 1.0) + return [(float(x), float(y)) for x, y in scaled] diff --git a/tasks/analysis.py b/tasks/analysis.py index f7b7e3bd..66f3e2a9 100644 --- a/tasks/analysis.py +++ b/tasks/analysis.py @@ -33,11 +33,6 @@ # Import other project modules -from .voyager_manager import build_and_store_voyager_index -from .clap_text_search import build_and_store_clap_index -from .lyrics_manager import build_and_store_lyrics_index, build_and_store_lyrics_axes_index -from .sem_grove_manager import build_and_store_sem_grove_index -from .artist_gmm_manager import build_and_store_artist_index from .mediaserver import get_recent_albums, get_tracks_from_album, download_track from .memory_utils import ( cleanup_cuda_memory, @@ -132,7 +127,18 @@ def _run_all_index_builds(log_fn=None): shows which builder is currently active (otherwise users see "Building CLAP text search index..." for the entire 95–97 % window even while the lyrics or SemGrove builds are running). + + The index-builder modules are imported here rather than at module top so + that importing ``tasks.analysis`` does not pull in the voyager / CLAP / + lyrics / SemGrove / artist-GMM subsystems; they are only needed when a + rebuild actually runs. """ + from .voyager_manager import build_and_store_voyager_index + from .clap_text_search import build_and_store_clap_index + from .lyrics_manager import build_and_store_lyrics_index, build_and_store_lyrics_axes_index + from .sem_grove_manager import build_and_store_sem_grove_index + from .artist_gmm_manager import build_and_store_artist_index + def _step(label, fn, progress=None, banner=None, fatal=False): if log_fn and progress is not None and banner is not None: try: diff --git a/tasks/artist_gmm_manager.py b/tasks/artist_gmm_manager.py index be4f661a..eef333e8 100644 --- a/tasks/artist_gmm_manager.py +++ b/tasks/artist_gmm_manager.py @@ -536,7 +536,7 @@ def build_and_store_artist_index(db_conn=None): num_parts = len(parts) logger.info(f"Artist index size {len(index_bytes)} exceeds {ARTIST_INDEX_MAX_PART_SIZE_MB}MB - storing as {num_parts} segmented rows.") - insert_q = "INSERT INTO artist_index_data (index_name, index_data, artist_map_json, gmm_params_json, created_at) VALUES (%s, %s, %s, %s, NOW())" + insert_q = "INSERT INTO artist_index_data (index_name, index_data, artist_map_json, gmm_params_json, created_at) VALUES (%s, %s, %s, %s, NOW()) ON CONFLICT (index_name) DO UPDATE SET index_data = EXCLUDED.index_data, artist_map_json = EXCLUDED.artist_map_json, gmm_params_json = EXCLUDED.gmm_params_json, created_at = EXCLUDED.created_at" for idx, part in enumerate(parts, start=1): part_name = f"{ARTIST_INDEX_NAME}_{idx}_{num_parts}" cur.execute(insert_q, (part_name, part, '', '')) diff --git a/tasks/clap_text_search.py b/tasks/clap_text_search.py index 34e7010d..dc99cba0 100644 --- a/tasks/clap_text_search.py +++ b/tasks/clap_text_search.py @@ -3,12 +3,8 @@ Provides in-memory caching and fast text-based music search using CLAP embeddings. """ -import gc -import json import logging -import re import sys -import tempfile import threading import time @@ -60,25 +56,8 @@ def get_clap_cache_size() -> int: def _fetch_clap_metadata(item_ids: list) -> Dict[str, Dict[str, str]]: - """Fetch metadata for CLAP result item_ids from the database.""" - metadata_map: Dict[str, Dict[str, str]] = {} - if not item_ids: - return metadata_map - - from app_helper import get_score_data_by_ids - try: - track_details_list = get_score_data_by_ids(item_ids) - for row in track_details_list: - item_id = row['item_id'] - metadata_map[item_id] = { - 'title': row.get('title', ''), - 'author': row.get('author', ''), - 'album': row.get('album', ''), - } - except Exception: - pass - - return metadata_map + from .commons import fetch_track_metadata_map + return fetch_track_metadata_map(item_ids) def _load_clap_index_from_db() -> bool: @@ -86,119 +65,26 @@ def _load_clap_index_from_db() -> bool: from app_helper import get_db from config import CLAP_EMBEDDING_DIMENSION, VOYAGER_QUERY_EF + from .index_build_helpers import load_voyager_index_from_db try: - conn = get_db() - with conn.cursor() as cur: - cur.execute("SET LOCAL statement_timeout = 0") - cur.execute( - "SELECT index_data, id_map_json, embedding_dimension FROM clap_index_data WHERE index_name = %s", - ('clap_index',) - ) - row = cur.fetchone() - - index_stream = None - try: - if row: - index_binary_data, id_map_json, db_embedding_dim = row - index_stream = tempfile.TemporaryFile() - index_stream.write(index_binary_data) - index_stream.seek(0) - else: - seg_pattern = re.compile(r'^clap_index_(\d+)_(\d+)$') - parts = [] - total_expected = None - with conn.cursor(name='clap_index_segments') as seg_cur: - seg_cur.itersize = 50 - seg_cur.execute( - "SELECT index_name, index_data, id_map_json, embedding_dimension FROM clap_index_data WHERE index_name LIKE %s ESCAPE '\\'", - (r'clap_index\_%\_%',) - ) - for name, part_data, part_id_map_json, part_dim in seg_cur: - m = seg_pattern.match(name) - if not m: - continue - part_no = int(m.group(1)) - total = int(m.group(2)) - if total_expected is None: - total_expected = total - elif total_expected != total: - logger.error(f"Segment total mismatch for CLAP index parts ({total_expected} vs {total}).") - return False - parts.append((part_no, part_data, part_id_map_json, part_dim)) - - if total_expected is None or len(parts) != total_expected: - logger.error(f"Incomplete CLAP index segments: expected {total_expected}, found {len(parts)}.") - return False - - parts.sort(key=lambda p: p[0]) - from .index_build_helpers import reassemble_segmented_id_map - id_map_json_candidate = reassemble_segmented_id_map((p[0], p[2]) for p in parts) - for _, _, _, part_dim in parts: - if part_dim != CLAP_EMBEDDING_DIMENSION: - logger.error(f"CLAP index embedding_dimension mismatch in segmented parts: expected {CLAP_EMBEDDING_DIMENSION}, got {part_dim}.") - return False - - if not id_map_json_candidate: - logger.error("No id_map_json found in segmented CLAP index rows.") - return False - - db_embedding_dim = parts[0][3] - index_stream = tempfile.TemporaryFile() - for _, part_data, _, _ in parts: - index_stream.write(part_data) - index_stream.seek(0) - id_map_json = id_map_json_candidate - - if index_stream is None: - logger.error("CLAP index binary data was empty.") - return False - - if db_embedding_dim != CLAP_EMBEDDING_DIMENSION: - logger.error(f"CLAP index dimension mismatch: db={db_embedding_dim} expected={CLAP_EMBEDDING_DIMENSION}") - index_stream.close() - return False - - try: - try: - import voyager # type: ignore - except ImportError: - logger.warning("Voyager library is unavailable; cannot load persisted CLAP index.") - return False - - loaded_index = voyager.Index.load(index_stream) - loaded_index.ef = VOYAGER_QUERY_EF - finally: - if index_stream is not None: - try: - index_stream.close() - except Exception as close_error: - logger.warning("Failed to close CLAP index stream: %s", close_error, exc_info=True) - - except Exception: - if index_stream is not None: - try: - index_stream.close() - except Exception: - pass - raise - - id_map = {int(k): v for k, v in json.loads(id_map_json).items()} - reverse_id_map = {v: k for k, v in id_map.items()} - - if not id_map: - logger.error("CLAP index id_map is empty.") - return False - - _CLAP_CACHE['loaded'] = True - - _CLAP_INDEX_CACHE['index'] = loaded_index - _CLAP_INDEX_CACHE['id_map'] = id_map - _CLAP_INDEX_CACHE['reverse_id_map'] = reverse_id_map - _CLAP_INDEX_CACHE['loaded'] = True - - logger.info(f"CLAP index loaded from database with {len(id_map)} items.") - return True + loaded = load_voyager_index_from_db( + get_db(), 'clap_index_data', 'clap_index', + CLAP_EMBEDDING_DIMENSION, VOYAGER_QUERY_EF, label='CLAP', + ) + if loaded is None: + return False + loaded_index, id_map, reverse_id_map = loaded + + _CLAP_CACHE['loaded'] = True + + _CLAP_INDEX_CACHE['index'] = loaded_index + _CLAP_INDEX_CACHE['id_map'] = id_map + _CLAP_INDEX_CACHE['reverse_id_map'] = reverse_id_map + _CLAP_INDEX_CACHE['loaded'] = True + + logger.info(f"CLAP index loaded from database with {len(id_map)} items.") + return True except Exception as e: logger.error(f"Failed to load CLAP index from DB: {e}", exc_info=True) return False @@ -208,63 +94,21 @@ def build_and_store_clap_index(db_conn=None): """Build a CLAP text search voyager index from stored CLAP embeddings and save it to the DB.""" from app_helper import get_db from config import CLAP_EMBEDDING_DIMENSION, VOYAGER_METRIC - from .index_build_helpers import ( - iter_embedding_batches, - build_voyager_index_bytes_streaming, - store_voyager_index_segmented, - build_id_map, - EmptyIndexError, - ) - - try: - import voyager # type: ignore # noqa: F401 - except ImportError: - logger.warning("Voyager library is unavailable; cannot build CLAP index.") - return False + from .index_build_helpers import build_and_store_index_streaming if db_conn is None: db_conn = get_db() - try: - logger.info("Building CLAP voyager index (streaming)...") - batches = iter_embedding_batches( - table="clap_embedding", - column="embedding", - dim=CLAP_EMBEDDING_DIMENSION, - ) - try: - index_bytes, item_ids = build_voyager_index_bytes_streaming( - batches, CLAP_EMBEDDING_DIMENSION, metric=VOYAGER_METRIC, - ) - except EmptyIndexError as ve: - logger.warning(f"No valid CLAP embedding vectors found for CLAP index build: {ve}") - return False - gc.collect() - - if not index_bytes: - logger.error("Generated CLAP index binary is empty. Aborting storage.") - return False - - id_map = build_id_map(item_ids) - store_voyager_index_segmented( - db_conn, - target_table="clap_index_data", - index_name="clap_index", - index_bytes=index_bytes, - id_map=id_map, - embedding_dimension=CLAP_EMBEDDING_DIMENSION, - ) - - db_conn.commit() - logger.info("CLAP text search index build successful.") - return True - except Exception as e: - logger.error(f"Failed to build and store CLAP index: {e}", exc_info=True) - try: - db_conn.rollback() - except Exception: - pass - return False + return build_and_store_index_streaming( + db_conn, + source_table="clap_embedding", + source_column="embedding", + dim=CLAP_EMBEDDING_DIMENSION, + target_table="clap_index_data", + index_name="clap_index", + metric=VOYAGER_METRIC, + label="CLAP", + ) def _unload_timer_worker(): @@ -360,9 +204,8 @@ def load_clap_cache_from_db(): Returns True if successful, False otherwise. """ - from app_helper import get_db from config import CLAP_ENABLED - + if not CLAP_ENABLED: logger.info("CLAP is disabled, skipping cache load.") return False diff --git a/tasks/cleaning.py b/tasks/cleaning.py index ed3e9517..0f61a365 100644 --- a/tasks/cleaning.py +++ b/tasks/cleaning.py @@ -28,7 +28,7 @@ def identify_and_clean_orphaned_albums_task(): Main RQ task to identify and automatically clean orphaned albums from the database. This combines identification and deletion into a single automated process. """ - from app import app + from flask_app import app from app_helper import redis_conn, get_db, save_task_status, TASK_STATUS_STARTED, TASK_STATUS_PROGRESS, TASK_STATUS_SUCCESS, TASK_STATUS_FAILURE current_job = get_current_job(redis_conn) @@ -276,7 +276,7 @@ def delete_orphaned_albums_sync(orphaned_track_ids): Returns: dict: Result summary with deletion statistics """ - from app import get_db + from app_helper import get_db if not orphaned_track_ids: return {"status": "SUCCESS", "message": "No tracks to delete", "deleted_count": 0} diff --git a/tasks/clustering.py b/tasks/clustering.py index 69736405..f03395d6 100644 --- a/tasks/clustering.py +++ b/tasks/clustering.py @@ -54,7 +54,7 @@ def batch_task_failure_handler(job, connection, type, value, tb): """A failure handler for the clustering batch sub-task, executed by the worker.""" - from app import app + from flask_app import app from app_helper import save_task_status, TASK_STATUS_FAILURE with app.app_context(): task_id = getattr(job, 'id', None) or getattr(job, 'get_id', lambda: None)() @@ -133,7 +133,7 @@ def run_clustering_batch_task( Executes a batch of clustering iterations. This task is enqueued by the main clustering task. """ # --- Local imports to prevent circular dependency --- - from app import app + from flask_app import app from app_helper import (redis_conn, save_task_status, get_task_info_from_db, TASK_STATUS_PROGRESS, TASK_STATUS_REVOKED, TASK_STATUS_FAILURE, TASK_STATUS_SUCCESS) @@ -275,7 +275,7 @@ def run_clustering_task( Orchestrates data preparation, batch job creation, result aggregation, and playlist creation. """ # --- Local imports to prevent circular dependency --- - from app import app + from flask_app import app from app_helper import redis_conn, get_db, save_task_status, get_task_info_from_db, update_playlist_table, get_child_tasks_from_db, TASK_STATUS_STARTED, TASK_STATUS_PROGRESS, TASK_STATUS_SUCCESS, TASK_STATUS_FAILURE, TASK_STATUS_REVOKED current_job = get_current_job(redis_conn) @@ -668,7 +668,7 @@ def _monitor_and_process_batches(state_dict, parent_task_id, initial_check=False CRITICAL: This prevents the main task from hanging at 4980/5000 runs by implementing timeouts and forced progress tracking. """ - from app_helper import redis_conn, get_child_tasks_from_db, get_task_info_from_db, TASK_STATUS_SUCCESS, TASK_STATUS_FAILURE, TASK_STATUS_REVOKED, TASK_STATUS_STARTED, TASK_STATUS_PROGRESS + from app_helper import redis_conn, get_child_tasks_from_db, TASK_STATUS_SUCCESS, TASK_STATUS_FAILURE, TASK_STATUS_REVOKED, TASK_STATUS_STARTED current_time = time.time() timeout_seconds = CLUSTERING_BATCH_TIMEOUT_MINUTES * 60 diff --git a/tasks/clustering_helper.py b/tasks/clustering_helper.py index 3ce5e236..c14598c8 100644 --- a/tasks/clustering_helper.py +++ b/tasks/clustering_helper.py @@ -113,7 +113,7 @@ def _perform_single_clustering_iteration( """ try: # Local import to prevent circular dependency - from app import app + from flask_app import app if not item_ids_for_subset: logger.warning(f"{log_prefix} Iteration {run_idx}: Received empty item ID subset. Skipping.") @@ -717,7 +717,8 @@ def get_job_result_safely(job_id, parent_task_id, task_type="child task"): 'final_subset_track_ids') so the caller can use a single code path, or None on failure. """ # Local imports to prevent circular dependency - from app import app, JobStatus + from flask_app import app + from rq.job import JobStatus from app_helper import redis_conn, get_task_info_from_db, TASK_STATUS_SUCCESS try: diff --git a/tasks/commons.py b/tasks/commons.py index 7bae8028..e77e0ef0 100644 --- a/tasks/commons.py +++ b/tasks/commons.py @@ -1,5 +1,7 @@ # tasks/commons.py +import logging + import numpy as np # Import necessary constants from config @@ -8,6 +10,31 @@ ENERGY_MAX, ENERGY_MIN ) +logger = logging.getLogger(__name__) + + +def fetch_track_metadata_map(item_ids): + """Fetch {item_id: {title, author, album}} from the score table. + + Shared by the CLAP, lyrics, and SemGrove search paths. Returns an empty + dict (and logs a warning) when the lookup fails, so callers can always + treat the result as a best-effort metadata overlay. + """ + metadata_map = {} + if not item_ids: + return metadata_map + from app_helper import get_score_data_by_ids + try: + for row in get_score_data_by_ids(item_ids): + metadata_map[row['item_id']] = { + 'title': row.get('title', '') or '', + 'author': row.get('author', '') or '', + 'album': row.get('album', '') or '', + } + except Exception as e: + logger.warning(f"Failed to fetch track metadata: {e}") + return metadata_map + def score_vector(row, mood_labels_list, other_feature_labels_list): # other_feature_labels_list is now passed """Converts a database row into a numerical feature vector for clustering.""" # Extract features from the database row diff --git a/tasks/index_build_helpers.py b/tasks/index_build_helpers.py index a4ab18ab..2ab8b276 100644 --- a/tasks/index_build_helpers.py +++ b/tasks/index_build_helpers.py @@ -367,81 +367,6 @@ def _resolve_voyager_space(metric: str): return voyager.Space.Cosine -def build_voyager_index_bytes( - buf: np.ndarray, - dim: int, - metric: str = "angular", - m: Optional[int] = None, - ef_construction: Optional[int] = None, -) -> bytes: - """Build a Voyager HNSW index over ``buf`` and serialize it to bytes. - - Item ids are assigned densely as ``0..N-1``; callers persist their own - ``{voyager_id: item_id}`` map alongside the index bytes (see - ``store_voyager_index_segmented``). - - Args: - buf: contiguous ``np.ndarray`` of shape ``(N, dim)`` and dtype - ``float32``. N must be >= 1. - dim: number of dimensions, must equal ``buf.shape[1]``. - metric: ``"angular"`` (cosine), ``"euclidean"``, or ``"dot"``. - m, ef_construction: HNSW graph parameters. Default to the values in - ``config.VOYAGER_M`` / ``config.VOYAGER_EF_CONSTRUCTION``. - - Returns: - Serialized index bytes. - - Raises: - ImportError: if the ``voyager`` library is not installed. - ValueError: if ``buf`` is empty or has the wrong shape/dtype. - """ - import voyager - - if not isinstance(buf, np.ndarray) or buf.ndim != 2: - raise ValueError("buf must be a 2-D numpy array") - if buf.shape[0] == 0: - raise ValueError("buf is empty; refusing to build an empty index") - if buf.shape[1] != dim: - raise ValueError( - f"buf has dim={buf.shape[1]} but caller declared dim={dim}" - ) - if buf.dtype != np.float32: - buf = buf.astype(np.float32, copy=False) - if not buf.flags["C_CONTIGUOUS"]: - buf = np.ascontiguousarray(buf) - - m_val = config.VOYAGER_M if m is None else int(m) - ef_val = config.VOYAGER_EF_CONSTRUCTION if ef_construction is None else int(ef_construction) - - space = _resolve_voyager_space(metric) - builder = voyager.Index( - space=space, - num_dimensions=dim, - M=m_val, - ef_construction=ef_val, - ) - - n = buf.shape[0] - ids = np.arange(n, dtype=np.int64) - builder.add_items(buf, ids=ids) - - temp_file_path: Optional[str] = None - try: - with tempfile.NamedTemporaryFile(delete=False, suffix=".voyager") as tmp: - temp_file_path = tmp.name - builder.save(temp_file_path) - del builder - gc.collect() - with open(temp_file_path, "rb") as f: - return f.read() - finally: - if temp_file_path and os.path.exists(temp_file_path): - try: - os.remove(temp_file_path) - except Exception: - pass - - def build_voyager_index_bytes_streaming( batch_iter: Iterable[Tuple[np.ndarray, List[str]]], dim: int, @@ -568,6 +493,188 @@ def reassemble_segmented_id_map(fragments: Iterable[Tuple[int, Optional[str]]]) return "".join(frag or "" for _, frag in sorted(fragments, key=lambda p: p[0])) +def build_and_store_index_streaming( + db_conn, + source_table: str, + source_column: str, + dim: int, + target_table: str, + index_name: str, + metric: str, + where_clause: Optional[str] = None, + label: Optional[str] = None, +) -> bool: + """Stream embeddings, build a Voyager index, and persist it. + + Wraps the canonical build pipeline shared by the CLAP, lyrics, and + lyrics-axes builders: ``iter_embedding_batches`` -> + ``build_voyager_index_bytes_streaming`` -> ``store_voyager_index_segmented``. + Commits on success and rolls back on failure using the caller's + ``db_conn``. Returns True on success, False when the source table has no + usable vectors or the build/store fails. + """ + label = label or index_name + try: + import voyager # type: ignore # noqa: F401 + except ImportError: + logger.warning("Voyager library is unavailable; cannot build %s index.", label) + return False + + try: + logger.info("Building %s voyager index (streaming)...", label) + batches = iter_embedding_batches( + table=source_table, + column=source_column, + dim=dim, + where_clause=where_clause, + ) + try: + index_bytes, item_ids = build_voyager_index_bytes_streaming( + batches, dim, metric=metric, + ) + except EmptyIndexError as ve: + logger.warning("No valid %s vectors found for index build: %s", label, ve) + return False + gc.collect() + + if not index_bytes: + logger.error("Generated %s index binary is empty; aborting storage.", label) + return False + + id_map = build_id_map(item_ids) + store_voyager_index_segmented( + db_conn, + target_table=target_table, + index_name=index_name, + index_bytes=index_bytes, + id_map=id_map, + embedding_dimension=dim, + ) + + db_conn.commit() + logger.info("%s index build successful.", label) + return True + except Exception as e: + logger.error("Failed to build/store %s index: %s", label, e, exc_info=True) + try: + db_conn.rollback() + except Exception: + pass + return False + + +def load_voyager_index_from_db( + conn, + table: str, + index_name: str, + expected_dim: int, + query_ef: int, + label: Optional[str] = None, +): + """Load a Voyager index persisted by ``store_voyager_index_segmented``. + + Tries the classic single-row layout first, then the segmented + ``__`` layout. Returns ``(index, id_map, + reverse_id_map)`` on success or ``None`` when the index is missing, + incomplete, or fails validation. DB and deserialization exceptions + propagate to the caller, which is expected to log and treat the load + as failed (the historical per-manager behavior). + """ + label = label or index_name + _validate_sql_identifier(table, "table") + _validate_sql_identifier(index_name, "index_name") + + try: + import voyager # type: ignore + except ImportError: + logger.warning("Voyager library is unavailable; cannot load %s index.", label) + return None + + with conn.cursor() as cur: + cur.execute("SET LOCAL statement_timeout = 0") + cur.execute( + f"SELECT index_data, id_map_json, embedding_dimension FROM {table} " + f"WHERE index_name = %s", + (index_name,), + ) + row = cur.fetchone() + + index_stream = None + try: + if row: + binary, id_map_json, db_dim = row + index_stream = tempfile.TemporaryFile() + index_stream.write(binary) + index_stream.seek(0) + else: + seg_pattern = re.compile(rf'^{re.escape(index_name)}_(\d+)_(\d+)$') + parts = [] + total_expected = None + with conn.cursor(name=f'{index_name}_segments') as seg_cur: + seg_cur.itersize = 50 + seg_cur.execute( + f"SELECT index_name, index_data, id_map_json, embedding_dimension " + f"FROM {table} WHERE index_name LIKE %s ESCAPE '\\'", + (index_name.replace('_', r'\_') + r'\_%\_%',), + ) + for name, part_data, part_id_map, part_dim in seg_cur: + m = seg_pattern.match(name) + if not m: + continue + part_no = int(m.group(1)) + total = int(m.group(2)) + if total_expected is None: + total_expected = total + elif total_expected != total: + logger.error( + "%s index segment total mismatch: %s vs %s", + label, total_expected, total, + ) + return None + parts.append((part_no, part_data, part_id_map, part_dim)) + + if total_expected is None or len(parts) != total_expected: + logger.info( + "No complete persisted %s index found (expected %s, have %d).", + label, total_expected, len(parts), + ) + return None + + parts.sort(key=lambda p: p[0]) + db_dim = parts[0][3] + index_stream = tempfile.TemporaryFile() + for _, part_data, _, _ in parts: + index_stream.write(part_data) + index_stream.seek(0) + id_map_json = reassemble_segmented_id_map((p[0], p[2]) for p in parts) + + if index_stream is None or not id_map_json: + logger.info("%s index not found or empty in the database.", label) + return None + if db_dim != expected_dim: + logger.error( + "%s index dimension mismatch: db=%s expected=%s", + label, db_dim, expected_dim, + ) + return None + + loaded_index = voyager.Index.load(index_stream) + loaded_index.ef = query_ef + finally: + if index_stream is not None: + try: + index_stream.close() + except Exception: + pass + + id_map = {int(k): v for k, v in json.loads(id_map_json).items()} + if not id_map: + logger.warning("%s index id_map is empty.", label) + return None + reverse_id_map = {v: k for k, v in id_map.items()} + return loaded_index, id_map, reverse_id_map + + def store_voyager_index_segmented( db_conn, target_table: str, @@ -602,7 +709,7 @@ def store_voyager_index_segmented( index_name: logical index name (e.g. ``"clap_index"``). Must be a bare SQL identifier so the LIKE-escape pattern is unambiguous. index_bytes: serialized index payload (from - ``build_voyager_index_bytes``). + ``build_voyager_index_bytes_streaming``). id_map: ``{voyager_id_int: item_id_str}`` mapping. Serialized to JSON for the first row. embedding_dimension: stored alongside the index for validation on @@ -638,7 +745,12 @@ def store_voyager_index_segmented( insert_sql = ( f"INSERT INTO {target_table} " f"(index_name, {binary_column}, id_map_json, embedding_dimension, created_at) " - f"VALUES (%s, %s, %s, %s, CURRENT_TIMESTAMP)" + f"VALUES (%s, %s, %s, %s, CURRENT_TIMESTAMP) " + f"ON CONFLICT (index_name) DO UPDATE SET " + f"{binary_column} = EXCLUDED.{binary_column}, " + f"id_map_json = EXCLUDED.id_map_json, " + f"embedding_dimension = EXCLUDED.embedding_dimension, " + f"created_at = EXCLUDED.created_at" ) id_map_fits = len(id_map_json.encode("utf-8")) <= max_part_size @@ -820,7 +932,10 @@ def store_segmented_blob( ) insert_sql = ( f"INSERT INTO {target_table} (name, blob_data, created_at) " - f"VALUES (%s, %s, CURRENT_TIMESTAMP)" + f"VALUES (%s, %s, CURRENT_TIMESTAMP) " + f"ON CONFLICT (name) DO UPDATE SET " + f"blob_data = EXCLUDED.blob_data, " + f"created_at = EXCLUDED.created_at" ) with db_conn.cursor() as cur: diff --git a/tasks/lyrics_manager.py b/tasks/lyrics_manager.py index c08ec384..b7cd720b 100644 --- a/tasks/lyrics_manager.py +++ b/tasks/lyrics_manager.py @@ -14,12 +14,8 @@ * search_by_text(query, limit) for the open free-form text tab """ -import gc -import json import logging -import re import sys -import tempfile from typing import Dict, List, Optional import numpy as np @@ -52,21 +48,8 @@ # --------------------------------------------------------------------------- def _fetch_lyrics_metadata(item_ids: List[str]) -> Dict[str, Dict[str, str]]: - metadata_map: Dict[str, Dict[str, str]] = {} - if not item_ids: - return metadata_map - from app_helper import get_score_data_by_ids - try: - track_details = get_score_data_by_ids(item_ids) - for row in track_details: - metadata_map[row['item_id']] = { - 'title': row.get('title', '') or '', - 'author': row.get('author', '') or '', - 'album': row.get('album', '') or '', - } - except Exception as e: - logger.warning(f"Failed to fetch lyrics metadata: {e}") - return metadata_map + from .commons import fetch_track_metadata_map + return fetch_track_metadata_map(item_ids) def _axis_columns_from_axes() -> List[tuple]: @@ -83,68 +66,26 @@ def build_and_store_lyrics_index(db_conn=None) -> bool: """Build a voyager index from stored lyrics embeddings and persist it.""" from app_helper import get_db from config import LYRICS_ENABLED, LYRICS_EMBEDDING_DIMENSION, VOYAGER_METRIC - from .index_build_helpers import ( - iter_embedding_batches, - build_voyager_index_bytes_streaming, - store_voyager_index_segmented, - build_id_map, - EmptyIndexError, - ) + from .index_build_helpers import build_and_store_index_streaming if not LYRICS_ENABLED: logger.info("Lyrics analysis is disabled; skipping lyrics index build.") return False - try: - import voyager # type: ignore # noqa: F401 - except ImportError: - logger.warning("Voyager library is unavailable; cannot build lyrics index.") - return False - if db_conn is None: db_conn = get_db() - try: - logger.info("Building lyrics voyager index (streaming)...") - batches = iter_embedding_batches( - table="lyrics_embedding", - column="embedding", - dim=LYRICS_EMBEDDING_DIMENSION, - where_clause="embedding IS NOT NULL", - ) - try: - index_bytes, item_ids = build_voyager_index_bytes_streaming( - batches, LYRICS_EMBEDDING_DIMENSION, metric=VOYAGER_METRIC, - ) - except EmptyIndexError as ve: - logger.warning(f"No valid lyrics embedding vectors for index build: {ve}") - return False - gc.collect() - - if not index_bytes: - logger.error("Generated lyrics index binary is empty; aborting storage.") - return False - - id_map = build_id_map(item_ids) - store_voyager_index_segmented( - db_conn, - target_table="lyrics_index_data", - index_name="lyrics_index", - index_bytes=index_bytes, - id_map=id_map, - embedding_dimension=LYRICS_EMBEDDING_DIMENSION, - ) - - db_conn.commit() - logger.info("Lyrics search index build successful.") - return True - except Exception as e: - logger.error(f"Failed to build/store lyrics index: {e}", exc_info=True) - try: - db_conn.rollback() - except Exception: - pass - return False + return build_and_store_index_streaming( + db_conn, + source_table="lyrics_embedding", + source_column="embedding", + dim=LYRICS_EMBEDDING_DIMENSION, + target_table="lyrics_index_data", + index_name="lyrics_index", + metric=VOYAGER_METRIC, + where_clause="embedding IS NOT NULL", + label="lyrics", + ) # --------------------------------------------------------------------------- @@ -155,24 +96,12 @@ def build_and_store_lyrics_axes_index(db_conn=None) -> bool: """Build a voyager index from the per-song axis_scores flattened to a fixed-order vector.""" from app_helper import get_db from config import LYRICS_ENABLED - from .index_build_helpers import ( - iter_embedding_batches, - build_voyager_index_bytes_streaming, - store_voyager_index_segmented, - build_id_map, - EmptyIndexError, - ) + from .index_build_helpers import build_and_store_index_streaming if not LYRICS_ENABLED: logger.info("Lyrics analysis is disabled; skipping lyrics axes index build.") return False - try: - import voyager # type: ignore # noqa: F401 - except ImportError: - logger.warning("Voyager library is unavailable; cannot build lyrics axes index.") - return False - if db_conn is None: db_conn = get_db() @@ -182,47 +111,17 @@ def build_and_store_lyrics_axes_index(db_conn=None) -> bool: return False dim = len(columns) - try: - logger.info(f"Building lyrics axes voyager index (streaming, dim={dim})...") - batches = iter_embedding_batches( - table="lyrics_embedding", - column="axis_vector", - dim=dim, - where_clause="axis_vector IS NOT NULL", - ) - try: - index_bytes, item_ids = build_voyager_index_bytes_streaming( - batches, dim, metric="angular", - ) - except EmptyIndexError as ve: - logger.warning(f"No usable axis_vector rows; aborting axes index build: {ve}") - return False - gc.collect() - - if not index_bytes: - logger.error("Generated lyrics axes index binary is empty; aborting storage.") - return False - - id_map = build_id_map(item_ids) - store_voyager_index_segmented( - db_conn, - target_table="lyrics_axes_index_data", - index_name="lyrics_axes_index", - index_bytes=index_bytes, - id_map=id_map, - embedding_dimension=dim, - ) - - db_conn.commit() - logger.info("Lyrics axes index build successful.") - return True - except Exception as e: - logger.error(f"Failed to build/store lyrics axes index: {e}", exc_info=True) - try: - db_conn.rollback() - except Exception: - pass - return False + return build_and_store_index_streaming( + db_conn, + source_table="lyrics_embedding", + source_column="axis_vector", + dim=dim, + target_table="lyrics_axes_index_data", + index_name="lyrics_axes_index", + metric="angular", + where_clause="axis_vector IS NOT NULL", + label="lyrics axes", + ) # --------------------------------------------------------------------------- @@ -233,105 +132,24 @@ def _load_lyrics_index_from_db() -> bool: """Load persisted voyager index for lyrics from the DB into the global cache.""" from app_helper import get_db from config import LYRICS_EMBEDDING_DIMENSION, VOYAGER_QUERY_EF + from .index_build_helpers import load_voyager_index_from_db try: - conn = get_db() - with conn.cursor() as cur: - cur.execute("SET LOCAL statement_timeout = 0") - cur.execute( - "SELECT index_data, id_map_json, embedding_dimension FROM lyrics_index_data " - "WHERE index_name = %s", - ('lyrics_index',), - ) - row = cur.fetchone() - - index_stream = None - try: - if row: - binary, id_map_json, db_dim = row - index_stream = tempfile.TemporaryFile() - index_stream.write(binary) - index_stream.seek(0) - else: - seg_pattern = re.compile(r'^lyrics_index_(\d+)_(\d+)$') - parts = [] - total_expected = None - with conn.cursor(name='lyrics_index_segments') as seg_cur: - seg_cur.itersize = 50 - seg_cur.execute( - "SELECT index_name, index_data, id_map_json, embedding_dimension " - "FROM lyrics_index_data WHERE index_name LIKE %s ESCAPE '\\'", - (r'lyrics_index\_%\_%',), - ) - for name, part_data, part_id_map, part_dim in seg_cur: - m = seg_pattern.match(name) - if not m: - continue - part_no = int(m.group(1)) - total = int(m.group(2)) - if total_expected is None: - total_expected = total - elif total_expected != total: - logger.error( - f"Lyrics index segment total mismatch: {total_expected} vs {total}" - ) - return False - parts.append((part_no, part_data, part_id_map, part_dim)) - - if total_expected is None or len(parts) != total_expected: - logger.info( - f"No complete persisted lyrics index found (expected {total_expected}, " - f"have {len(parts)})." - ) - return False - - parts.sort(key=lambda p: p[0]) - from .index_build_helpers import reassemble_segmented_id_map - db_dim = parts[0][3] - index_stream = tempfile.TemporaryFile() - for _, part_data, _, _ in parts: - index_stream.write(part_data) - index_stream.seek(0) - id_map_json = reassemble_segmented_id_map((p[0], p[2]) for p in parts) - - if index_stream is None: - return False - if db_dim != LYRICS_EMBEDDING_DIMENSION: - logger.error( - f"Lyrics index dimension mismatch: db={db_dim} expected={LYRICS_EMBEDDING_DIMENSION}" - ) - index_stream.close() - return False - - try: - import voyager # type: ignore - except ImportError: - logger.warning("Voyager library is unavailable; cannot load lyrics index.") - return False - - loaded_index = voyager.Index.load(index_stream) - loaded_index.ef = VOYAGER_QUERY_EF - finally: - if index_stream is not None: - try: - index_stream.close() - except Exception: - pass - - id_map = {int(k): v for k, v in json.loads(id_map_json).items()} - reverse_id_map = {v: k for k, v in id_map.items()} - - if not id_map: - logger.warning("Lyrics index id_map is empty.") - return False - - _LYRICS_INDEX_CACHE['index'] = loaded_index - _LYRICS_INDEX_CACHE['id_map'] = id_map - _LYRICS_INDEX_CACHE['reverse_id_map'] = reverse_id_map - _LYRICS_INDEX_CACHE['loaded'] = True - - logger.info(f"Lyrics index loaded from database with {len(id_map)} items.") - return True + loaded = load_voyager_index_from_db( + get_db(), 'lyrics_index_data', 'lyrics_index', + LYRICS_EMBEDDING_DIMENSION, VOYAGER_QUERY_EF, label='lyrics', + ) + if loaded is None: + return False + loaded_index, id_map, reverse_id_map = loaded + + _LYRICS_INDEX_CACHE['index'] = loaded_index + _LYRICS_INDEX_CACHE['id_map'] = id_map + _LYRICS_INDEX_CACHE['reverse_id_map'] = reverse_id_map + _LYRICS_INDEX_CACHE['loaded'] = True + + logger.info(f"Lyrics index loaded from database with {len(id_map)} items.") + return True except Exception as e: logger.error(f"Failed to load lyrics index from DB: {e}", exc_info=True) return False @@ -345,112 +163,31 @@ def _load_lyrics_axes_index_from_db() -> bool: """Load persisted voyager index for the lyrics axis vectors.""" from app_helper import get_db from config import VOYAGER_QUERY_EF + from .index_build_helpers import load_voyager_index_from_db columns = _axis_columns_from_axes() expected_dim = len(columns) try: - conn = get_db() - with conn.cursor() as cur: - cur.execute("SET LOCAL statement_timeout = 0") - cur.execute( - "SELECT index_data, id_map_json, embedding_dimension FROM lyrics_axes_index_data " - "WHERE index_name = %s", - ('lyrics_axes_index',), - ) - row = cur.fetchone() - - index_stream = None - try: - if row: - binary, id_map_json, db_dim = row - index_stream = tempfile.TemporaryFile() - index_stream.write(binary) - index_stream.seek(0) - else: - seg_pattern = re.compile(r'^lyrics_axes_index_(\d+)_(\d+)$') - parts = [] - total_expected = None - with conn.cursor(name='lyrics_axes_index_segments') as seg_cur: - seg_cur.itersize = 50 - seg_cur.execute( - "SELECT index_name, index_data, id_map_json, embedding_dimension " - "FROM lyrics_axes_index_data WHERE index_name LIKE %s ESCAPE '\\'", - (r'lyrics_axes_index\_%\_%',), - ) - for name, part_data, part_id_map, part_dim in seg_cur: - m = seg_pattern.match(name) - if not m: - continue - part_no = int(m.group(1)) - total = int(m.group(2)) - if total_expected is None: - total_expected = total - elif total_expected != total: - logger.error( - f"Lyrics axes index segment total mismatch: {total_expected} vs {total}" - ) - return False - parts.append((part_no, part_data, part_id_map, part_dim)) - - if total_expected is None or len(parts) != total_expected: - logger.info( - f"No complete persisted lyrics axes index found (expected {total_expected}, " - f"have {len(parts)})." - ) - return False - - parts.sort(key=lambda p: p[0]) - from .index_build_helpers import reassemble_segmented_id_map - db_dim = parts[0][3] - index_stream = tempfile.TemporaryFile() - for _, part_data, _, _ in parts: - index_stream.write(part_data) - index_stream.seek(0) - id_map_json = reassemble_segmented_id_map((p[0], p[2]) for p in parts) - - if index_stream is None: - return False - if db_dim != expected_dim: - logger.error( - f"Lyrics axes index dimension mismatch: db={db_dim} expected={expected_dim}" - ) - index_stream.close() - return False - - try: - import voyager # type: ignore - except ImportError: - logger.warning("Voyager library is unavailable; cannot load lyrics axes index.") - return False - - loaded_index = voyager.Index.load(index_stream) - loaded_index.ef = VOYAGER_QUERY_EF - finally: - if index_stream is not None: - try: - index_stream.close() - except Exception: - pass - - id_map = {int(k): v for k, v in json.loads(id_map_json).items()} - reverse_id_map = {v: k for k, v in id_map.items()} - - if not id_map: - logger.warning("Lyrics axes index id_map is empty.") - return False - - metadata_map = _fetch_lyrics_metadata(list(id_map.values())) - - _LYRICS_AXIS_CACHE['index'] = loaded_index - _LYRICS_AXIS_CACHE['id_map'] = id_map - _LYRICS_AXIS_CACHE['reverse_id_map'] = reverse_id_map - _LYRICS_AXIS_CACHE['axis_columns'] = columns - _LYRICS_AXIS_CACHE['metadata'] = metadata_map - _LYRICS_AXIS_CACHE['loaded'] = True - - logger.info(f"Lyrics axes index loaded from database with {len(id_map)} items.") - return True + loaded = load_voyager_index_from_db( + get_db(), 'lyrics_axes_index_data', 'lyrics_axes_index', + expected_dim, VOYAGER_QUERY_EF, label='lyrics axes', + ) + if loaded is None: + return False + loaded_index, id_map, reverse_id_map = loaded + + metadata_map = _fetch_lyrics_metadata(list(id_map.values())) + + _LYRICS_AXIS_CACHE['index'] = loaded_index + _LYRICS_AXIS_CACHE['id_map'] = id_map + _LYRICS_AXIS_CACHE['reverse_id_map'] = reverse_id_map + _LYRICS_AXIS_CACHE['axis_columns'] = columns + _LYRICS_AXIS_CACHE['metadata'] = metadata_map + _LYRICS_AXIS_CACHE['loaded'] = True + + logger.info(f"Lyrics axes index loaded from database with {len(id_map)} items.") + return True except Exception as e: logger.error(f"Failed to load lyrics axes index from DB: {e}", exc_info=True) return False diff --git a/tasks/mediaserver.py b/tasks/mediaserver.py deleted file mode 100644 index 1d3ee9e1..00000000 --- a/tasks/mediaserver.py +++ /dev/null @@ -1,413 +0,0 @@ -# tasks/mediaserver.py - -import logging -import os -import config # Import the config module to access server type and settings - -# Import the specific implementations -from tasks.mediaserver_jellyfin import ( - resolve_user as jellyfin_resolve_user, - get_all_playlists as jellyfin_get_all_playlists, - get_lyrics as jellyfin_get_lyrics, - delete_playlist as jellyfin_delete_playlist, - get_recent_albums as jellyfin_get_recent_albums, - get_tracks_from_album as jellyfin_get_tracks_from_album, - search_albums as jellyfin_search_albums, - test_connection as jellyfin_test_connection, - download_track as jellyfin_download_track, - get_all_songs as jellyfin_get_all_songs, - get_playlist_by_name as jellyfin_get_playlist_by_name, - create_playlist as jellyfin_create_playlist, - create_instant_playlist as jellyfin_create_instant_playlist, - create_or_replace_playlist as jellyfin_create_or_replace_playlist, - get_top_played_songs as jellyfin_get_top_played_songs, - get_last_played_time as jellyfin_get_last_played_time, - list_libraries as _jellyfin_list_libraries, -) -from tasks.mediaserver_navidrome import ( - get_all_playlists as navidrome_get_all_playlists, - get_lyrics as navidrome_get_lyrics, - delete_playlist as navidrome_delete_playlist, - get_recent_albums as navidrome_get_recent_albums, - get_tracks_from_album as navidrome_get_tracks_from_album, - search_albums as navidrome_search_albums, - test_connection as navidrome_test_connection, - download_track as navidrome_download_track, - get_all_songs as navidrome_get_all_songs, - get_playlist_by_name as navidrome_get_playlist_by_name, - create_playlist as navidrome_create_playlist, - create_instant_playlist as navidrome_create_instant_playlist, - create_or_replace_playlist as navidrome_create_or_replace_playlist, - get_top_played_songs as navidrome_get_top_played_songs, - get_last_played_time as navidrome_get_last_played_time, - list_libraries as _navidrome_list_libraries, -) -from tasks.mediaserver_lyrion import ( - get_all_playlists as lyrion_get_all_playlists, - get_lyrics as lyrion_get_lyrics, - delete_playlist as lyrion_delete_playlist, - get_recent_albums as lyrion_get_recent_albums, - get_tracks_from_album as lyrion_get_tracks_from_album, - search_albums as lyrion_search_albums, - test_connection as lyrion_test_connection, - download_track as lyrion_download_track, - get_all_songs as lyrion_get_all_songs, - get_playlist_by_name as lyrion_get_playlist_by_name, - create_playlist as lyrion_create_playlist, - create_instant_playlist as lyrion_create_instant_playlist, - create_or_replace_playlist as lyrion_create_or_replace_playlist, - get_top_played_songs as lyrion_get_top_played_songs, - get_last_played_time as lyrion_get_last_played_time, - list_libraries as _lyrion_list_libraries, -) -from tasks.mediaserver_mpd import ( - get_all_playlists as mpd_get_all_playlists, - delete_playlist as mpd_delete_playlist, - get_recent_albums as mpd_get_recent_albums, - get_tracks_from_album as mpd_get_tracks_from_album, - download_track as mpd_download_track, - get_all_songs as mpd_get_all_songs, - get_playlist_by_name as mpd_get_playlist_by_name, - create_playlist as mpd_create_playlist, - create_instant_playlist as mpd_create_instant_playlist, - get_top_played_songs as mpd_get_top_played_songs, - get_last_played_time as mpd_get_last_played_time, -) -from tasks.mediaserver_emby import ( - resolve_user as emby_resolve_user, - get_all_playlists as emby_get_all_playlists, - get_lyrics as emby_get_lyrics, - delete_playlist as emby_delete_playlist, - get_recent_albums as emby_get_recent_albums, - get_tracks_from_album as emby_get_tracks_from_album, - search_albums as emby_search_albums, - test_connection as emby_test_connection, - download_track as emby_download_track, - get_all_songs as emby_get_all_songs, - get_playlist_by_name as emby_get_playlist_by_name, - create_playlist as emby_create_playlist, - create_instant_playlist as emby_create_instant_playlist, - create_or_replace_playlist as emby_create_or_replace_playlist, - get_top_played_songs as emby_get_top_played_songs, - get_last_played_time as emby_get_last_played_time, - list_libraries as _emby_list_libraries, -) - -logger = logging.getLogger(__name__) - - -# ############################################################################## -# PUBLIC API (Dispatcher functions) -# ############################################################################## - -def resolve_emby_jellyfin_user(identifier, token): - """Public dispatcher for resolving a Jellyfin or Emby user identifier.""" - # This is specific to Jellyfin, so we call it directly. - if config.MEDIASERVER_TYPE == 'jellyfin': return jellyfin_resolve_user(identifier, token) - if config.MEDIASERVER_TYPE == 'emby': return emby_resolve_user(identifier, token) - return [] - -def _delete_matching_playlists(playlists_to_check, delete_function, suffix): - """Deletes every playlist whose name ends with the suffix; keeps going if one deletion fails.""" - deleted_count = 0 - for p in playlists_to_check: - # Navidrome uses 'id', others use 'Id'. Check for both. - playlist_id = p.get('Id') or p.get('id') - try: - if p.get('Name', '').endswith(suffix) and delete_function(playlist_id): - deleted_count += 1 - except Exception: - logger.exception(f"Failed to delete playlist {playlist_id}; continuing with the remaining playlists.") - return deleted_count - -def delete_playlists_by_suffix(suffix): - """Deletes all playlists whose name ends with the given suffix using admin credentials.""" - logger.info(f"Starting deletion of all '{suffix}' playlists.") - deleted_count = 0 - - playlists_to_check = [] - delete_function = None - - if config.MEDIASERVER_TYPE == 'jellyfin': - playlists_to_check = jellyfin_get_all_playlists() - delete_function = jellyfin_delete_playlist - elif config.MEDIASERVER_TYPE == 'navidrome': - playlists_to_check = navidrome_get_all_playlists() - delete_function = navidrome_delete_playlist - elif config.MEDIASERVER_TYPE == 'lyrion': - playlists_to_check = lyrion_get_all_playlists() - delete_function = lyrion_delete_playlist - elif config.MEDIASERVER_TYPE == 'mpd': - playlists_to_check = mpd_get_all_playlists() - delete_function = mpd_delete_playlist - elif config.MEDIASERVER_TYPE == 'emby': - playlists_to_check = emby_get_all_playlists() - delete_function = emby_delete_playlist - - if delete_function: - deleted_count = _delete_matching_playlists(playlists_to_check, delete_function, suffix) - - logger.info(f"Finished deletion. Deleted {deleted_count} playlists.") - -def delete_automatic_playlists(): - """Deletes all playlists ending with '_automatic' using admin credentials.""" - delete_playlists_by_suffix('_automatic') - -def get_recent_albums(limit): - """Fetches recently added albums using admin credentials.""" - if config.MEDIASERVER_TYPE == 'jellyfin': return jellyfin_get_recent_albums(limit) - if config.MEDIASERVER_TYPE == 'navidrome': return navidrome_get_recent_albums(limit) - if config.MEDIASERVER_TYPE == 'lyrion': return lyrion_get_recent_albums(limit) - if config.MEDIASERVER_TYPE == 'mpd': return mpd_get_recent_albums(limit) - if config.MEDIASERVER_TYPE == 'emby': return emby_get_recent_albums(limit) - return [] - -def get_tracks_from_album(album_id, user_creds=None, provider_type=None): - """Fetches tracks for an album, optionally using explicit creds.""" - provider_type = provider_type or config.MEDIASERVER_TYPE - if provider_type == 'jellyfin': return jellyfin_get_tracks_from_album(album_id, user_creds=user_creds) - if provider_type == 'navidrome': return navidrome_get_tracks_from_album(album_id, user_creds=user_creds) - if provider_type == 'lyrion': return lyrion_get_tracks_from_album(album_id, user_creds=user_creds) - if provider_type == 'mpd': return mpd_get_tracks_from_album(album_id) - if provider_type == 'emby': return emby_get_tracks_from_album(album_id, user_creds=user_creds) - return [] - -def download_track(temp_dir, item): - """Downloads a track using admin credentials. Detects format from file if .tmp extension is used.""" - downloaded_path = None - - if config.MEDIASERVER_TYPE == 'jellyfin': downloaded_path = jellyfin_download_track(temp_dir, item) - elif config.MEDIASERVER_TYPE == 'navidrome': downloaded_path = navidrome_download_track(temp_dir, item) - elif config.MEDIASERVER_TYPE == 'lyrion': downloaded_path = lyrion_download_track(temp_dir, item) - elif config.MEDIASERVER_TYPE == 'mpd': downloaded_path = mpd_download_track(temp_dir, item) - elif config.MEDIASERVER_TYPE == 'emby': downloaded_path = emby_download_track(temp_dir, item) - - # If download failed or returned None, return as is - if not downloaded_path: - return None - - # If file has .tmp extension, try to detect real format from file content - if downloaded_path.endswith('.tmp'): - try: - # Check if file exists before trying to detect format - if not os.path.exists(downloaded_path): - logger.warning(f"Downloaded file does not exist: {downloaded_path}") - return downloaded_path - - detected_ext = _detect_audio_format(downloaded_path) - if detected_ext and detected_ext != '.tmp': - new_path = downloaded_path.replace('.tmp', detected_ext) - # Check if target file already exists (avoid overwriting) - if os.path.exists(new_path): - logger.warning(f"Target file already exists, keeping .tmp: {new_path}") - return downloaded_path - os.rename(downloaded_path, new_path) - logger.info(f"Detected format and renamed: {os.path.basename(downloaded_path)} -> {os.path.basename(new_path)}") - return new_path - except Exception as e: - logger.debug(f"Format detection failed for {os.path.basename(downloaded_path)}, keeping .tmp: {e}") - - return downloaded_path - - -def _detect_audio_format(filepath): - """Detects audio format from file magic numbers. Returns extension like '.mp3' or '.flac'.""" - try: - with open(filepath, 'rb') as f: - header = f.read(12) - - # Check magic numbers for common audio formats - if len(header) < 4: - return '.tmp' - - # FLAC: fLaC - if header[:4] == b'fLaC': - return '.flac' - - # MP3: ID3 tag or MP3 sync bits - if header[:3] == b'ID3' or (len(header) >= 2 and header[0] == 0xFF and (header[1] & 0xE0) == 0xE0): - return '.mp3' - - # OGG: OggS - if header[:4] == b'OggS': - return '.ogg' - - # WAV/RIFF: RIFF....WAVE - if header[:4] == b'RIFF' and len(header) >= 12 and header[8:12] == b'WAVE': - return '.wav' - - # M4A/AAC: ftyp - if len(header) >= 8 and header[4:8] == b'ftyp': - return '.m4a' - - # WMA: ASF header - if header[:4] == b'\x30\x26\xb2\x75': - return '.wma' - - logger.debug(f"Unknown audio format, header: {header[:4].hex()}") - return '.tmp' - - except Exception as e: - logger.debug(f"Error detecting audio format: {e}") - return '.tmp' - -def get_all_songs(user_creds=None, provider_type=None, apply_filter=True): - """Fetches all songs using admin credentials or explicit creds. - - ``apply_filter`` is forwarded to providers that honor - ``config.MUSIC_LIBRARIES`` (currently Navidrome). Migration probes pass - ``apply_filter=False`` so the source provider's library filter does not - falsely exclude tracks from the target server during dry-run. - """ - provider_type = provider_type or config.MEDIASERVER_TYPE - if provider_type == 'jellyfin': return jellyfin_get_all_songs(user_creds=user_creds) - if provider_type == 'navidrome': return navidrome_get_all_songs(user_creds=user_creds, apply_filter=apply_filter) - if provider_type == 'lyrion': return lyrion_get_all_songs(user_creds=user_creds) - if provider_type == 'mpd': return mpd_get_all_songs() - if provider_type == 'emby': return emby_get_all_songs(user_creds=user_creds) - return [] - -def list_libraries(user_creds=None, provider_type=None): - """List all music libraries/folders a provider exposes. - - Returns {'libraries': [{'id': str, 'name': str}, ...], 'unsupported': bool}. - The setup wizard and migration assistant use this to render a checkbox list - after a successful test-connection. Uses admin credentials when - ``user_creds`` is None, or the supplied creds when probing a target. - """ - provider_type = provider_type or config.MEDIASERVER_TYPE - if provider_type == 'jellyfin': return {'libraries': _jellyfin_list_libraries(user_creds=user_creds), 'unsupported': False} - if provider_type == 'navidrome': return {'libraries': _navidrome_list_libraries(user_creds=user_creds), 'unsupported': False} - if provider_type == 'lyrion': return {'libraries': _lyrion_list_libraries(user_creds=user_creds), 'unsupported': False} - if provider_type == 'emby': return {'libraries': _emby_list_libraries(user_creds=user_creds), 'unsupported': False} - return {'libraries': [], 'unsupported': True} - -def search_albums(query, user_creds=None, provider_type=None): - """Searches for albums using admin credentials or explicit creds.""" - provider_type = provider_type or config.MEDIASERVER_TYPE - if provider_type == 'jellyfin': return jellyfin_search_albums(query, user_creds=user_creds) - if provider_type == 'navidrome': return navidrome_search_albums(query, user_creds=user_creds) - if provider_type == 'lyrion': return lyrion_search_albums(query, user_creds=user_creds) - if provider_type == 'mpd': raise NotImplementedError('MPD album search is not supported') - if provider_type == 'emby': return emby_search_albums(query, user_creds=user_creds) - return [] - -def test_connection(user_creds=None, provider_type=None): - """Tests provider connection using admin credentials or explicit creds.""" - provider_type = provider_type or config.MEDIASERVER_TYPE - if provider_type == 'jellyfin': return jellyfin_test_connection(user_creds=user_creds) - if provider_type == 'navidrome': return navidrome_test_connection(user_creds=user_creds) - if provider_type == 'lyrion': return lyrion_test_connection(user_creds=user_creds) - if provider_type == 'mpd': - return {'ok': False, 'error': 'MPD migration probe is not supported', 'sample_count': 0, 'path_format': 'none', 'warnings': []} - if provider_type == 'emby': return emby_test_connection(user_creds=user_creds) - return {'ok': False, 'error': f"Provider '{provider_type}' not supported", 'sample_count': 0, 'path_format': 'none', 'warnings': []} - -def get_playlist_by_name(playlist_name): - """Finds a playlist by name using admin credentials.""" - if not playlist_name: raise ValueError("Playlist name is required.") - if config.MEDIASERVER_TYPE == 'jellyfin': return jellyfin_get_playlist_by_name(playlist_name) - if config.MEDIASERVER_TYPE == 'navidrome': return navidrome_get_playlist_by_name(playlist_name) - if config.MEDIASERVER_TYPE == 'lyrion': return lyrion_get_playlist_by_name(playlist_name) - if config.MEDIASERVER_TYPE == 'mpd': return mpd_get_playlist_by_name(playlist_name) - if config.MEDIASERVER_TYPE == 'emby': return emby_get_playlist_by_name(playlist_name) - return None - -def create_playlist(base_name, item_ids): - """Creates a playlist using admin credentials.""" - if not base_name: raise ValueError("Playlist name is required.") - if not item_ids: raise ValueError("Track IDs are required.") - if config.MEDIASERVER_TYPE == 'jellyfin': jellyfin_create_playlist(base_name, item_ids) - elif config.MEDIASERVER_TYPE == 'navidrome': navidrome_create_playlist(base_name, item_ids) - elif config.MEDIASERVER_TYPE == 'lyrion': lyrion_create_playlist(base_name, item_ids) - elif config.MEDIASERVER_TYPE == 'mpd': mpd_create_playlist(base_name, item_ids) - elif config.MEDIASERVER_TYPE == 'emby': emby_create_playlist(base_name, item_ids) - -def create_instant_playlist(playlist_name, item_ids, user_creds=None): - """Creates an instant playlist. Uses user_creds if provided, otherwise admin.""" - if not playlist_name: raise ValueError("Playlist name is required.") - if not item_ids: raise ValueError("Track IDs are required.") - - if config.MEDIASERVER_TYPE == 'jellyfin': - return jellyfin_create_instant_playlist(playlist_name, item_ids, user_creds) - if config.MEDIASERVER_TYPE == 'navidrome': - return navidrome_create_instant_playlist(playlist_name, item_ids, user_creds) - if config.MEDIASERVER_TYPE == 'lyrion': - return lyrion_create_instant_playlist(playlist_name, item_ids) - if config.MEDIASERVER_TYPE == 'mpd': - return mpd_create_instant_playlist(playlist_name, item_ids, user_creds) - if config.MEDIASERVER_TYPE == 'emby': - return emby_create_instant_playlist(playlist_name, item_ids, user_creds) - return None - - -def create_or_replace_playlist(playlist_name, item_ids, user_creds=None): - """Cron-only upsert: create the playlist if missing, or replace its contents in place. - - Used by the scheduled sonic_fingerprint task so the same server-side playlist (and ID, - where the backend allows) gets reused across runs. Raises NotImplementedError for MPD - and any other unsupported backend — the cron handler catches that and falls back to - legacy date-suffixed playlist creation. - """ - if not playlist_name: - raise ValueError("Playlist name is required.") - if not item_ids: - raise ValueError("Track IDs are required.") - - if config.MEDIASERVER_TYPE == 'jellyfin': - return jellyfin_create_or_replace_playlist(playlist_name, item_ids, user_creds) - if config.MEDIASERVER_TYPE == 'navidrome': - return navidrome_create_or_replace_playlist(playlist_name, item_ids, user_creds) - if config.MEDIASERVER_TYPE == 'emby': - return emby_create_or_replace_playlist(playlist_name, item_ids, user_creds) - if config.MEDIASERVER_TYPE == 'lyrion': - return lyrion_create_or_replace_playlist(playlist_name, item_ids, user_creds) - raise NotImplementedError( - f"create_or_replace_playlist not supported for MEDIASERVER_TYPE={config.MEDIASERVER_TYPE!r}" - ) - -def get_top_played_songs(limit, user_creds=None): - """Fetches top played songs. Uses user_creds if provided, otherwise admin.""" - if config.MEDIASERVER_TYPE == 'jellyfin': - return jellyfin_get_top_played_songs(limit, user_creds) - if config.MEDIASERVER_TYPE == 'navidrome': - return navidrome_get_top_played_songs(limit, user_creds) - if config.MEDIASERVER_TYPE == 'lyrion': - return lyrion_get_top_played_songs(limit) - if config.MEDIASERVER_TYPE == 'mpd': - return mpd_get_top_played_songs(limit, user_creds) - if config.MEDIASERVER_TYPE == 'emby': - return emby_get_top_played_songs(limit, user_creds) - return [] - -def get_last_played_time(item_id, user_creds=None): - """Fetches last played time for a track. Uses user_creds if provided, otherwise admin.""" - if config.MEDIASERVER_TYPE == 'jellyfin': - return jellyfin_get_last_played_time(item_id, user_creds) - if config.MEDIASERVER_TYPE == 'navidrome': - return navidrome_get_last_played_time(item_id, user_creds) - if config.MEDIASERVER_TYPE == 'lyrion': - return lyrion_get_last_played_time(item_id) - if config.MEDIASERVER_TYPE == 'mpd': - return mpd_get_last_played_time(item_id, user_creds) - if config.MEDIASERVER_TYPE == 'emby': - return emby_get_last_played_time(item_id, user_creds) - return None - -def get_lyrics(track_id: str, timeout: float = 2.5): - """Fetch lyrics embedded in the media server for a given track ID. - - Supported servers: Jellyfin, Emby, Navidrome, Lyrion. - MPD does not provide a lyrics API; always returns None. - Returns plain text or None. - """ - if config.MEDIASERVER_TYPE == 'jellyfin': - return jellyfin_get_lyrics(track_id, timeout=timeout) - if config.MEDIASERVER_TYPE == 'emby': - return emby_get_lyrics(track_id, timeout=timeout) - if config.MEDIASERVER_TYPE == 'navidrome': - return navidrome_get_lyrics(track_id, timeout=timeout) - if config.MEDIASERVER_TYPE == 'lyrion': - return lyrion_get_lyrics(track_id, timeout=timeout) - return None - diff --git a/tasks/mediaserver/__init__.py b/tasks/mediaserver/__init__.py new file mode 100644 index 00000000..e25b358c --- /dev/null +++ b/tasks/mediaserver/__init__.py @@ -0,0 +1,283 @@ +# tasks/mediaserver/__init__.py + +import logging +import os +from importlib import import_module + +import config + +logger = logging.getLogger(__name__) + +_PROVIDER_NAMES = ('jellyfin', 'navidrome', 'lyrion', 'emby') +_warned_unsupported = set() + +_PLAYLIST_NAME_REQUIRED = "Playlist name is required." +_TRACK_IDS_REQUIRED = "Track IDs are required." + + +def _provider(provider_type=None): + """Return the backend module for the given (or configured) provider type. + + Provider modules are imported lazily on first use so that importing + ``tasks.mediaserver`` does not load the four inactive backends or + initialize their HTTP sessions. Returns None for unsupported types, + matching the old dispatcher fall-through behavior. + """ + name = provider_type or config.MEDIASERVER_TYPE + if name not in _PROVIDER_NAMES: + if name not in _warned_unsupported: + _warned_unsupported.add(name) + logger.warning( + "Unsupported MEDIASERVER_TYPE %r (supported: %s); media-server operations are no-ops.", + name, ', '.join(_PROVIDER_NAMES)) + return None + return import_module('.' + name, __name__) + + +# ############################################################################## +# PUBLIC API (Dispatcher functions) +# ############################################################################## + +def resolve_emby_jellyfin_user(identifier, token): + """Public dispatcher for resolving a Jellyfin or Emby user identifier.""" + if config.MEDIASERVER_TYPE in ('jellyfin', 'emby'): + return _provider().resolve_user(identifier, token) + return [] + +def _delete_matching_playlists(playlists_to_check, delete_function, suffix): + """Deletes every playlist whose name ends with the suffix; keeps going if one deletion fails.""" + deleted_count = 0 + for p in playlists_to_check: + # Navidrome uses 'id', others use 'Id'. Check for both. + playlist_id = p.get('Id') or p.get('id') + try: + if p.get('Name', '').endswith(suffix) and delete_function(playlist_id): + deleted_count += 1 + except Exception: + logger.exception(f"Failed to delete playlist {playlist_id}; continuing with the remaining playlists.") + return deleted_count + +def delete_playlists_by_suffix(suffix): + """Deletes all playlists whose name ends with the given suffix using admin credentials.""" + logger.info(f"Starting deletion of all '{suffix}' playlists.") + deleted_count = 0 + + provider = _provider() + if provider is not None: + deleted_count = _delete_matching_playlists(provider.get_all_playlists(), provider.delete_playlist, suffix) + + logger.info(f"Finished deletion. Deleted {deleted_count} playlists.") + +def delete_automatic_playlists(): + """Deletes all playlists ending with '_automatic' using admin credentials.""" + delete_playlists_by_suffix('_automatic') + +def get_recent_albums(limit): + """Fetches recently added albums using admin credentials.""" + provider = _provider() + if provider is None: + return [] + return provider.get_recent_albums(limit) + +def get_tracks_from_album(album_id, user_creds=None, provider_type=None): + """Fetches tracks for an album, optionally using explicit creds.""" + provider = _provider(provider_type) + if provider is None: + return [] + return provider.get_tracks_from_album(album_id, user_creds=user_creds) + +def download_track(temp_dir, item): + """Downloads a track using admin credentials. Detects format from file if .tmp extension is used.""" + provider = _provider() + downloaded_path = provider.download_track(temp_dir, item) if provider is not None else None + + # If download failed or returned None, return as is + if not downloaded_path: + return None + + # If file has .tmp extension, try to detect real format from file content + if downloaded_path.endswith('.tmp'): + try: + # Check if file exists before trying to detect format + if not os.path.exists(downloaded_path): + logger.warning(f"Downloaded file does not exist: {downloaded_path}") + return downloaded_path + + detected_ext = _detect_audio_format(downloaded_path) + if detected_ext and detected_ext != '.tmp': + new_path = downloaded_path.replace('.tmp', detected_ext) + # Check if target file already exists (avoid overwriting) + if os.path.exists(new_path): + logger.warning(f"Target file already exists, keeping .tmp: {new_path}") + return downloaded_path + os.rename(downloaded_path, new_path) + logger.info(f"Detected format and renamed: {os.path.basename(downloaded_path)} -> {os.path.basename(new_path)}") + return new_path + except Exception as e: + logger.debug(f"Format detection failed for {os.path.basename(downloaded_path)}, keeping .tmp: {e}") + + return downloaded_path + + +def _detect_audio_format(filepath): + """Detects audio format from file magic numbers. Returns extension like '.mp3' or '.flac'.""" + try: + with open(filepath, 'rb') as f: + header = f.read(12) + + # Check magic numbers for common audio formats + if len(header) < 4: + return '.tmp' + + # FLAC: fLaC + if header[:4] == b'fLaC': + return '.flac' + + # MP3: ID3 tag or MP3 sync bits + if header[:3] == b'ID3' or (len(header) >= 2 and header[0] == 0xFF and (header[1] & 0xE0) == 0xE0): + return '.mp3' + + # OGG: OggS + if header[:4] == b'OggS': + return '.ogg' + + # WAV/RIFF: RIFF....WAVE + if header[:4] == b'RIFF' and len(header) >= 12 and header[8:12] == b'WAVE': + return '.wav' + + # M4A/AAC: ftyp + if len(header) >= 8 and header[4:8] == b'ftyp': + return '.m4a' + + # WMA: ASF header + if header[:4] == b'\x30\x26\xb2\x75': + return '.wma' + + logger.debug(f"Unknown audio format, header: {header[:4].hex()}") + return '.tmp' + + except Exception as e: + logger.debug(f"Error detecting audio format: {e}") + return '.tmp' + +def get_all_songs(user_creds=None, provider_type=None, apply_filter=True): + """Fetches all songs using admin credentials or explicit creds. + + ``apply_filter`` is forwarded to providers that honor + ``config.MUSIC_LIBRARIES`` (currently Navidrome). Migration probes pass + ``apply_filter=False`` so the source provider's library filter does not + falsely exclude tracks from the target server during dry-run. + """ + provider_type = provider_type or config.MEDIASERVER_TYPE + provider = _provider(provider_type) + if provider is None: + return [] + if provider_type == 'navidrome': + return provider.get_all_songs(user_creds=user_creds, apply_filter=apply_filter) + return provider.get_all_songs(user_creds=user_creds) + +def list_libraries(user_creds=None, provider_type=None): + """List all music libraries/folders a provider exposes. + + Returns {'libraries': [{'id': str, 'name': str}, ...], 'unsupported': bool}. + The setup wizard and migration assistant use this to render a checkbox list + after a successful test-connection. Uses admin credentials when + ``user_creds`` is None, or the supplied creds when probing a target. + """ + provider = _provider(provider_type) + if provider is None: + return {'libraries': [], 'unsupported': True} + return {'libraries': provider.list_libraries(user_creds=user_creds), 'unsupported': False} + +def search_albums(query, user_creds=None, provider_type=None): + """Searches for albums using admin credentials or explicit creds.""" + provider = _provider(provider_type) + if provider is None: + return [] + return provider.search_albums(query, user_creds=user_creds) + +def test_connection(user_creds=None, provider_type=None): + """Tests provider connection using admin credentials or explicit creds.""" + provider_type = provider_type or config.MEDIASERVER_TYPE + provider = _provider(provider_type) + if provider is None: + return {'ok': False, 'error': f"Provider '{provider_type}' not supported", 'sample_count': 0, 'path_format': 'none', 'warnings': []} + return provider.test_connection(user_creds=user_creds) + +def get_playlist_by_name(playlist_name): + """Finds a playlist by name using admin credentials.""" + if not playlist_name: raise ValueError(_PLAYLIST_NAME_REQUIRED) + provider = _provider() + if provider is None: + return None + return provider.get_playlist_by_name(playlist_name) + +def create_playlist(base_name, item_ids): + """Creates a playlist using admin credentials.""" + if not base_name: raise ValueError(_PLAYLIST_NAME_REQUIRED) + if not item_ids: raise ValueError(_TRACK_IDS_REQUIRED) + provider = _provider() + if provider is not None: + provider.create_playlist(base_name, item_ids) + +def create_instant_playlist(playlist_name, item_ids, user_creds=None): + """Creates an instant playlist. Uses user_creds if provided, otherwise admin.""" + if not playlist_name: raise ValueError(_PLAYLIST_NAME_REQUIRED) + if not item_ids: raise ValueError(_TRACK_IDS_REQUIRED) + + provider = _provider() + if provider is None: + return None + if config.MEDIASERVER_TYPE == 'lyrion': + return provider.create_instant_playlist(playlist_name, item_ids) + return provider.create_instant_playlist(playlist_name, item_ids, user_creds) + + +def create_or_replace_playlist(playlist_name, item_ids, user_creds=None): + """Cron-only upsert: create the playlist if missing, or replace its contents in place. + + Used by the scheduled sonic_fingerprint task so the same server-side playlist (and ID, + where the backend allows) gets reused across runs. Raises NotImplementedError for any + unsupported backend — the cron handler catches that and falls back to legacy + date-suffixed playlist creation. + """ + if not playlist_name: + raise ValueError(_PLAYLIST_NAME_REQUIRED) + if not item_ids: + raise ValueError(_TRACK_IDS_REQUIRED) + + provider = _provider() + if provider is None: + raise NotImplementedError( + f"create_or_replace_playlist not supported for MEDIASERVER_TYPE={config.MEDIASERVER_TYPE!r}" + ) + return provider.create_or_replace_playlist(playlist_name, item_ids, user_creds) + +def get_top_played_songs(limit, user_creds=None): + """Fetches top played songs. Uses user_creds if provided, otherwise admin.""" + provider = _provider() + if provider is None: + return [] + if config.MEDIASERVER_TYPE == 'lyrion': + return provider.get_top_played_songs(limit) + return provider.get_top_played_songs(limit, user_creds) + +def get_last_played_time(item_id, user_creds=None): + """Fetches last played time for a track. Uses user_creds if provided, otherwise admin.""" + provider = _provider() + if provider is None: + return None + if config.MEDIASERVER_TYPE == 'lyrion': + return provider.get_last_played_time(item_id) + return provider.get_last_played_time(item_id, user_creds) + +def get_lyrics(track_id: str, timeout: float = 2.5): + """Fetch lyrics embedded in the media server for a given track ID. + + Supported servers: Jellyfin, Emby, Navidrome, Lyrion. + Returns plain text or None. + """ + provider = _provider() + if provider is None: + return None + return provider.get_lyrics(track_id, timeout=timeout) diff --git a/tasks/mediaserver_emby.py b/tasks/mediaserver/emby.py similarity index 95% rename from tasks/mediaserver_emby.py rename to tasks/mediaserver/emby.py index 39bd57f1..ffe8fcb3 100644 --- a/tasks/mediaserver_emby.py +++ b/tasks/mediaserver/emby.py @@ -1,11 +1,12 @@ -# tasks/mediaserver_emby.py +# tasks/mediaserver/emby.py -from tasks import mediaserver_http as requests +from . import http as requests import logging import os import config -from tasks.mediaserver_helper import detect_path_format +from .helper import detect_path_format, detect_download_extension +from .helper import select_best_artist as _select_best_artist logger = logging.getLogger(__name__) @@ -218,7 +219,7 @@ def _get_recent_standalone_tracks(limit, target_library_ids=None, user_creds=Non # If parent is not a MusicAlbum, treat track as standalone if parent_info.get('Type') != 'MusicAlbum': standalone_tracks.append(track) - except: + except Exception: # If we can't check parent, assume it's standalone to be safe standalone_tracks.append(track) @@ -273,7 +274,7 @@ def _get_recent_standalone_tracks(limit, target_library_ids=None, user_creds=Non parent_info = parent_r.json() if parent_info.get('Type') != 'MusicAlbum': standalone_tracks.append(track) - except: + except Exception: standalone_tracks.append(track) all_tracks.extend(standalone_tracks) @@ -502,22 +503,7 @@ def download_track(temp_dir, item): # https://dev.emby.media/reference/RestAPI/LibraryService/getItemsByIdDownload.html try: track_id = item['Id'] - - # Try to get format from Container field first (most reliable) - file_extension = '.tmp' - try: - container = item.get('Container') - if container and isinstance(container, str) and container.strip(): - # Ensure container value is safe (no path separators, etc.) - safe_container = container.strip().replace('/', '').replace('\\', '') - if safe_container: - file_extension = f".{safe_container}" - logger.debug(f"Using Container field for format: {file_extension}") - elif item.get('Path'): - file_extension = os.path.splitext(item['Path'])[1] or '.tmp' - except Exception as e: - logger.debug(f"Error getting format from Container/Path, using .tmp: {e}") - + file_extension = detect_download_extension(item) download_url = f"{config.EMBY_URL}/emby/Items/{track_id}/Download" local_filename = os.path.join(temp_dir, f"{track_id}{file_extension}") with requests.get(download_url, headers=config.HEADERS, stream=True, timeout=REQUESTS_TIMEOUT) as r: @@ -530,29 +516,6 @@ def download_track(temp_dir, item): logger.error(f"Failed to download track {item.get('Name', 'Unknown')}: {e}", exc_info=True) return None -def _select_best_artist(item, title="Unknown"): - """ - Selects the best artist field from Emby item, prioritizing track artists over album artists. - This helps avoid "Various Artists" issues in compilation albums. - Returns tuple: (artist_name, artist_id) - """ - # Priority: Artists array (track artists) > AlbumArtist > fallback - # Emby provides ArtistItems array with Id and Name - if item.get('ArtistItems') and len(item['ArtistItems']) > 0: - track_artist = item['ArtistItems'][0].get('Name', 'Unknown Artist') - artist_id = item['ArtistItems'][0].get('Id') - elif item.get('Artists') and len(item['Artists']) > 0: - track_artist = item['Artists'][0] # Take first artist if multiple - artist_id = None - elif item.get('AlbumArtist'): - track_artist = item['AlbumArtist'] - artist_id = None - else: - track_artist = 'Unknown Artist' - artist_id = None - - return track_artist, artist_id - def get_all_songs(user_creds=None): # Emby might have a maximum number of items returned per request. # not sure if this approach would work.. It defnitly needs testing. diff --git a/tasks/mediaserver/helper.py b/tasks/mediaserver/helper.py new file mode 100644 index 00000000..0187baec --- /dev/null +++ b/tasks/mediaserver/helper.py @@ -0,0 +1,94 @@ +"""Shared media server helper utilities.""" + +import logging +import os +import re + +logger = logging.getLogger(__name__) + + +def select_best_artist(item, title="Unknown"): + """ + Selects the best artist field from a Jellyfin/Emby item, prioritizing track + artists over album artists. This helps avoid "Various Artists" issues in + compilation albums. + Returns tuple: (artist_name, artist_id) + """ + # Priority: Artists array (track artists) > AlbumArtist > fallback + # Jellyfin/Emby provides ArtistItems array with Id and Name + if item.get('ArtistItems') and len(item['ArtistItems']) > 0: + track_artist = item['ArtistItems'][0].get('Name', 'Unknown Artist') + artist_id = item['ArtistItems'][0].get('Id') + elif item.get('Artists') and len(item['Artists']) > 0: + track_artist = item['Artists'][0] # Take first artist if multiple + artist_id = None + elif item.get('AlbumArtist'): + track_artist = item['AlbumArtist'] + artist_id = None + else: + track_artist = 'Unknown Artist' + artist_id = None + + return track_artist, artist_id + + +def detect_download_extension(item): + """Derive a file extension for a Jellyfin/Emby download. + + Prefers the item's Container field (most reliable), falls back to the + Path extension, and defaults to '.tmp' so the dispatcher's magic-number + sniffing can rename the file after download. + """ + file_extension = '.tmp' + try: + container = item.get('Container') + if container and isinstance(container, str) and container.strip(): + # Ensure container value is safe (no path separators, etc.) + safe_container = container.strip().replace('/', '').replace('\\', '') + if safe_container: + file_extension = f".{safe_container}" + logger.debug(f"Using Container field for format: {file_extension}") + elif item.get('Path'): + file_extension = os.path.splitext(item['Path'])[1] or '.tmp' + except Exception as e: + logger.debug(f"Error getting format from Container/Path, using .tmp: {e}") + return file_extension + + +def detect_path_format(tracks): + """Classify track path samples as absolute, relative, none, or mixed.""" + def _is_absolute_path(path): + if not path: + return False + path_str = str(path) + lower = path_str.lower() + return ( + path_str.startswith('/') + or path_str.startswith('\\') + or lower.startswith('file://') + or re.match(r'^[A-Za-z]:[\\/]', path_str) + ) + + paths = [] + for track in tracks or []: + if not isinstance(track, dict): + continue + # Support lowercase/uppercase path keys and legacy URL fields. + path = ( + track.get('path') + or track.get('Path') + or track.get('url') + or track.get('Url') + ) + if path: + paths.append(path) + + if not paths: + return 'none' + + ratio = sum(1 for p in paths if _is_absolute_path(p)) / len(paths) + if ratio >= 0.8: + return 'absolute' + if ratio <= 0.2: + return 'relative' + return 'mixed' diff --git a/tasks/mediaserver_http.py b/tasks/mediaserver/http.py similarity index 97% rename from tasks/mediaserver_http.py rename to tasks/mediaserver/http.py index 567de696..1a64ee4b 100644 --- a/tasks/mediaserver_http.py +++ b/tasks/mediaserver/http.py @@ -1,4 +1,4 @@ -# tasks/mediaserver_http.py +# tasks/mediaserver/http.py """Centralized HTTP layer for the media-server clients. Drop-in stand-in for the parts of ``requests`` the media-server modules use. @@ -6,7 +6,7 @@ import requests -> - from tasks import mediaserver_http as requests + from . import http as requests Every existing ``requests.get(...)`` / ``requests.post(...)`` / etc. call then gains a *connection-only* retry, and any other attribute diff --git a/tasks/mediaserver_jellyfin.py b/tasks/mediaserver/jellyfin.py similarity index 93% rename from tasks/mediaserver_jellyfin.py rename to tasks/mediaserver/jellyfin.py index 42663b23..7d2d2549 100644 --- a/tasks/mediaserver_jellyfin.py +++ b/tasks/mediaserver/jellyfin.py @@ -1,11 +1,12 @@ -# tasks/mediaserver_jellyfin.py +# tasks/mediaserver/jellyfin.py -from tasks import mediaserver_http as requests +from . import http as requests import logging import os import config -from tasks.mediaserver_helper import detect_path_format +from .helper import detect_path_format, detect_download_extension +from .helper import select_best_artist as _select_best_artist logger = logging.getLogger(__name__) @@ -259,22 +260,7 @@ def download_track(temp_dir, item): """Downloads a single track from Jellyfin using admin credentials.""" try: track_id = item['Id'] - - # Try to get format from Container field first (most reliable) - file_extension = '.tmp' - try: - container = item.get('Container') - if container and isinstance(container, str) and container.strip(): - # Ensure container value is safe (no path separators, etc.) - safe_container = container.strip().replace('/', '').replace('\\', '') - if safe_container: - file_extension = f".{safe_container}" - logger.debug(f"Using Container field for format: {file_extension}") - elif item.get('Path'): - file_extension = os.path.splitext(item['Path'])[1] or '.tmp' - except Exception as e: - logger.debug(f"Error getting format from Container/Path, using .tmp: {e}") - + file_extension = detect_download_extension(item) download_url = f"{config.JELLYFIN_URL}/Items/{track_id}/Download" local_filename = os.path.join(temp_dir, f"{track_id}{file_extension}") with requests.get(download_url, headers=config.HEADERS, stream=True, timeout=REQUESTS_TIMEOUT) as r: @@ -287,29 +273,6 @@ def download_track(temp_dir, item): logger.error(f"Failed to download track {item.get('Name', 'Unknown')}: {e}", exc_info=True) return None -def _select_best_artist(item, title="Unknown"): - """ - Selects the best artist field from Jellyfin item, prioritizing track artists over album artists. - This helps avoid "Various Artists" issues in compilation albums. - Returns tuple: (artist_name, artist_id) - """ - # Priority: Artists array (track artists) > AlbumArtist > fallback - # Jellyfin/Emby provides ArtistItems array with Id and Name - if item.get('ArtistItems') and len(item['ArtistItems']) > 0: - track_artist = item['ArtistItems'][0].get('Name', 'Unknown Artist') - artist_id = item['ArtistItems'][0].get('Id') - elif item.get('Artists') and len(item['Artists']) > 0: - track_artist = item['Artists'][0] # Take first artist if multiple - artist_id = None - elif item.get('AlbumArtist'): - track_artist = item['AlbumArtist'] - artist_id = None - else: - track_artist = 'Unknown Artist' - artist_id = None - - return track_artist, artist_id - def get_all_songs(user_creds=None): """Fetches all songs from Jellyfin using admin or override credentials, paginated. diff --git a/tasks/mediaserver_lyrion.py b/tasks/mediaserver/lyrion.py similarity index 99% rename from tasks/mediaserver_lyrion.py rename to tasks/mediaserver/lyrion.py index e3f8e6f1..968edc9d 100644 --- a/tasks/mediaserver_lyrion.py +++ b/tasks/mediaserver/lyrion.py @@ -1,12 +1,12 @@ -# tasks/mediaserver_lyrion.py +# tasks/mediaserver/lyrion.py -from tasks import mediaserver_http as requests +from . import http as requests import logging import os from urllib.parse import unquote, urlparse import config -from tasks.mediaserver_helper import detect_path_format +from .helper import detect_path_format logger = logging.getLogger(__name__) @@ -835,7 +835,7 @@ def _add_to_playlist(playlist_id, item_ids): return False # Method: Load playlist to player, add tracks, then use playlists edit to update - logger.info(f"Using method: Load → Add → Update original playlist via edit command") + logger.info("Using method: Load -> Add -> Update original playlist via edit command") # Step 1: Load the saved playlist into the player's current playlist logger.debug(f"Step 1: Loading playlist {playlist_id} to player {player_id}") diff --git a/tasks/mediaserver_navidrome.py b/tasks/mediaserver/navidrome.py similarity index 99% rename from tasks/mediaserver_navidrome.py rename to tasks/mediaserver/navidrome.py index 86eeadde..f085db5f 100644 --- a/tasks/mediaserver_navidrome.py +++ b/tasks/mediaserver/navidrome.py @@ -1,12 +1,12 @@ -# tasks/mediaserver_navidrome.py +# tasks/mediaserver/navidrome.py -from tasks import mediaserver_http as requests +from . import http as requests import logging import os import random import config -from tasks.mediaserver_helper import detect_path_format +from .helper import detect_path_format logger = logging.getLogger(__name__) diff --git a/tasks/mediaserver_helper.py b/tasks/mediaserver_helper.py deleted file mode 100644 index 34005c05..00000000 --- a/tasks/mediaserver_helper.py +++ /dev/null @@ -1,42 +0,0 @@ -"""Shared media server helper utilities.""" - -import re - - -def detect_path_format(tracks): - """Classify track path samples as absolute, relative, none, or mixed.""" - def _is_absolute_path(path): - if not path: - return False - path_str = str(path) - lower = path_str.lower() - return ( - path_str.startswith('/') - or path_str.startswith('\\') - or lower.startswith('file://') - or re.match(r'^[A-Za-z]:[\\/]', path_str) - ) - - paths = [] - for track in tracks or []: - if not isinstance(track, dict): - continue - # Support lowercase/uppercase path keys and legacy URL fields. - path = ( - track.get('path') - or track.get('Path') - or track.get('url') - or track.get('Url') - ) - if path: - paths.append(path) - - if not paths: - return 'none' - - ratio = sum(1 for p in paths if _is_absolute_path(p)) / len(paths) - if ratio >= 0.8: - return 'absolute' - if ratio <= 0.2: - return 'relative' - return 'mixed' diff --git a/tasks/mediaserver_mpd.py b/tasks/mediaserver_mpd.py deleted file mode 100644 index e4b6ed28..00000000 --- a/tasks/mediaserver_mpd.py +++ /dev/null @@ -1,307 +0,0 @@ -# tasks/mediaserver_mpd.py - -import logging -import os -from datetime import datetime -import config -from tasks import mediaserver_http as requests # <-- shared retrying HTTP session -import random # <-- ADDED: For shuffling albums - -# Add the MPD client library dependency -# NOTE: This implementation requires the 'python-mpd2' library. -# Install it with: pip install python-mpd2 -try: - import mpd -except ImportError: - # Handle the case where the library isn't installed. - # You might want to log a more prominent warning or exit if MPD is the configured server type. - pass - -logger = logging.getLogger(__name__) - -# ############################################################################## -# MPD (MUSIC PLAYER DAEMON) IMPLEMENTATION -# ############################################################################## - -def _connect(): - """Establishes a connection to the MPD server.""" - # Set use_unicode=True to ensure all communication with the server, - # including file paths, is handled as UTF-8. - client = mpd.MPDClient(use_unicode=True) - client.timeout = 60 - client.idletimeout = 30 - - - try: - logger.info(f"Calling MPD connect('{config.MPD_HOST}', {config.MPD_PORT}, timeout=None)") - client.connect(config.MPD_HOST, config.MPD_PORT) - logger.info(f"Successfully connected to MPD server. Status: {client.status()}") - - if config.MPD_PASSWORD: - logger.info("Authenticating with MPD password") - client.password(config.MPD_PASSWORD) - - return client - except Exception as e: - logger.error(f"Failed to connect or configure MPD server: {e}", exc_info=True) - _disconnect_safely(client) - return None - -def _format_song(song_dict): - """Formats an MPD song dictionary to the standard format used in this script.""" - # The 'Id' will be the file path, which is unique. - return { - 'Id': song_dict.get('file'), - 'Name': song_dict.get('title', os.path.basename(song_dict.get('file', ''))), - 'AlbumArtist': song_dict.get('albumartist'), - 'OriginalAlbumArtist': song_dict.get('albumartist'), - 'Artist': song_dict.get('artist'), - 'Album': song_dict.get('album'), - 'Path': song_dict.get('file'), - 'last-modified': song_dict.get('last-modified') - } - -def _disconnect_safely(client): - """Safely closes and disconnects the MPD client.""" - if not client: - return - try: - client.close() - client.disconnect() - except (mpd.ConnectionError, IOError, BrokenPipeError): - pass # Ignore errors on disconnect, as the connection might already be lost. - -def get_recent_albums(limit): - """ - [EFFICIENT VERSION] Fetches a random selection of albums from MPD. - - NOTE: Finding the chronologically "most recent" albums requires a full - scan of every song in the library, which is extremely slow on large collections. - This function provides a fast and practical alternative by returning a random - sample, which is much better for discovering content to analyze. - """ - client = _connect() - if not client: - return [] - - albums = [] - try: - logger.info("Fetching a random selection of albums for analysis...") - - # This is a very fast command that gets all unique album names. - album_names = client.list('album') - - # If the user wants all albums, don't shuffle, just format. - fetch_all = (limit == 0) - if fetch_all: - logger.info(f"Formatting all {len(album_names)} albums.") - albums_to_process = album_names - else: - logger.info(f"Found {len(album_names)} total albums. Shuffling to select {limit} random ones.") - random.shuffle(album_names) - albums_to_process = album_names[:limit] - - # Format the selected albums into the expected dictionary structure. - # We use a placeholder date as the true 'last_modified' is unknown without a full scan. - now = datetime.now() - albums = [{'Id': name, 'Name': name, 'last_modified': now} for name in albums_to_process] - - logger.info(f"Selected {len(albums)} albums to process.") - - except Exception as e: - logger.error(f"MPD get_recent_albums failed: {e}", exc_info=True) - finally: - _disconnect_safely(client) - - return albums - -def get_tracks_from_album(album_id): - """Fetches all audio tracks for a given album name from MPD. album_id is the album name.""" - client = _connect() - if not client: - return [] - - tracks = [] - try: - # Use client.find("album", album_id) to search by metadata tag. - songs = client.find("album", album_id) - tracks = [_format_song(s) for s in songs if 'file' in s] - logger.info(f"Found {len(tracks)} tracks for album '{album_id}'.") - except Exception as e: - logger.error(f"MPD get_tracks_from_album failed for album '{album_id}': {e}", exc_info=True) - finally: - _disconnect_safely(client) - return tracks - -def download_track(temp_dir, item): - """ - Downloads a track from a remote MPD server using its built-in HTTP streamer. - This function assumes MPD's HTTP stream is available on port 8000. - """ - try: - track_path = item.get('Path') - if not track_path: - logger.error("MPD item has no 'Path' attribute to download.") - return None - - # Construct the HTTP stream URL for the track. - # This uses the same MPD_HOST but assumes port 8000 for the HTTP stream. - # The path needs to be URL-encoded to handle special characters. - from urllib.parse import quote - encoded_path = quote(track_path) - download_url = f"http://{config.MPD_HOST}:8000/{encoded_path}" - - logger.info(f"Downloading track from URL: {download_url}") - - # Use requests library to download the file in a streaming fashion - with requests.get(download_url, stream=True) as r: - r.raise_for_status() # This will raise an exception for bad status codes (4xx or 5xx) - - # Create a safe local filename - file_extension = os.path.splitext(track_path)[1] - track_id = os.path.basename(track_path).replace(file_extension, '') - local_filename = os.path.join(temp_dir, f"{track_id}{file_extension}") - - # Write the content to the local file in chunks - with open(local_filename, 'wb') as f: - for chunk in r.iter_content(chunk_size=8192): - f.write(chunk) - - logger.info(f"Successfully downloaded '{item.get('Name', 'Unknown')}' to '{local_filename}'") - return local_filename - - except requests.exceptions.RequestException as e: - logger.error(f"HTTP download failed for track {item.get('Name', 'Unknown')}: {e}", exc_info=True) - return None - except Exception as e: - logger.error(f"An unexpected error occurred during download of {item.get('Name', 'Unknown')}: {e}", exc_info=True) - return None - -def get_all_songs(): - """Fetches all songs from MPD using a robust, song-by-song method.""" - client = _connect() - if not client: - return [] - - all_formatted_songs = [] - try: - logger.info("Fetching all songs from MPD database (robust method)...") - - all_files = client.list('file') - logger.info(f"Found {len(all_files)} files to process.") - - for i, file_path_dict in enumerate(all_files): - try: - # FIX: The list command returns a list of dicts. We need the value from the 'file' key. - file_path_str = file_path_dict.get('file') - if not file_path_str: - continue - - song_info_list = client.listallinfo(file_path_str) - if song_info_list and 'file' in song_info_list[0]: - all_formatted_songs.append(_format_song(song_info_list[0])) - - if (i + 1) % 1000 == 0: - logger.info(f"Formatted {i+1}/{len(all_files)} songs...") - except Exception: - # Ignore errors for individual files (e.g., playlist files, etc.) - pass - - logger.info(f"Successfully formatted {len(all_formatted_songs)} songs from MPD") - - except Exception as e: - logger.error(f"MPD get_all_songs failed: {e}", exc_info=True) - finally: - _disconnect_safely(client) - return all_formatted_songs - -def get_playlist_by_name(playlist_name): - """Finds an MPD playlist by its exact name.""" - client = _connect() - if not client: - return None - - try: - playlists = client.listplaylists() - for p in playlists: - if p.get('playlist') == playlist_name: - return {'Id': playlist_name, 'Name': playlist_name} - except Exception as e: - logger.error(f"MPD get_playlist_by_name failed for '{playlist_name}': {e}", exc_info=True) - finally: - _disconnect_safely(client) - return None - -def create_playlist(base_name, item_ids): - """Creates a new playlist on MPD. item_ids are file paths.""" - client = _connect() - if not client: - return - - try: - # Check if playlist exists and clear it, otherwise MPD appends. - if any(p.get('playlist') == base_name for p in client.listplaylists()): - client.playlistclear(base_name) - logger.info(f"Cleared existing MPD playlist '{base_name}'.") - - for item_path in item_ids: - client.playlistadd(base_name, item_path) - logger.info(f"✅ Created/updated MPD playlist '{base_name}' with {len(item_ids)} songs.") - except Exception as e: - logger.error(f"Exception creating MPD playlist '{base_name}': {e}", exc_info=True) - finally: - _disconnect_safely(client) - -def get_all_playlists(): - """Fetches all playlists from MPD.""" - client = _connect() - if not client: - return [] - - playlists = [] - try: - mpd_playlists = client.listplaylists() - playlists = [{'Id': p.get('playlist'), 'Name': p.get('playlist')} for p in mpd_playlists] - except Exception as e: - logger.error(f"MPD get_all_playlists failed: {e}", exc_info=True) - finally: - _disconnect_safely(client) - return playlists - -def delete_playlist(playlist_id): - """Deletes a playlist on MPD. playlist_id is the playlist name.""" - client = _connect() - if not client: - return False - - success = False - try: - client.rm(playlist_id) - logger.info(f"🗑️ Deleted MPD playlist: {playlist_id}") - success = True - except Exception as e: - logger.error(f"Exception deleting MPD playlist '{playlist_id}': {e}", exc_info=True) - finally: - _disconnect_safely(client) - return success - -# --- User-specific MPD functions (STUBS) --- -# MPD is a single-user daemon and does not track play counts or last played times by default. -def get_top_played_songs(limit, user_creds=None): - """Not supported by MPD. Returns an empty list.""" - logger.warning("get_top_played_songs is not supported by the MPD backend.") - return [] - -def get_last_played_time(item_id, user_creds=None): - """Not supported by MPD. Returns None.""" - logger.warning("get_last_played_time is not supported by the MPD backend.") - return None - -def create_instant_playlist(playlist_name, item_ids, user_creds=None): - """Creates a new instant playlist on MPD.""" - final_playlist_name = f"{playlist_name.strip()}_instant" - # For MPD, this is the same as a regular playlist. user_creds are ignored. - create_playlist(final_playlist_name, item_ids) - # The return value for this function in other implementations is a dict. - return {'Id': final_playlist_name, 'Name': final_playlist_name} - diff --git a/tasks/path_manager.py b/tasks/path_manager.py index 3bb1fca2..5920dec6 100644 --- a/tasks/path_manager.py +++ b/tasks/path_manager.py @@ -290,7 +290,7 @@ def find_path_between_songs(start_item_id, end_item_id, Lreq=PATH_DEFAULT_LENGTH final merge fails catastrophically. """ # Local import to prevent circular dependency - from app_helper import get_score_data_by_ids, get_tracks_by_ids + from app_helper import get_score_data_by_ids logger.info(f"Starting centroid path generation (with merge logic) from {start_item_id} to {end_item_id} with requested length {Lreq}.") if metric is None: diff --git a/tasks/provider_migration_matcher.py b/tasks/provider_migration_matcher.py index 05e04dae..3c98dcb9 100644 --- a/tasks/provider_migration_matcher.py +++ b/tasks/provider_migration_matcher.py @@ -179,7 +179,7 @@ def _best_artist_old(row): """Track-level artist for a source (``score``) row. Precedence: ``author`` → ``artist`` → ``album_artist``. - ``score.author`` holds the track performer that mediaserver_*.py picked via + ``score.author`` holds the track performer that the tasks/mediaserver backends picked via ``_select_best_artist``, while ``score.album_artist`` preserves the album-level artist (often "Various Artists" on compilations). Preferring ``author`` keeps compilation tracks matchable to their real performer on diff --git a/tasks/provider_migration_tasks.py b/tasks/provider_migration_tasks.py index 2e7efa66..6f51c37c 100644 --- a/tasks/provider_migration_tasks.py +++ b/tasks/provider_migration_tasks.py @@ -580,8 +580,6 @@ def _run_migration_transaction(cur, mapping, new_meta, 'emby': {'url': 'EMBY_URL', 'user_id': 'EMBY_USER_ID', 'token': 'EMBY_TOKEN'}, 'navidrome': {'url': 'NAVIDROME_URL', 'user': 'NAVIDROME_USER', 'password': 'NAVIDROME_PASSWORD'}, 'lyrion': {'url': 'LYRION_URL'}, - 'mpd': {'host': 'MPD_HOST', 'port': 'MPD_PORT', 'password': 'MPD_PASSWORD', - 'music_directory': 'MPD_MUSIC_DIRECTORY'}, } diff --git a/tasks/radio_manager.py b/tasks/radio_manager.py index 30a75e59..170a7b11 100644 --- a/tasks/radio_manager.py +++ b/tasks/radio_manager.py @@ -14,7 +14,7 @@ def run_radio_playlists(): playlists on online-first sync clients (e.g. Symfonium on Navidrome) that track playlists by ID. - Falls back to create_playlist for MPD and other unsupported backends. + Falls back to create_playlist for unsupported backends. """ from app_helper import get_alchemy_radios @@ -47,7 +47,7 @@ def run_radio_playlists(): try: create_or_replace_playlist(playlist_name, item_ids) except NotImplementedError: - # MPD or unsupported backend: fall back to plain create. + # Unsupported backend: fall back to plain create. create_playlist(playlist_name, item_ids) created += 1 logger.info(f"Radio playlist '{playlist_name}' upserted with {len(item_ids)} tracks.") diff --git a/tasks/sem_grove_manager.py b/tasks/sem_grove_manager.py index da46d397..a0b007bf 100644 --- a/tasks/sem_grove_manager.py +++ b/tasks/sem_grove_manager.py @@ -31,9 +31,7 @@ import json import logging import math -import re import sys -import tempfile from typing import Dict, List, Optional import numpy as np @@ -122,22 +120,8 @@ def _make_merged_vector( def _fetch_metadata(item_ids: List[str]) -> Dict[str, Dict]: - if not item_ids: - return {} - from app_helper import get_score_data_by_ids - try: - rows = get_score_data_by_ids(item_ids) - return { - r["item_id"]: { - "title": r.get("title", "") or "", - "author": r.get("author", "") or "", - "album": r.get("album", "") or "", - } - for r in rows - } - except Exception as exc: - logger.warning("SemGrove metadata fetch failed: %s", exc) - return {} + from .commons import fetch_track_metadata_map + return fetch_track_metadata_map(item_ids) # --------------------------------------------------------------------------- @@ -386,109 +370,33 @@ def _load_sem_grove_index_from_db() -> bool: audio_dim = int(whitening["audio_dim"]) merged_dim = lyrics_dim + audio_dim - # ---- Index binary ---- - cur.execute( - "SELECT index_data, id_map_json, embedding_dimension " - "FROM lyrics_index_data WHERE index_name = %s", - (SEM_GROVE_INDEX_NAME,), - ) - row = cur.fetchone() - - index_stream = None - id_map_json = None - db_dim = None - try: - if row: - binary, id_map_json, db_dim = row - index_stream = tempfile.TemporaryFile() - index_stream.write(binary) - index_stream.seek(0) - else: - seg_pattern = re.compile(r"^sem_grove_index_(\d+)_(\d+)$") - parts = [] - total_expected = None - - with conn.cursor(name="sem_grove_index_segments") as seg_cur: - seg_cur.itersize = 50 - seg_cur.execute( - "SELECT index_name, index_data, id_map_json, embedding_dimension " - "FROM lyrics_index_data WHERE index_name LIKE %s ESCAPE '\\'", - (r"sem_grove_index\_%\_%",), - ) - for name, part_data, part_id_map, part_dim in seg_cur: - m = seg_pattern.match(name) - if not m: - continue - part_no = int(m.group(1)) - total = int(m.group(2)) - if total_expected is None: - total_expected = total - elif total_expected != total: - logger.error("SemGrove: segment total mismatch.") - return False - parts.append((part_no, part_data, part_id_map, part_dim)) - - if total_expected is None or len(parts) != total_expected: - logger.info( - "SemGrove: no complete index found in DB " - "(expected=%s, got=%d).", total_expected, len(parts) - ) - return False - - parts.sort(key=lambda p: p[0]) - from .index_build_helpers import reassemble_segmented_id_map - db_dim = parts[0][3] - id_map_json = reassemble_segmented_id_map((p[0], p[2]) for p in parts) - index_stream = tempfile.TemporaryFile() - for _, part_data, _, _ in parts: - index_stream.write(part_data) - index_stream.seek(0) - - if index_stream is None or not id_map_json: - return False - - if db_dim != merged_dim: - logger.error( - "SemGrove: dimension mismatch (db=%d, expected=%d).", - db_dim, merged_dim, - ) - return False - - loaded_index = voyager.Index.load(index_stream) - loaded_index.ef = VOYAGER_QUERY_EF - - finally: - if index_stream is not None: - try: - index_stream.close() - except Exception: - pass - - id_map = {int(k): v for k, v in json.loads(id_map_json).items()} - reverse_id_map = {v: k for k, v in id_map.items()} + from .index_build_helpers import load_voyager_index_from_db + loaded = load_voyager_index_from_db( + conn, 'lyrics_index_data', SEM_GROVE_INDEX_NAME, + merged_dim, VOYAGER_QUERY_EF, label='SemGrove', + ) + if loaded is None: + return False + loaded_index, id_map, reverse_id_map = loaded - if not id_map: - logger.warning("SemGrove: id_map is empty after load.") - return False + _SEM_GROVE_CACHE.update({ + "index": loaded_index, + "id_map": id_map, + "reverse_id_map": reverse_id_map, + "std_lyrics": std_lyrics, + "std_audio": std_audio, + "lyrics_dim": lyrics_dim, + "audio_dim": audio_dim, + "w_lyrics": w_lyrics, + "w_audio": w_audio, + "loaded": True, + "song_count": len(id_map), + }) - _SEM_GROVE_CACHE.update({ - "index": loaded_index, - "id_map": id_map, - "reverse_id_map": reverse_id_map, - "std_lyrics": std_lyrics, - "std_audio": std_audio, - "lyrics_dim": lyrics_dim, - "audio_dim": audio_dim, - "w_lyrics": w_lyrics, - "w_audio": w_audio, - "loaded": True, - "song_count": len(id_map), - }) - - logger.info( - "SemGrove index loaded: %d items, dim=%d.", len(id_map), merged_dim - ) - return True + logger.info( + "SemGrove index loaded: %d items, dim=%d.", len(id_map), merged_dim + ) + return True except Exception as exc: logger.error("SemGrove index load failed: %s", exc, exc_info=True) diff --git a/tasks/setup_manager.py b/tasks/setup_manager.py index f14df93a..553c252e 100644 --- a/tasks/setup_manager.py +++ b/tasks/setup_manager.py @@ -301,3 +301,6 @@ def get_all_fields(self, config_module): "overridden": overridden, }) return fields + + +setup_manager = SetupManager() diff --git a/tasks/song_alchemy.py b/tasks/song_alchemy.py index 4810fcb3..a8b985ce 100644 --- a/tasks/song_alchemy.py +++ b/tasks/song_alchemy.py @@ -3,17 +3,13 @@ import numpy as np from .voyager_manager import find_nearest_neighbors_by_vector, find_nearest_neighbors_by_id, get_vector_by_id +from .alchemy_projections import ( + _project_to_2d, + _project_with_discriminant, +) from app_helper import get_score_data_by_ids, load_map_projection import config -try: - # sklearn is already a dependency; import lazily for environments where it's present - from sklearn.decomposition import PCA - from sklearn.linear_model import LogisticRegression -except Exception: - PCA = None - LogisticRegression = None - logger = logging.getLogger(__name__) @@ -152,208 +148,6 @@ def _compute_centroid_from_items(items: List[dict]) -> np.ndarray: return weighted_centroid -def _project_to_2d(vectors: List[np.ndarray]) -> List[Tuple[float, float]]: - """Simple PCA via SVD to project a list of vectors to 2D. - Returns a list of (x, y) tuples in the same order as input vectors. - If there are fewer than 2 vectors, returns zeros for all. - """ - if not vectors: - return [] - mat = np.vstack(vectors) - # Center - mean = np.mean(mat, axis=0) - mat_c = mat - mean - # SVD - try: - u, s, vh = np.linalg.svd(mat_c, full_matrices=False) - except Exception: - # Fallback: return zeros - return [(0.0, 0.0) for _ in vectors] - # Take first two principal components - pcs = vh[:2] - proj = mat_c.dot(pcs.T) - # Normalize projection for nicer plotting - if proj.size == 0: - return [(0.0, 0.0) for _ in vectors] - # Normalize preserving aspect ratio: use a single global scale so x/y units are comparable - # center at zero - proj_centered = proj - proj.mean(axis=0) - max_abs = np.max(np.abs(proj_centered)) - if max_abs == 0: - return [(0.0, 0.0) for _ in vectors] - scaled = proj_centered / max_abs - # clamp to [-1,1] for safety - scaled = np.clip(scaled, -1.0, 1.0) - return [(float(x), float(y)) for x, y in scaled] - - -def _project_aligned_add_sub(vectors: List[np.ndarray], add_centroid: np.ndarray, subtract_centroid: np.ndarray) -> List[Tuple[float, float]]: - """Project vectors to 2D where the x-axis is aligned with the vector - from add_centroid -> subtract_centroid. The y-axis is the leading - orthogonal component (first PC of residuals). - This emphasizes separation along the add-vs-subtract direction. - """ - if not vectors: - return [] - # Convert list to matrix and center relative to add_centroid - mat = np.vstack(vectors) - rel = mat - add_centroid - axis = subtract_centroid - add_centroid - axis_norm = np.linalg.norm(axis) - if axis_norm == 0: - # Fallback to PCA if centroids coincide - return _project_to_2d(vectors) - axis_u = axis / axis_norm - - # Compute x coordinates as projection on axis - x_coords = rel.dot(axis_u) - - # Remove axis component to get residuals for y-axis computation - proj_on_axis = np.outer(x_coords, axis_u) - residuals = rel - proj_on_axis - - # Find leading direction in residuals via SVD - try: - # If residuals are all near-zero, SVD will still succeed but produce small values - u, s, vh = np.linalg.svd(residuals, full_matrices=False) - if vh.shape[0] >= 1: - y_u = vh[0] - else: - y_u = None - except Exception: - y_u = None - - if y_u is None or np.linalg.norm(y_u) == 0: - # Create an arbitrary orthogonal vector to axis_u - # pick an index where axis_u has smallest absolute value - idx = int(np.argmin(np.abs(axis_u))) - e = np.zeros_like(axis_u) - e[idx] = 1.0 - y_u = e - np.dot(e, axis_u) * axis_u - norm_y = np.linalg.norm(y_u) - if norm_y == 0: - # fallback - return _project_to_2d(vectors) - y_u = y_u / norm_y - else: - # ensure orthogonal to axis_u (numerical stability) - y_u = y_u - np.dot(y_u, axis_u) * axis_u - y_u_norm = np.linalg.norm(y_u) - if y_u_norm == 0: - return _project_to_2d(vectors) - y_u = y_u / y_u_norm - - y_coords = residuals.dot(y_u) - - coords = np.vstack([x_coords, y_coords]).T - # Center and scale uniformly so x and y share same units - coords_centered = coords - coords.mean(axis=0) - max_abs = np.max(np.abs(coords_centered)) - if max_abs == 0: - return [(0.0, 0.0) for _ in vectors] - scaled = coords_centered / max_abs - scaled = np.clip(scaled, -1.0, 1.0) - return [(float(x), float(y)) for x, y in scaled] - - -def _project_with_umap(vectors: List[np.ndarray], n_components: int = 2) -> List[Tuple[float, float]]: - """Try to project using UMAP if available. Raises ImportError if umap is not installed.""" - import umap - if not vectors: - return [] - mat = np.vstack(vectors) - reducer = umap.UMAP(n_components=n_components, random_state=None, n_jobs=-1) - embedding = reducer.fit_transform(mat) - # Center and scale uniformly so x and y share same units - emb_centered = embedding - embedding.mean(axis=0) - max_abs = np.max(np.abs(emb_centered)) - if max_abs == 0: - return [(0.0, 0.0) for _ in vectors] - scaled = emb_centered / max_abs - scaled = np.clip(scaled, -1.0, 1.0) - return [(float(x), float(y)) for x, y in scaled] - - -def _project_with_discriminant(add_vectors: List[np.ndarray], sub_vectors: List[np.ndarray], all_vectors: List[np.ndarray]) -> List[Tuple[float, float]]: - """Compute a discriminant direction separating add and sub using PCA+LogisticRegression. - Returns 2D coords for all_vectors projected onto (discriminant axis, residual axis). - Falls back (raises) if sklearn not available or insufficient samples. - """ - if LogisticRegression is None or PCA is None: - raise RuntimeError('sklearn not available') - # Need at least one sample in each class - if not add_vectors or not sub_vectors: - raise RuntimeError('Insufficient classes for discriminant') - - X_train = np.vstack([np.vstack(add_vectors), np.vstack(sub_vectors)]) - y_train = np.array([1] * len(add_vectors) + [0] * len(sub_vectors)) - - n_samples, n_features = X_train.shape - # Reduce dimensionality so training is stable (components <= n_samples-1) - max_components = min(32, n_samples - 1, n_features) - if max_components < 1: - raise RuntimeError('Not enough samples for discriminant PCA') - - pca = PCA(n_components=max_components, random_state=42) - Xp = pca.fit_transform(X_train) - - # Fit logistic regression with regularization for robustness - try: - clf = LogisticRegression(l1_ratio=0, C=1.0, solver='saga', max_iter=1000) - clf.fit(Xp, y_train) - except Exception: - # Fallback with less regularization if solver fails - clf = LogisticRegression(l1_ratio=0, C=0.1, solver='saga', max_iter=1000) - clf.fit(Xp, y_train) - - # direction in PCA space - coef = clf.coef_.ravel() - norm = np.linalg.norm(coef) - if norm == 0: - raise RuntimeError('Discriminant produced zero vector') - dir_pca = coef / norm - - # Project all vectors into PCA space then onto discriminant for x coords - all_mat = np.vstack(all_vectors) - all_pca = pca.transform(all_mat) - x_coords = all_pca.dot(dir_pca) - - # Residuals in PCA space - proj_on_dir = np.outer(x_coords, dir_pca) - residuals = all_pca - proj_on_dir - # y direction: leading PC of residuals - try: - u, s, vh = np.linalg.svd(residuals, full_matrices=False) - if vh.shape[0] >= 1: - y_u = vh[0] - else: - y_u = None - except Exception: - y_u = None - - if y_u is None or np.linalg.norm(y_u) == 0: - # fallback: arbitrary orthogonal - idx = int(np.argmin(np.abs(dir_pca))) - e = np.zeros_like(dir_pca) - e[idx] = 1.0 - y_u = e - np.dot(e, dir_pca) * dir_pca - y_u = y_u / (np.linalg.norm(y_u) or 1.0) - else: - y_u = y_u - np.dot(y_u, dir_pca) * dir_pca - y_u = y_u / (np.linalg.norm(y_u) or 1.0) - - y_coords = residuals.dot(y_u) - - coords = np.vstack([x_coords, y_coords]).T - coords_centered = coords - coords.mean(axis=0) - max_abs = np.max(np.abs(coords_centered)) - if max_abs == 0: - return [(0.0, 0.0) for _ in all_vectors] - scaled = coords_centered / max_abs - scaled = np.clip(scaled, -1.0, 1.0) - return [(float(x), float(y)) for x, y in scaled] - - def song_alchemy(add_items=None, subtract_items=None, add_ids=None, subtract_ids=None, n_results: int = None, subtract_distance: float = None, temperature: float = None) -> dict: """Perform Song Alchemy: - add_items: list of dicts with 'type' ('song'/'artist') and 'id' diff --git a/tasks/voyager_manager.py b/tasks/voyager_manager.py index 534334b1..17420d1c 100644 --- a/tasks/voyager_manager.py +++ b/tasks/voyager_manager.py @@ -21,11 +21,8 @@ from concurrent.futures import ThreadPoolExecutor, as_completed from functools import lru_cache import threading -import math # Import math for ceiling function from config import EMBEDDING_DIMENSION, INDEX_NAME, VOYAGER_METRIC, VOYAGER_QUERY_EF, VOYAGER_MAX_PART_SIZE_MB, MAX_SONGS_PER_ARTIST, DUPLICATE_DISTANCE_THRESHOLD_COSINE, DUPLICATE_DISTANCE_THRESHOLD_EUCLIDEAN, DUPLICATE_DISTANCE_CHECK_LOOKBACK, MOOD_SIMILARITY_THRESHOLD, SIMILARITY_ELIMINATE_DUPLICATES_DEFAULT, SIMILARITY_RADIUS_DEFAULT, MOOD_SIMILARITY_ENABLE -# Import from other project modules -from .mediaserver import create_instant_playlist logger = logging.getLogger(__name__) @@ -1322,8 +1319,7 @@ def create_playlist_from_ids(playlist_name: str, track_ids: list, user_creds: di Creates a new playlist on the configured media server with the provided name and track IDs. """ try: - # Use the mediaserver dispatcher (imported at module top) to create the playlist. - # This avoids importing app_external which may not export the helper. + from .mediaserver import create_instant_playlist created_playlist = create_instant_playlist(playlist_name, track_ids, user_creds=user_creds) if not created_playlist: diff --git a/templates/provider_migration.html b/templates/provider_migration.html index 422b4f37..316e9f96 100644 --- a/templates/provider_migration.html +++ b/templates/provider_migration.html @@ -930,7 +930,7 @@

6 Execution

sessionId = s.session_id; } // Populate the library checkbox list once we have a session. Hidden - // for providers without a listable folder API (e.g. MPD). + // for providers without a listable folder API. fetchMigLibraries(); unlock(3); setActive(3); diff --git a/test/README.md b/test/README.md index bf91f4c3..e44c60c1 100644 --- a/test/README.md +++ b/test/README.md @@ -1,6 +1,12 @@ # AudioMuse-AI Developer Tests -These integration tests are for developer purposes to verify the functionality of the AudioMuse-AI API endpoints. They are not included in the production Docker container and should be run from a local development machine. +This folder contains the whole test suite: + +- `unit/` — fast, mock-based unit tests (run in CI by `.github/workflows/tests.yml` with `pytest test/unit/`) +- `integration/` — integration tests and developer scripts that exercise the real models, database or a running AudioMuse-AI instance (run in CI by `.github/workflows/test.yml`) +- support assets shared by the integration tests: `songs/`, `models/` (downloaded by CI), `lyrics_expected*.json`, `requirements.txt`, `docker-compose.yaml`, `nginx-confd/`, `provider_testing_stack/` + +The API endpoint tests below are for developer purposes to verify the functionality of the AudioMuse-AI API endpoints. They are not included in the production Docker container and should be run from a local development machine. ## Prerequisites @@ -50,9 +56,9 @@ Once you are inside the `test` directory, follow these steps: ``` 3. **Configure the API endpoint:** - Open the `test.py` file and update the `BASE_URL` to point to your running AudioMuse-AI instance. + Open the `integration/test.py` file and update the `BASE_URL` to point to your running AudioMuse-AI instance. ```python - # test/test.py + # test/integration/test.py BASE_URL = 'http://YOUR_AUDIOMUSE_IP:8000' ``` @@ -61,7 +67,7 @@ Once you are inside the `test` directory, follow these steps: To run all tests, execute the following command from your terminal (while in the `test` directory with the virtual environment activated): ```bash -python test.py +python integration/test.py ``` ## Succesfull result example @@ -111,5 +117,5 @@ See `./nginx-confd/reverse-proxy.conf` and the line with `proxy_pass http://audi Do run the tests against it you have to start execute: `docker compose up`. Set the base url to `BASE_URL = 'http://localhost:7777/am'` Your audiomuse instance should have set the following configuration to true `ENABLE_PROXY_FIX`. -And run the test by executing: `python test.py`. +And run the test by executing: `python integration/test.py`. diff --git a/tests/__init__.py b/test/__init__.py similarity index 100% rename from tests/__init__.py rename to test/__init__.py diff --git a/test/integration/__init__.py b/test/integration/__init__.py new file mode 100644 index 00000000..a2650482 --- /dev/null +++ b/test/integration/__init__.py @@ -0,0 +1 @@ +# Integration tests package diff --git a/test/test.py b/test/integration/test.py similarity index 100% rename from test/test.py rename to test/integration/test.py diff --git a/test/test_analysis_integration.py b/test/integration/test_analysis_integration.py similarity index 99% rename from test/test_analysis_integration.py rename to test/integration/test_analysis_integration.py index faeb91dc..2ea829b8 100644 --- a/test/test_analysis_integration.py +++ b/test/integration/test_analysis_integration.py @@ -8,7 +8,7 @@ # 3. Install requirements: # pip install -r test/requirements.txt # 4. Run this script: -# pytest test/test_analysis_integration.py -s -q +# pytest test/integration/test_analysis_integration.py -s -q # # Note: Test audio files should be in test/songs/ # ONNX models should be in test/models/ @@ -128,7 +128,7 @@ def test_real_analysis_runs_and_returns_expected_shape(): importable in the environment. It injects lightweight stubs for optional AI/voyager libraries so module import succeeds. """ - project_root = Path(__file__).resolve().parents[1] + project_root = Path(__file__).resolve().parents[2] models_dir = project_root / 'test' / 'models' required = [ 'musicnn_embedding.onnx', 'musicnn_prediction.onnx', diff --git a/test/test_app_endpoints_integration.py b/test/integration/test_app_endpoints_integration.py similarity index 98% rename from test/test_app_endpoints_integration.py rename to test/integration/test_app_endpoints_integration.py index 8150e0a9..e2265b69 100644 --- a/test/test_app_endpoints_integration.py +++ b/test/integration/test_app_endpoints_integration.py @@ -13,7 +13,7 @@ Run locally: pip install pgserver - pytest test/test_app_endpoints_integration.py -m integration -s -v --tb=short + pytest test/integration/test_app_endpoints_integration.py -m integration -s -v --tb=short """ import os import sys @@ -25,7 +25,7 @@ import pytest from flask import Flask -_REPO_ROOT = os.path.normpath(os.path.join(os.path.dirname(os.path.abspath(__file__)), '..')) +_REPO_ROOT = os.path.normpath(os.path.join(os.path.dirname(os.path.abspath(__file__)), '..', '..')) if _REPO_ROOT not in sys.path: sys.path.insert(0, _REPO_ROOT) diff --git a/test/test_auth_users_integration.py b/test/integration/test_auth_users_integration.py similarity index 97% rename from test/test_auth_users_integration.py rename to test/integration/test_auth_users_integration.py index 9822e637..d52594db 100644 --- a/test/test_auth_users_integration.py +++ b/test/integration/test_auth_users_integration.py @@ -13,7 +13,7 @@ Run locally: pip install pgserver - pytest test/test_auth_users_integration.py -m integration -s -v --tb=short + pytest test/integration/test_auth_users_integration.py -m integration -s -v --tb=short """ import os import sys @@ -21,7 +21,7 @@ import pytest -_REPO_ROOT = os.path.normpath(os.path.join(os.path.dirname(os.path.abspath(__file__)), '..')) +_REPO_ROOT = os.path.normpath(os.path.join(os.path.dirname(os.path.abspath(__file__)), '..', '..')) if _REPO_ROOT not in sys.path: sys.path.insert(0, _REPO_ROOT) diff --git a/test/test_clap_analysis_integration.py b/test/integration/test_clap_analysis_integration.py similarity index 97% rename from test/test_clap_analysis_integration.py rename to test/integration/test_clap_analysis_integration.py index 4f0b7804..0bbbcd9a 100644 --- a/test/test_clap_analysis_integration.py +++ b/test/integration/test_clap_analysis_integration.py @@ -8,7 +8,7 @@ # 3. Install requirements: # pip install -r test/requirements.txt # 4. Run this script: -# pytest test/test_clap_analysis_integration.py -s -q +# pytest test/integration/test_clap_analysis_integration.py -s -q # # Note: Test audio files should be in test/songs/ # CLAP ONNX models: @@ -62,7 +62,7 @@ def test_clap_analysis_runs_and_shows_output(): 'acoustic': 0.116278, }, } - project_root = Path(__file__).resolve().parents[1] + project_root = Path(__file__).resolve().parents[2] models_dir = project_root / 'test' / 'models' clap_audio_model = models_dir / 'model_epoch_36.onnx' clap_text_model = models_dir / 'clap_text_model.onnx' @@ -212,5 +212,5 @@ def test_clap_analysis_runs_and_shows_output(): if __name__ == '__main__': - # Allow running directly with: python test/test_clap_analysis_integration.py + # Allow running directly with: python test/integration/test_clap_analysis_integration.py pytest.main([__file__, '-s', '-v']) diff --git a/test/test_gpu_status.py b/test/integration/test_gpu_status.py similarity index 99% rename from test/test_gpu_status.py rename to test/integration/test_gpu_status.py index 584d5650..f32ecd3d 100644 --- a/test/test_gpu_status.py +++ b/test/integration/test_gpu_status.py @@ -8,7 +8,7 @@ 3. cuML/cupy for clustering Usage: - docker exec audiomuse-ai-worker-instance-dev python3 /app/test/test_gpu_status.py + docker exec audiomuse-ai-worker-instance-dev python3 /app/test/integration/test_gpu_status.py """ import sys diff --git a/test/test_lyrics_analysis_integration.py b/test/integration/test_lyrics_analysis_integration.py similarity index 99% rename from test/test_lyrics_analysis_integration.py rename to test/integration/test_lyrics_analysis_integration.py index f352ff17..50a1b23f 100644 --- a/test/test_lyrics_analysis_integration.py +++ b/test/integration/test_lyrics_analysis_integration.py @@ -134,7 +134,7 @@ def test_real_lyrics_analysis_runs_and_matches_expected_vectors(monkeypatch): and checks the gte-multilingual-base embedding + axis vector against pre-recorded values via cosine similarity (threshold = 0.98). """ - project_root = Path(__file__).resolve().parents[1] + project_root = Path(__file__).resolve().parents[2] models_dir = project_root / 'test' / 'models' expected_path = project_root / 'test' / 'lyrics_expected_gte_512.json' diff --git a/test/test_provider_migration_integration.py b/test/integration/test_provider_migration_integration.py similarity index 99% rename from test/test_provider_migration_integration.py rename to test/integration/test_provider_migration_integration.py index c4da0abb..a3319605 100644 --- a/test/test_provider_migration_integration.py +++ b/test/integration/test_provider_migration_integration.py @@ -43,7 +43,7 @@ Run locally: pip install pgserver - pytest test/test_provider_migration_integration.py -s -v --tb=short + pytest test/integration/test_provider_migration_integration.py -s -v --tb=short """ import importlib.util import json @@ -68,7 +68,7 @@ def _load_module(mod_name, *rel_parts): if mod_name in sys.modules: return sys.modules[mod_name] repo_root = os.path.normpath( - os.path.join(os.path.dirname(os.path.abspath(__file__)), '..') + os.path.join(os.path.dirname(os.path.abspath(__file__)), '..', '..') ) if repo_root not in sys.path: sys.path.insert(0, repo_root) diff --git a/test/verify_onnx_embeddings.py b/test/integration/verify_onnx_embeddings.py similarity index 99% rename from test/verify_onnx_embeddings.py rename to test/integration/verify_onnx_embeddings.py index 93099a70..d4667a9a 100644 --- a/test/verify_onnx_embeddings.py +++ b/test/integration/verify_onnx_embeddings.py @@ -11,7 +11,7 @@ import librosa.feature # Add parent directory to path -sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))) def compare_pytorch_vs_onnx(): """Compare embeddings from PyTorch .pt model vs ONNX model""" @@ -21,7 +21,7 @@ def compare_pytorch_vs_onnx(): print("=" * 80) # Find all test audio files - test_songs_dir = os.path.join(os.path.dirname(__file__), "songs") + test_songs_dir = os.path.join(os.path.dirname(__file__), "..", "songs") test_audio_files = [] if os.path.exists(test_songs_dir): diff --git a/test/requirements.txt b/test/requirements.txt index 67657f7c..d6ba75ca 100644 --- a/test/requirements.txt +++ b/test/requirements.txt @@ -41,7 +41,6 @@ sqlglot six flasgger umap-learn -python-mpd2 google-genai==1.57.0 argon2-cffi==25.1.0 PyJWT==2.12.1 diff --git a/tests/unit/__init__.py b/test/unit/__init__.py similarity index 100% rename from tests/unit/__init__.py rename to test/unit/__init__.py diff --git a/tests/conftest.py b/test/unit/conftest.py similarity index 91% rename from tests/conftest.py rename to test/unit/conftest.py index 7a0dfd0a..08762e32 100644 --- a/tests/conftest.py +++ b/test/unit/conftest.py @@ -2,10 +2,16 @@ Centralises duplicated helpers across test files: - importlib bypass loader (avoids tasks/__init__.py -> pydub -> audioop chain) -- Session-scoped module fixtures for mcp_server, ai_mcp_client, mediaserver_localfiles +- Session-scoped module fixtures for mcp_server, ai_mcp_client - FakeRow / mock-connection helpers - Autouse config restoration fixture """ +import sys as _sys +if _sys.platform == 'win32': + import multiprocessing as _mp + _o = _mp.get_context + _mp.get_context = lambda m=None: _o('spawn') if m == 'fork' else _o(m) + import os import sys import importlib.util @@ -27,7 +33,7 @@ def _import_module(mod_name: str, relative_path: str): (e.g. 'tasks/mcp_helper.py'). """ repo_root = os.path.normpath( - os.path.join(os.path.dirname(os.path.abspath(__file__)), '..') + os.path.join(os.path.dirname(os.path.abspath(__file__)), '..', '..') ) mod_path = os.path.normpath(os.path.join(repo_root, relative_path)) @@ -53,15 +59,6 @@ def mcp_server_mod(): return _import_module('tasks.mcp_helper', 'tasks/mcp_helper.py') -@pytest.fixture(scope='session') -def localfiles_mod(): - """Load tasks.mediaserver_localfiles directly (session-scoped).""" - return _import_module( - 'tasks.mediaserver_localfiles', - 'tasks/mediaserver_localfiles.py', - ) - - # --------------------------------------------------------------------------- # DB mock helpers # --------------------------------------------------------------------------- diff --git a/tests/unit/test_ai.py b/test/unit/test_ai.py similarity index 100% rename from tests/unit/test_ai.py rename to test/unit/test_ai.py diff --git a/tests/unit/test_analysis.py b/test/unit/test_analysis.py similarity index 99% rename from tests/unit/test_analysis.py rename to test/unit/test_analysis.py index 4dcb462e..11e937d6 100644 --- a/tests/unit/test_analysis.py +++ b/test/unit/test_analysis.py @@ -7,7 +7,7 @@ # 3. Install requirements: # pip install -r test/requirements.txt # 4. Run this script: -# python -m pytest tests/unit/test_analysis.py --tb=short +# python -m pytest test/unit/test_analysis.py --tb=short """Unit tests for tasks/analysis.py""" import numpy as np diff --git a/tests/unit/test_api_sync.py b/test/unit/test_api_sync.py similarity index 97% rename from tests/unit/test_api_sync.py rename to test/unit/test_api_sync.py index 7d8298dc..b84e4a8b 100644 --- a/tests/unit/test_api_sync.py +++ b/test/unit/test_api_sync.py @@ -161,19 +161,6 @@ def _setup_ids(cur, tracks=None): cur._fetchall_queue.append(tracks if tracks is not None else []) -# --------------------------------------------------------------------------- # -# Entry gate -# --------------------------------------------------------------------------- # - -class TestMpdGate: - def test_mpd_returns_501(self, bp_mod, client): - import config - config.MEDIASERVER_TYPE = 'mpd' - resp = client.get('/api/sync?limit=1') - assert resp.status_code == 501 - assert 'mpd' in resp.get_json()['error'].lower() - - # --------------------------------------------------------------------------- # # Envelope (payload) # --------------------------------------------------------------------------- # diff --git a/tests/unit/test_app_alchemy_anchor.py b/test/unit/test_app_alchemy_anchor.py similarity index 100% rename from tests/unit/test_app_alchemy_anchor.py rename to test/unit/test_app_alchemy_anchor.py diff --git a/tests/unit/test_app_alchemy_payload.py b/test/unit/test_app_alchemy_payload.py similarity index 100% rename from tests/unit/test_app_alchemy_payload.py rename to test/unit/test_app_alchemy_payload.py diff --git a/tests/unit/test_app_alchemy_radio.py b/test/unit/test_app_alchemy_radio.py similarity index 99% rename from tests/unit/test_app_alchemy_radio.py rename to test/unit/test_app_alchemy_radio.py index 74003e52..2d478df3 100644 --- a/tests/unit/test_app_alchemy_radio.py +++ b/test/unit/test_app_alchemy_radio.py @@ -242,8 +242,8 @@ def test_radio_with_no_results_creates_no_playlist(self, mock_alchemy, mock_upse class TestDeletePlaylistsBySuffix: @patch('tasks.mediaserver.config') - @patch('tasks.mediaserver.jellyfin_get_all_playlists') - @patch('tasks.mediaserver.jellyfin_delete_playlist') + @patch('tasks.mediaserver.jellyfin.get_all_playlists') + @patch('tasks.mediaserver.jellyfin.delete_playlist') def test_only_deletes_radio_suffix_playlists(self, mock_delete, mock_get, mock_config): from tasks.mediaserver import delete_playlists_by_suffix diff --git a/tests/unit/test_app_analysis.py b/test/unit/test_app_analysis.py similarity index 100% rename from tests/unit/test_app_analysis.py rename to test/unit/test_app_analysis.py diff --git a/tests/unit/test_app_auth.py b/test/unit/test_app_auth.py similarity index 100% rename from tests/unit/test_app_auth.py rename to test/unit/test_app_auth.py diff --git a/tests/unit/test_app_backup_restore.py b/test/unit/test_app_backup_restore.py similarity index 100% rename from tests/unit/test_app_backup_restore.py rename to test/unit/test_app_backup_restore.py diff --git a/tests/unit/test_app_chat.py b/test/unit/test_app_chat.py similarity index 100% rename from tests/unit/test_app_chat.py rename to test/unit/test_app_chat.py diff --git a/tests/unit/test_app_chat_ssrf.py b/test/unit/test_app_chat_ssrf.py similarity index 100% rename from tests/unit/test_app_chat_ssrf.py rename to test/unit/test_app_chat_ssrf.py diff --git a/tests/unit/test_app_clustering.py b/test/unit/test_app_clustering.py similarity index 100% rename from tests/unit/test_app_clustering.py rename to test/unit/test_app_clustering.py diff --git a/tests/unit/test_app_cron.py b/test/unit/test_app_cron.py similarity index 95% rename from tests/unit/test_app_cron.py rename to test/unit/test_app_cron.py index f0d410fa..b29cc9bc 100644 --- a/tests/unit/test_app_cron.py +++ b/test/unit/test_app_cron.py @@ -3,7 +3,7 @@ Verifies: - Empty fingerprint results → previous playlist is preserved (no upsert call) - Non-empty results → create_or_replace_playlist called with the constant name -- Backend that raises NotImplementedError (e.g. MPD) → falls back to legacy +- Backend that raises NotImplementedError → falls back to legacy date-suffixed create_playlist_from_ids path """ from unittest.mock import MagicMock, patch @@ -72,7 +72,7 @@ def test_sonic_fingerprint_branch_calls_upsert_with_constant_name(mock_get_db, _ @patch('app_cron.cron_matches_now', return_value=True) @patch('app_cron.get_db') -def test_sonic_fingerprint_branch_falls_back_for_mpd(mock_get_db, _matches): +def test_sonic_fingerprint_branch_falls_back_for_unsupported_backend(mock_get_db, _matches): """Backend raising NotImplementedError → legacy create_playlist_from_ids called.""" from app_cron import run_due_cron_jobs diff --git a/tests/unit/test_app_cron_parsing.py b/test/unit/test_app_cron_parsing.py similarity index 100% rename from tests/unit/test_app_cron_parsing.py rename to test/unit/test_app_cron_parsing.py diff --git a/tests/unit/test_app_dashboard_parsing.py b/test/unit/test_app_dashboard_parsing.py similarity index 100% rename from tests/unit/test_app_dashboard_parsing.py rename to test/unit/test_app_dashboard_parsing.py diff --git a/tests/unit/test_app_helper_enrichment.py b/test/unit/test_app_helper_enrichment.py similarity index 100% rename from tests/unit/test_app_helper_enrichment.py rename to test/unit/test_app_helper_enrichment.py diff --git a/tests/unit/test_app_helper_task_note.py b/test/unit/test_app_helper_task_note.py similarity index 100% rename from tests/unit/test_app_helper_task_note.py rename to test/unit/test_app_helper_task_note.py diff --git a/tests/unit/test_app_map_helpers.py b/test/unit/test_app_map_helpers.py similarity index 100% rename from tests/unit/test_app_map_helpers.py rename to test/unit/test_app_map_helpers.py diff --git a/tests/unit/test_artist_gmm_manager.py b/test/unit/test_artist_gmm_manager.py similarity index 100% rename from tests/unit/test_artist_gmm_manager.py rename to test/unit/test_artist_gmm_manager.py diff --git a/tests/unit/test_artist_metadata_codec.py b/test/unit/test_artist_metadata_codec.py similarity index 98% rename from tests/unit/test_artist_metadata_codec.py rename to test/unit/test_artist_metadata_codec.py index 08a59a91..df7e0c91 100644 --- a/tests/unit/test_artist_metadata_codec.py +++ b/test/unit/test_artist_metadata_codec.py @@ -1,4 +1,4 @@ -# tests/unit/test_artist_metadata_codec.py +# test/unit/test_artist_metadata_codec.py """ Unit tests for the artist metadata storage helpers in ``tasks/index_build_helpers.py``: @@ -267,7 +267,7 @@ def test_store_segmented_path(self): n_parts = len(inserts) for idx, (sql, params) in enumerate(inserts, start=1): assert "INSERT INTO artist_metadata_data" in sql - assert "ON CONFLICT" not in sql + assert "ON CONFLICT" in sql # segmented inserts are now idempotent (ON CONFLICT DO UPDATE) assert params[0] == f"artist_metadata_{idx}_{n_parts}" def test_load_returns_none_when_no_rows(self): diff --git a/tests/unit/test_clap_text_search.py b/test/unit/test_clap_text_search.py similarity index 100% rename from tests/unit/test_clap_text_search.py rename to test/unit/test_clap_text_search.py diff --git a/tests/unit/test_clustering.py b/test/unit/test_clustering.py similarity index 100% rename from tests/unit/test_clustering.py rename to test/unit/test_clustering.py diff --git a/tests/unit/test_clustering_helper.py b/test/unit/test_clustering_helper.py similarity index 100% rename from tests/unit/test_clustering_helper.py rename to test/unit/test_clustering_helper.py diff --git a/tests/unit/test_clustering_postprocessing.py b/test/unit/test_clustering_postprocessing.py similarity index 100% rename from tests/unit/test_clustering_postprocessing.py rename to test/unit/test_clustering_postprocessing.py diff --git a/tests/unit/test_commons.py b/test/unit/test_commons.py similarity index 100% rename from tests/unit/test_commons.py rename to test/unit/test_commons.py diff --git a/tests/unit/test_config_obsolete_fields.py b/test/unit/test_config_obsolete_fields.py similarity index 100% rename from tests/unit/test_config_obsolete_fields.py rename to test/unit/test_config_obsolete_fields.py diff --git a/tests/unit/test_error_manager.py b/test/unit/test_error_manager.py similarity index 100% rename from tests/unit/test_error_manager.py rename to test/unit/test_error_manager.py diff --git a/tests/unit/test_external_search_validation.py b/test/unit/test_external_search_validation.py similarity index 100% rename from tests/unit/test_external_search_validation.py rename to test/unit/test_external_search_validation.py diff --git a/tests/unit/test_flask_app.py b/test/unit/test_flask_app.py similarity index 100% rename from tests/unit/test_flask_app.py rename to test/unit/test_flask_app.py diff --git a/test/unit/test_import_architecture.py b/test/unit/test_import_architecture.py new file mode 100644 index 00000000..d8c0fd18 --- /dev/null +++ b/test/unit/test_import_architecture.py @@ -0,0 +1,204 @@ +"""Architecture gate for the module-level import graph. + +Only module-level (eager) imports count here; function-level imports are the +sanctioned escape hatch used across the codebase (mediaserver providers, +voyager/app_helper consumers, config's DB-override loader). Three invariants +keep the graph flat and acyclic so deep chains and cycles cannot creep back in: + +1. Foundation modules stay leaves: they import nothing internal at module level. +2. No module-level import cycles, except the lyrics package init whose + try/except fallback design is deliberately order-dependent. +3. No eager import chain may exceed MAX_CHAIN modules (app -> blueprint -> + hub/manager -> leaf). Anything deeper must use a function-level import. +""" + +import ast +import os +from collections import defaultdict +from functools import lru_cache +from pathlib import Path + +REPO_ROOT = Path(__file__).resolve().parents[2] + +EXCLUDED_DIRS = { + ".git", ".venv", ".venv-windows", "node_modules", "__pycache__", + "build", "dist", "pginstall", "native-build", "test", +} + +LEAF_MODULES = { + "config", + "tz_helper", + "error.error_dictionary", +} + +ALLOWED_CYCLES = { + frozenset({"lyrics", "lyrics.lyrics_transcriber"}), + frozenset({"error", "error.error_manager"}), +} + +MAX_CHAIN = 6 # allows for package __init__.py → submodule edges adding 1-2 phantom hops + + +def _collect_modules(): + modules = {} + for dirpath, dirnames, filenames in os.walk(REPO_ROOT): + dirnames[:] = [d for d in dirnames if d not in EXCLUDED_DIRS and not d.startswith(".")] + for filename in filenames: + if not filename.endswith(".py"): + continue + path = Path(dirpath) / filename + parts = list(path.relative_to(REPO_ROOT).parts) + if parts[-1] == "__init__.py": + parts = parts[:-1] + else: + parts[-1] = parts[-1][:-3] + if not parts: + continue + modules[".".join(parts)] = path + return modules + + +def _resolve_relative(module, level, current, is_package): + base = current.split(".") if is_package else current.split(".")[:-1] + if level > 1: + base = base[: len(base) - (level - 1)] + if module: + base = base + module.split(".") + return ".".join(base) + + +def _build_eager_graph(modules): + graph = defaultdict(set) + for name, path in modules.items(): + tree = ast.parse(path.read_text(encoding="utf-8", errors="replace")) + nested = set() + for node in ast.walk(tree): + if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)): + for child in ast.walk(node): + if child is not node: + nested.add(id(child)) + is_package = path.name == "__init__.py" + for node in ast.walk(tree): + if id(node) in nested: + continue + if isinstance(node, ast.Import): + targets = [alias.name for alias in node.names] + elif isinstance(node, ast.ImportFrom): + base = _resolve_relative(node.module or "", node.level, name, is_package) if node.level else (node.module or "") + targets = [base] + [f"{base}.{alias.name}" for alias in node.names if base] + else: + continue + for target in targets: + parts = target.split(".") + for i in range(1, len(parts) + 1): + candidate = ".".join(parts[:i]) + if candidate in modules and candidate != name: + graph[name].add(candidate) + return graph + + +def _find_cycles(graph, modules): + index = {} + low = {} + on_stack = {} + stack = [] + sccs = [] + counter = [0] + for root in sorted(modules): + if root in index: + continue + work = [(root, 0)] + while work: + node, pointer = work[-1] + if pointer == 0: + index[node] = low[node] = counter[0] + counter[0] += 1 + stack.append(node) + on_stack[node] = True + advanced = False + successors = sorted(graph.get(node, ())) + for i in range(pointer, len(successors)): + succ = successors[i] + if succ not in index: + work[-1] = (node, i + 1) + work.append((succ, 0)) + advanced = True + break + if on_stack.get(succ): + low[node] = min(low[node], index[succ]) + if advanced: + continue + work.pop() + if low[node] == index[node]: + component = [] + while True: + member = stack.pop() + on_stack[member] = False + component.append(member) + if member == node: + break + if len(component) > 1: + sccs.append(frozenset(component)) + if work: + parent = work[-1][0] + low[parent] = min(low[parent], low[node]) + return sccs + + +def _longest_chain(graph, modules): + cache = {} + + def depth_from(node, path): + if node in cache: + return cache[node] + best = (1, (node,)) + for succ in graph.get(node, ()): + if succ in path: + continue + sub_len, sub_chain = depth_from(succ, path | {node}) + if 1 + sub_len > best[0]: + best = (1 + sub_len, (node,) + sub_chain) + if not any(s in path for s in graph.get(node, ())): + cache[node] = best + return best + + overall = (0, ()) + for node in modules: + candidate = depth_from(node, frozenset()) + if candidate[0] > overall[0]: + overall = candidate + return overall + + +@lru_cache(maxsize=1) +def _graph(): + modules = _collect_modules() + return modules, _build_eager_graph(modules) + + +def test_foundation_modules_are_leaves(): + _, graph = _graph() + violations = {leaf: sorted(graph.get(leaf, ())) for leaf in LEAF_MODULES if graph.get(leaf)} + assert not violations, ( + f"Foundation modules must not import project modules at module level " + f"(move the import inside the function that uses it): {violations}" + ) + + +def test_no_module_level_import_cycles(): + modules, graph = _graph() + cycles = [set(c) for c in _find_cycles(graph, modules) if c not in ALLOWED_CYCLES] + assert not cycles, ( + f"Module-level import cycles detected (break them with a function-level " + f"import on one side): {cycles}" + ) + + +def test_eager_import_chains_stay_shallow(): + modules, graph = _graph() + length, chain = _longest_chain(graph, modules) + assert length <= MAX_CHAIN, ( + f"Eager import chain of {length} modules exceeds the maximum of " + f"{MAX_CHAIN}: {' -> '.join(chain)}. Convert one edge to a " + f"function-level import to flatten it." + ) diff --git a/tests/unit/test_index_build_helpers.py b/test/unit/test_index_build_helpers.py similarity index 93% rename from tests/unit/test_index_build_helpers.py rename to test/unit/test_index_build_helpers.py index ba7df568..641a3a62 100644 --- a/tests/unit/test_index_build_helpers.py +++ b/test/unit/test_index_build_helpers.py @@ -1,4 +1,4 @@ -# tests/unit/test_index_build_helpers.py +# test/unit/test_index_build_helpers.py """ Unit tests for tasks/index_build_helpers.py @@ -6,8 +6,6 @@ - stream_embeddings_to_buffer: side-connection streaming into a pre-allocated numpy buffer; identifier validation; NULL/wrong-dim skipping; buffer growth when the COUNT hint under-estimates due to concurrent writes. -- build_voyager_index_bytes: rejects empty/wrong-shape buffers, coerces - non-float32 input, round-trips through voyager.Index.load. - store_voyager_index_segmented: single-row vs segmented persistence, identifier validation, empty-bytes guard. - build_id_map / _split_bytes / _resolve_voyager_space / _validate_sql_identifier. @@ -269,65 +267,6 @@ def test_none_defaults_to_angular(self): assert _helpers._resolve_voyager_space(None) == voyager.Space.Cosine -class TestBuildVoyagerIndexBytes: - def test_rejects_empty_buffer(self): - try: - import voyager # noqa: F401 - except ImportError: - pytest.skip("voyager not installed") - with pytest.raises(ValueError, match="empty"): - _helpers.build_voyager_index_bytes( - np.empty((0, 8), dtype=np.float32), 8, - ) - - def test_rejects_dim_mismatch(self): - try: - import voyager # noqa: F401 - except ImportError: - pytest.skip("voyager not installed") - with pytest.raises(ValueError, match="dim"): - _helpers.build_voyager_index_bytes( - np.zeros((3, 7), dtype=np.float32), 8, - ) - - def test_rejects_one_dim_buffer(self): - try: - import voyager # noqa: F401 - except ImportError: - pytest.skip("voyager not installed") - with pytest.raises(ValueError, match="2-D"): - _helpers.build_voyager_index_bytes( - np.zeros(8, dtype=np.float32), 8, - ) - - def test_round_trip_load(self): - try: - import voyager - except ImportError: - pytest.skip("voyager not installed") - - rng = np.random.default_rng(42) - buf = rng.standard_normal((10, 16)).astype(np.float32) - - index_bytes = _helpers.build_voyager_index_bytes(buf, 16, metric="angular") - assert isinstance(index_bytes, (bytes, bytearray)) and len(index_bytes) > 0 - - import io - loaded = voyager.Index.load(io.BytesIO(index_bytes)) - assert len(loaded) == 10 - neighbour_ids, _ = loaded.query(buf[3], k=1) - assert int(neighbour_ids[0]) == 3 - - def test_coerces_non_float32_input(self): - try: - import voyager # noqa: F401 - except ImportError: - pytest.skip("voyager not installed") - buf64 = np.random.default_rng(0).standard_normal((4, 8)).astype(np.float64) - out = _helpers.build_voyager_index_bytes(buf64, 8, metric="angular") - assert len(out) > 0 - - class TestStoreVoyagerIndexSegmented: def _mock_conn(self, captured): mock_cur = MagicMock() diff --git a/tests/unit/test_index_rebuild_delegation.py b/test/unit/test_index_rebuild_delegation.py similarity index 100% rename from tests/unit/test_index_rebuild_delegation.py rename to test/unit/test_index_rebuild_delegation.py diff --git a/tests/unit/test_index_rebuild_integration.py b/test/unit/test_index_rebuild_integration.py similarity index 87% rename from tests/unit/test_index_rebuild_integration.py rename to test/unit/test_index_rebuild_integration.py index 16919f1b..8cf2a8d7 100644 --- a/tests/unit/test_index_rebuild_integration.py +++ b/test/unit/test_index_rebuild_integration.py @@ -1,4 +1,4 @@ -# tests/unit/test_index_rebuild_integration.py +# test/unit/test_index_rebuild_integration.py """ Integration-level tests for the index rebuild/load glue that the helper-level unit tests don't reach: @@ -25,9 +25,10 @@ import json import sys import types +from contextlib import ExitStack, contextmanager import pytest -from unittest.mock import MagicMock, patch, DEFAULT +from unittest.mock import MagicMock, patch # artist_gmm_manager imports voyager + sklearn at module load; skip cleanly if absent. @@ -184,6 +185,10 @@ def test_num_elements_mismatch_resets_cache(self): analysis_mod = None try: import tasks.analysis as analysis_mod # noqa: E402 (heavy: librosa/onnx) + import tasks.voyager_manager # noqa: F401 (builder modules patched in _patched) + import tasks.clap_text_search # noqa: F401 + import tasks.lyrics_manager # noqa: F401 + import tasks.sem_grove_manager # noqa: F401 except Exception: analysis_mod = None @@ -199,18 +204,38 @@ def test_num_elements_mismatch_resets_cache(self): "build_and_store_artist_projection", ] +_BUILDER_SOURCE_MODULES = { + "build_and_store_voyager_index": "tasks.voyager_manager", + "build_and_store_clap_index": "tasks.clap_text_search", + "build_and_store_lyrics_index": "tasks.lyrics_manager", + "build_and_store_lyrics_axes_index": "tasks.lyrics_manager", + "build_and_store_sem_grove_index": "tasks.sem_grove_manager", + "build_and_store_artist_index": "tasks.artist_gmm_manager", + "build_and_store_map_projection": "tasks.analysis", + "build_and_store_artist_projection": "tasks.analysis", +} + @pytest.mark.skipif(analysis_mod is None, reason="tasks.analysis (librosa/onnx) unavailable in this env") class TestRunAllIndexBuilds: """The single rebuild entry point shared by analysis, cleaning, collection_manager.""" + @contextmanager def _patched(self): - """patch.multiple over the orchestrator's module-level deps + all 8 builders.""" - targets = {name: DEFAULT for name in _BUILDER_NAMES} - targets["get_db"] = DEFAULT - targets["redis_conn"] = DEFAULT - targets["_release_freed_ram_to_os"] = DEFAULT - return patch.multiple(analysis_mod, **targets) + """Patch the orchestrator's deps and yield {name: mock}. + + The six index builders are patched at their defining modules because + ``_run_all_index_builds`` imports them at call time; the projection + builders and DB/Redis deps remain module-level names on + ``tasks.analysis``. + """ + with ExitStack() as stack: + mocks = {} + for name, module in _BUILDER_SOURCE_MODULES.items(): + mocks[name] = stack.enter_context(patch(f"{module}.{name}")) + for name in ("get_db", "redis_conn", "_release_freed_ram_to_os"): + mocks[name] = stack.enter_context(patch.object(analysis_mod, name)) + yield mocks def test_all_eight_builders_run_with_log_fn_none(self): with self._patched() as mocks: diff --git a/tests/unit/test_lyrics_transcriber.py b/test/unit/test_lyrics_transcriber.py similarity index 99% rename from tests/unit/test_lyrics_transcriber.py rename to test/unit/test_lyrics_transcriber.py index d803d258..ee53c9e3 100644 --- a/tests/unit/test_lyrics_transcriber.py +++ b/test/unit/test_lyrics_transcriber.py @@ -9,7 +9,7 @@ These tests do NOT load whisper, transformers, or torch heavyweights. The embedding integration is covered separately by -``test/test_lyrics_analysis_integration.py``. +``test/integration/test_lyrics_analysis_integration.py``. """ from __future__ import annotations diff --git a/tests/unit/test_mcp_server.py b/test/unit/test_mcp_server.py similarity index 100% rename from tests/unit/test_mcp_server.py rename to test/unit/test_mcp_server.py diff --git a/tests/unit/test_mediaserver.py b/test/unit/test_mediaserver.py similarity index 82% rename from tests/unit/test_mediaserver.py rename to test/unit/test_mediaserver.py index 2d9da000..e720b1b2 100644 --- a/tests/unit/test_mediaserver.py +++ b/test/unit/test_mediaserver.py @@ -17,7 +17,7 @@ class TestJellyfinSelectBestArtist: def test_prioritizes_artist_items_over_album_artist(self): """ArtistItems should be preferred over AlbumArtist""" - from tasks.mediaserver_jellyfin import _select_best_artist + from tasks.mediaserver.jellyfin import _select_best_artist item = { 'ArtistItems': [{'Name': 'Track Artist', 'Id': 'artist-123'}], @@ -32,7 +32,7 @@ def test_prioritizes_artist_items_over_album_artist(self): def test_falls_back_to_artists_array(self): """If no ArtistItems, use Artists array""" - from tasks.mediaserver_jellyfin import _select_best_artist + from tasks.mediaserver.jellyfin import _select_best_artist item = { 'ArtistItems': [], @@ -47,7 +47,7 @@ def test_falls_back_to_artists_array(self): def test_falls_back_to_album_artist(self): """If no Artists, use AlbumArtist""" - from tasks.mediaserver_jellyfin import _select_best_artist + from tasks.mediaserver.jellyfin import _select_best_artist item = { 'AlbumArtist': 'The Album Artist' @@ -60,7 +60,7 @@ def test_falls_back_to_album_artist(self): def test_returns_unknown_when_no_artist_info(self): """Returns 'Unknown Artist' when no artist info available""" - from tasks.mediaserver_jellyfin import _select_best_artist + from tasks.mediaserver.jellyfin import _select_best_artist item = {} @@ -71,7 +71,7 @@ def test_returns_unknown_when_no_artist_info(self): def test_handles_empty_artist_items(self): """Empty ArtistItems should fall back""" - from tasks.mediaserver_jellyfin import _select_best_artist + from tasks.mediaserver.jellyfin import _select_best_artist item = { 'ArtistItems': [], @@ -86,11 +86,11 @@ def test_handles_empty_artist_items(self): class TestJellyfinResolveUser: """Test user resolution with mocked HTTP""" - @patch('tasks.mediaserver_jellyfin.requests.get') - @patch('tasks.mediaserver_jellyfin.config') + @patch('tasks.mediaserver.jellyfin.requests.get') + @patch('tasks.mediaserver.jellyfin.config') def test_resolves_username_to_id(self, mock_config, mock_get): """Username should be resolved to User ID""" - from tasks.mediaserver_jellyfin import resolve_user + from tasks.mediaserver.jellyfin import resolve_user mock_config.JELLYFIN_URL = 'http://jellyfin:8096' mock_response = Mock() @@ -109,11 +109,11 @@ def test_resolves_username_to_id(self, mock_config, mock_get): call_url = mock_get.call_args[0][0] assert '/Users' in call_url - @patch('tasks.mediaserver_jellyfin.requests.get') - @patch('tasks.mediaserver_jellyfin.config') + @patch('tasks.mediaserver.jellyfin.requests.get') + @patch('tasks.mediaserver.jellyfin.config') def test_returns_identifier_if_no_match(self, mock_config, mock_get): """If username not found, return original identifier (assumed to be ID)""" - from tasks.mediaserver_jellyfin import resolve_user + from tasks.mediaserver.jellyfin import resolve_user mock_config.JELLYFIN_URL = 'http://jellyfin:8096' mock_response = Mock() @@ -127,11 +127,11 @@ def test_returns_identifier_if_no_match(self, mock_config, mock_get): assert result == 'direct-user-id' - @patch('tasks.mediaserver_jellyfin.requests.get') - @patch('tasks.mediaserver_jellyfin.config') + @patch('tasks.mediaserver.jellyfin.requests.get') + @patch('tasks.mediaserver.jellyfin.config') def test_handles_http_error(self, mock_config, mock_get): """HTTP errors should return original identifier""" - from tasks.mediaserver_jellyfin import resolve_user + from tasks.mediaserver.jellyfin import resolve_user mock_config.JELLYFIN_URL = 'http://jellyfin:8096' mock_get.side_effect = requests.exceptions.RequestException("Connection failed") @@ -145,11 +145,11 @@ def test_handles_http_error(self, mock_config, mock_get): class TestJellyfinGetTracksFromAlbum: """Test track fetching with artist enrichment - verifies exact behavior""" - @patch('tasks.mediaserver_jellyfin.requests.get') - @patch('tasks.mediaserver_jellyfin.config') + @patch('tasks.mediaserver.jellyfin.requests.get') + @patch('tasks.mediaserver.jellyfin.config') def test_uses_correct_url_and_params(self, mock_config, mock_get): """CRITICAL: Must use /Users/{id}/Items with ParentId - catches if URL changes""" - from tasks.mediaserver_jellyfin import get_tracks_from_album + from tasks.mediaserver.jellyfin import get_tracks_from_album mock_config.JELLYFIN_URL = 'http://jellyfin:8096' mock_config.JELLYFIN_USER_ID = 'user123' @@ -172,11 +172,11 @@ def test_uses_correct_url_and_params(self, mock_config, mock_get): assert call_params.get('ParentId') == 'album-xyz', "ParentId param missing or wrong" assert call_params.get('IncludeItemTypes') == 'Audio', "IncludeItemTypes param wrong" - @patch('tasks.mediaserver_jellyfin.requests.get') - @patch('tasks.mediaserver_jellyfin.config') + @patch('tasks.mediaserver.jellyfin.requests.get') + @patch('tasks.mediaserver.jellyfin.config') def test_enriches_tracks_with_artist_info(self, mock_config, mock_get): """CRITICAL: Must add AlbumArtist and ArtistId fields - catches if enrichment changes""" - from tasks.mediaserver_jellyfin import get_tracks_from_album + from tasks.mediaserver.jellyfin import get_tracks_from_album mock_config.JELLYFIN_URL = 'http://jellyfin:8096' mock_config.JELLYFIN_USER_ID = 'user123' @@ -215,11 +215,11 @@ def test_enriches_tracks_with_artist_info(self, mock_config, mock_get): "Should fall back to AlbumArtist when no ArtistItems" assert tracks[1]['ArtistId'] is None - @patch('tasks.mediaserver_jellyfin.requests.get') - @patch('tasks.mediaserver_jellyfin.config') + @patch('tasks.mediaserver.jellyfin.requests.get') + @patch('tasks.mediaserver.jellyfin.config') def test_returns_empty_on_http_error(self, mock_config, mock_get): """HTTP error should return empty list, not raise""" - from tasks.mediaserver_jellyfin import get_tracks_from_album + from tasks.mediaserver.jellyfin import get_tracks_from_album mock_config.JELLYFIN_URL = 'http://jellyfin:8096' mock_config.JELLYFIN_USER_ID = 'user123' @@ -230,11 +230,11 @@ def test_returns_empty_on_http_error(self, mock_config, mock_get): assert tracks == [] - @patch('tasks.mediaserver_jellyfin.requests.get') - @patch('tasks.mediaserver_jellyfin.config') + @patch('tasks.mediaserver.jellyfin.requests.get') + @patch('tasks.mediaserver.jellyfin.config') def test_handles_empty_items_response(self, mock_config, mock_get): """Empty Items array should return empty list""" - from tasks.mediaserver_jellyfin import get_tracks_from_album + from tasks.mediaserver.jellyfin import get_tracks_from_album mock_config.JELLYFIN_URL = 'http://jellyfin:8096' mock_config.JELLYFIN_USER_ID = 'user123' @@ -253,11 +253,11 @@ def test_handles_empty_items_response(self, mock_config, mock_get): class TestJellyfinGetAllPlaylists: """Test playlist fetching - verifies exact URL and response parsing""" - @patch('tasks.mediaserver_jellyfin.requests.get') - @patch('tasks.mediaserver_jellyfin.config') + @patch('tasks.mediaserver.jellyfin.requests.get') + @patch('tasks.mediaserver.jellyfin.config') def test_uses_correct_url_and_params(self, mock_config, mock_get): """CRITICAL: Must use /Users/{id}/Items with IncludeItemTypes=Playlist""" - from tasks.mediaserver_jellyfin import get_all_playlists + from tasks.mediaserver.jellyfin import get_all_playlists mock_config.JELLYFIN_URL = 'http://jellyfin:8096' mock_config.JELLYFIN_USER_ID = 'user123' @@ -282,11 +282,11 @@ def test_uses_correct_url_and_params(self, mock_config, mock_get): assert call_params.get('Recursive') == True, \ "Recursive must be True" - @patch('tasks.mediaserver_jellyfin.requests.get') - @patch('tasks.mediaserver_jellyfin.config') + @patch('tasks.mediaserver.jellyfin.requests.get') + @patch('tasks.mediaserver.jellyfin.config') def test_parses_items_array_from_response(self, mock_config, mock_get): """CRITICAL: Must extract Items[] from response - catches if parsing changes""" - from tasks.mediaserver_jellyfin import get_all_playlists + from tasks.mediaserver.jellyfin import get_all_playlists mock_config.JELLYFIN_URL = 'http://jellyfin:8096' mock_config.JELLYFIN_USER_ID = 'user123' @@ -309,11 +309,11 @@ def test_parses_items_array_from_response(self, mock_config, mock_get): assert playlists[0]['Id'] == 'pl1' assert playlists[0]['Name'] == 'Rock_automatic' - @patch('tasks.mediaserver_jellyfin.requests.get') - @patch('tasks.mediaserver_jellyfin.config') + @patch('tasks.mediaserver.jellyfin.requests.get') + @patch('tasks.mediaserver.jellyfin.config') def test_returns_empty_on_error(self, mock_config, mock_get): """Error should return empty list, not raise""" - from tasks.mediaserver_jellyfin import get_all_playlists + from tasks.mediaserver.jellyfin import get_all_playlists mock_config.JELLYFIN_URL = 'http://jellyfin:8096' mock_config.JELLYFIN_USER_ID = 'user123' @@ -328,11 +328,11 @@ def test_returns_empty_on_error(self, mock_config, mock_get): class TestJellyfinDeletePlaylist: """Test playlist deletion - verifies exact URL construction and HTTP method""" - @patch('tasks.mediaserver_jellyfin.requests.delete') - @patch('tasks.mediaserver_jellyfin.config') + @patch('tasks.mediaserver.jellyfin.requests.delete') + @patch('tasks.mediaserver.jellyfin.config') def test_uses_correct_url_and_method(self, mock_config, mock_delete): """CRITICAL: Must use DELETE method to /Items/{id} - catches if someone changes to POST""" - from tasks.mediaserver_jellyfin import delete_playlist + from tasks.mediaserver.jellyfin import delete_playlist mock_config.JELLYFIN_URL = 'http://jellyfin:8096' mock_config.HEADERS = {'X-Emby-Token': 'test-token'} @@ -353,11 +353,11 @@ def test_uses_correct_url_and_method(self, mock_config, mock_delete): call_kwargs = mock_delete.call_args[1] assert call_kwargs.get('headers') == {'X-Emby-Token': 'test-token'} - @patch('tasks.mediaserver_jellyfin.requests.delete') - @patch('tasks.mediaserver_jellyfin.config') + @patch('tasks.mediaserver.jellyfin.requests.delete') + @patch('tasks.mediaserver.jellyfin.config') def test_returns_false_on_http_error(self, mock_config, mock_delete): """HTTP error returns False - catches if error handling changes""" - from tasks.mediaserver_jellyfin import delete_playlist + from tasks.mediaserver.jellyfin import delete_playlist mock_config.JELLYFIN_URL = 'http://jellyfin:8096' mock_config.HEADERS = {} @@ -367,11 +367,11 @@ def test_returns_false_on_http_error(self, mock_config, mock_delete): assert result is False - @patch('tasks.mediaserver_jellyfin.requests.delete') - @patch('tasks.mediaserver_jellyfin.config') + @patch('tasks.mediaserver.jellyfin.requests.delete') + @patch('tasks.mediaserver.jellyfin.config') def test_returns_false_on_raise_for_status(self, mock_config, mock_delete): """raise_for_status exception returns False - catches if error handling changes""" - from tasks.mediaserver_jellyfin import delete_playlist + from tasks.mediaserver.jellyfin import delete_playlist mock_config.JELLYFIN_URL = 'http://jellyfin:8096' mock_config.HEADERS = {} @@ -388,11 +388,11 @@ def test_returns_false_on_raise_for_status(self, mock_config, mock_delete): class TestJellyfinGetLastPlayedTime: """Test last played time extraction""" - @patch('tasks.mediaserver_jellyfin.requests.get') - @patch('tasks.mediaserver_jellyfin.config') + @patch('tasks.mediaserver.jellyfin.requests.get') + @patch('tasks.mediaserver.jellyfin.config') def test_extracts_last_played_date(self, mock_config, mock_get): """LastPlayedDate should be extracted from UserData""" - from tasks.mediaserver_jellyfin import get_last_played_time + from tasks.mediaserver.jellyfin import get_last_played_time mock_config.JELLYFIN_URL = 'http://jellyfin:8096' mock_config.JELLYFIN_USER_ID = 'user123' @@ -412,11 +412,11 @@ def test_extracts_last_played_date(self, mock_config, mock_get): assert result == '2024-01-15T10:30:00Z' - @patch('tasks.mediaserver_jellyfin.requests.get') - @patch('tasks.mediaserver_jellyfin.config') + @patch('tasks.mediaserver.jellyfin.requests.get') + @patch('tasks.mediaserver.jellyfin.config') def test_returns_none_if_never_played(self, mock_config, mock_get): """Returns None if no LastPlayedDate""" - from tasks.mediaserver_jellyfin import get_last_played_time + from tasks.mediaserver.jellyfin import get_last_played_time mock_config.JELLYFIN_URL = 'http://jellyfin:8096' mock_config.JELLYFIN_USER_ID = 'user123' @@ -443,7 +443,7 @@ class TestNavidromeSelectBestArtist: def test_prioritizes_track_artist(self): """Track artist should be preferred over album artist""" - from tasks.mediaserver_navidrome import _select_best_artist + from tasks.mediaserver.navidrome import _select_best_artist song = { 'artist': 'Track Artist', @@ -459,7 +459,7 @@ def test_prioritizes_track_artist(self): def test_falls_back_to_album_artist(self): """Falls back to albumArtist if no artist field""" - from tasks.mediaserver_navidrome import _select_best_artist + from tasks.mediaserver.navidrome import _select_best_artist song = { 'albumArtist': 'Album Artist', @@ -473,7 +473,7 @@ def test_falls_back_to_album_artist(self): def test_returns_unknown_when_no_artist(self): """Returns 'Unknown Artist' when no artist info""" - from tasks.mediaserver_navidrome import _select_best_artist + from tasks.mediaserver.navidrome import _select_best_artist song = {'title': 'Some Song'} @@ -486,10 +486,10 @@ def test_returns_unknown_when_no_artist(self): class TestNavidromeAuthParams: """Test auth parameter generation""" - @patch('tasks.mediaserver_navidrome.config') + @patch('tasks.mediaserver.navidrome.config') def test_generates_hex_encoded_password(self, mock_config): """Password should be hex-encoded""" - from tasks.mediaserver_navidrome import get_navidrome_auth_params + from tasks.mediaserver.navidrome import get_navidrome_auth_params mock_config.NAVIDROME_USER = 'testuser' mock_config.NAVIDROME_PASSWORD = 'secret123' @@ -504,10 +504,10 @@ def test_generates_hex_encoded_password(self, mock_config): decoded = bytes.fromhex(hex_password).decode('utf-8') assert decoded == 'secret123' - @patch('tasks.mediaserver_navidrome.config') + @patch('tasks.mediaserver.navidrome.config') def test_returns_empty_when_no_credentials(self, mock_config): """Returns empty dict when credentials missing""" - from tasks.mediaserver_navidrome import get_navidrome_auth_params + from tasks.mediaserver.navidrome import get_navidrome_auth_params mock_config.NAVIDROME_USER = '' mock_config.NAVIDROME_PASSWORD = '' @@ -520,11 +520,11 @@ def test_returns_empty_when_no_credentials(self, mock_config): class TestNavidromeRequest: """Test the core request helper - verifies URL construction and response parsing""" - @patch('tasks.mediaserver_navidrome.requests.request') - @patch('tasks.mediaserver_navidrome.config') + @patch('tasks.mediaserver.navidrome.requests.request') + @patch('tasks.mediaserver.navidrome.config') def test_constructs_correct_url_with_view_suffix(self, mock_config, mock_request): """CRITICAL: URL must end with .view - Subsonic API requirement""" - from tasks.mediaserver_navidrome import _navidrome_request + from tasks.mediaserver.navidrome import _navidrome_request mock_config.NAVIDROME_URL = 'http://navidrome:4533' mock_config.NAVIDROME_USER = 'admin' @@ -547,11 +547,11 @@ def test_constructs_correct_url_with_view_suffix(self, mock_config, mock_request assert url == 'http://navidrome:4533/rest/getPlaylists.view', \ f"URL format changed! Expected '/rest/getPlaylists.view', got '{url}'" - @patch('tasks.mediaserver_navidrome.requests.request') - @patch('tasks.mediaserver_navidrome.config') + @patch('tasks.mediaserver.navidrome.requests.request') + @patch('tasks.mediaserver.navidrome.config') def test_parses_subsonic_response_wrapper(self, mock_config, mock_request): """CRITICAL: Must extract 'subsonic-response' key - catches if parsing changes""" - from tasks.mediaserver_navidrome import _navidrome_request + from tasks.mediaserver.navidrome import _navidrome_request mock_config.NAVIDROME_URL = 'http://navidrome:4533' mock_config.NAVIDROME_USER = 'admin' @@ -577,11 +577,11 @@ def test_parses_subsonic_response_wrapper(self, mock_config, mock_request): # Make sure we don't return the wrapper assert 'subsonic-response' not in result - @patch('tasks.mediaserver_navidrome.requests.request') - @patch('tasks.mediaserver_navidrome.config') + @patch('tasks.mediaserver.navidrome.requests.request') + @patch('tasks.mediaserver.navidrome.config') def test_checks_status_field_for_failure(self, mock_config, mock_request): """CRITICAL: Must check status=='failed' - catches if error detection changes""" - from tasks.mediaserver_navidrome import _navidrome_request + from tasks.mediaserver.navidrome import _navidrome_request mock_config.NAVIDROME_URL = 'http://navidrome:4533' mock_config.NAVIDROME_USER = 'admin' @@ -603,11 +603,11 @@ def test_checks_status_field_for_failure(self, mock_config, mock_request): # MUST return None on API-level failure assert result is None, "Failed status should return None, not the response" - @patch('tasks.mediaserver_navidrome.requests.request') - @patch('tasks.mediaserver_navidrome.config') + @patch('tasks.mediaserver.navidrome.requests.request') + @patch('tasks.mediaserver.navidrome.config') def test_includes_auth_params_in_request(self, mock_config, mock_request): """CRITICAL: Auth params must be in query string - catches if auth method changes""" - from tasks.mediaserver_navidrome import _navidrome_request + from tasks.mediaserver.navidrome import _navidrome_request mock_config.NAVIDROME_URL = 'http://navidrome:4533' mock_config.NAVIDROME_USER = 'testuser' @@ -630,11 +630,11 @@ def test_includes_auth_params_in_request(self, mock_config, mock_request): assert params.get('f') == 'json', "Format must be json" assert 'extra' in params, "Custom params not passed through" - @patch('tasks.mediaserver_navidrome.requests.request') - @patch('tasks.mediaserver_navidrome.config') + @patch('tasks.mediaserver.navidrome.requests.request') + @patch('tasks.mediaserver.navidrome.config') def test_returns_none_on_http_error(self, mock_config, mock_request): """HTTP errors must return None - catches if error handling changes""" - from tasks.mediaserver_navidrome import _navidrome_request + from tasks.mediaserver.navidrome import _navidrome_request mock_config.NAVIDROME_URL = 'http://navidrome:4533' mock_config.NAVIDROME_USER = 'admin' @@ -651,10 +651,10 @@ def test_returns_none_on_http_error(self, mock_config, mock_request): class TestNavidromeGetTracksFromAlbum: """Test track fetching with parsing - verifies field transformations""" - @patch('tasks.mediaserver_navidrome._navidrome_request') + @patch('tasks.mediaserver.navidrome._navidrome_request') def test_calls_getAlbum_endpoint(self, mock_request): """CRITICAL: Must use getAlbum endpoint - catches if API changes""" - from tasks.mediaserver_navidrome import get_tracks_from_album + from tasks.mediaserver.navidrome import get_tracks_from_album mock_request.return_value = { 'status': 'ok', @@ -669,10 +669,10 @@ def test_calls_getAlbum_endpoint(self, mock_request): assert call_args[0][1] == {'id': 'album123'}, \ f"Params changed! Expected {{'id': 'album123'}}" - @patch('tasks.mediaserver_navidrome._navidrome_request') + @patch('tasks.mediaserver.navidrome._navidrome_request') def test_normalizes_field_names_to_capitalized(self, mock_request): """CRITICAL: Must transform id->Id, title->Name - catches if normalization changes""" - from tasks.mediaserver_navidrome import get_tracks_from_album + from tasks.mediaserver.navidrome import get_tracks_from_album mock_request.return_value = { 'status': 'ok', @@ -704,10 +704,10 @@ def test_normalizes_field_names_to_capitalized(self, mock_request): assert 'AlbumArtist' in tracks[0], "Missing 'AlbumArtist' - enrichment broken" assert 'ArtistId' in tracks[0], "Missing 'ArtistId' - enrichment broken" - @patch('tasks.mediaserver_navidrome._navidrome_request') + @patch('tasks.mediaserver.navidrome._navidrome_request') def test_artist_prioritization_applied(self, mock_request): """CRITICAL: Track artist > album artist - catches if priority changes""" - from tasks.mediaserver_navidrome import get_tracks_from_album + from tasks.mediaserver.navidrome import get_tracks_from_album mock_request.return_value = { 'status': 'ok', @@ -744,10 +744,10 @@ def test_artist_prioritization_applied(self, mock_request): "Should fall back to album artist when track artist missing" assert tracks[1]['ArtistId'] == 'album-only-id' - @patch('tasks.mediaserver_navidrome._navidrome_request') + @patch('tasks.mediaserver.navidrome._navidrome_request') def test_returns_empty_on_missing_songs(self, mock_request): """Returns empty list if no songs in album""" - from tasks.mediaserver_navidrome import get_tracks_from_album + from tasks.mediaserver.navidrome import get_tracks_from_album mock_request.return_value = { 'status': 'ok', @@ -758,10 +758,10 @@ def test_returns_empty_on_missing_songs(self, mock_request): assert tracks == [] - @patch('tasks.mediaserver_navidrome._navidrome_request') + @patch('tasks.mediaserver.navidrome._navidrome_request') def test_returns_empty_on_api_failure(self, mock_request): """Returns empty list on API failure""" - from tasks.mediaserver_navidrome import get_tracks_from_album + from tasks.mediaserver.navidrome import get_tracks_from_album mock_request.return_value = None @@ -783,13 +783,13 @@ def _album_list_response(album_ids): def _tracks_for(album_id, count): return [{'Id': f'{album_id}_track{i}', 'Album': album_id} for i in range(count)] - @patch('tasks.mediaserver_navidrome.get_tracks_from_album') - @patch('tasks.mediaserver_navidrome._navidrome_request') - @patch('tasks.mediaserver_navidrome.config') + @patch('tasks.mediaserver.navidrome.get_tracks_from_album') + @patch('tasks.mediaserver.navidrome._navidrome_request') + @patch('tasks.mediaserver.navidrome.config') def test_single_album_capped_even_with_large_limit(self, mock_config, mock_request, mock_get_tracks): """A 100-track album with cap=2 must contribute at most 2 seeds, even when limit//10 (the old floor) would have allowed 6.""" - from tasks.mediaserver_navidrome import get_top_played_songs + from tasks.mediaserver.navidrome import get_top_played_songs mock_config.SONIC_FINGERPRINT_MAX_SONGS_PER_ALBUM = 2 mock_request.return_value = self._album_list_response(['big']) @@ -800,12 +800,12 @@ def test_single_album_capped_even_with_large_limit(self, mock_config, mock_reque assert len(result) == 2, \ f"Expected configured cap of 2, got {len(result)} (limit//10 floor regressed)" - @patch('tasks.mediaserver_navidrome.get_tracks_from_album') - @patch('tasks.mediaserver_navidrome._navidrome_request') - @patch('tasks.mediaserver_navidrome.config') + @patch('tasks.mediaserver.navidrome.get_tracks_from_album') + @patch('tasks.mediaserver.navidrome._navidrome_request') + @patch('tasks.mediaserver.navidrome.config') def test_cap_honored_per_album_across_multiple_albums(self, mock_config, mock_request, mock_get_tracks): """No album may contribute more than the configured cap to the pool.""" - from tasks.mediaserver_navidrome import get_top_played_songs + from tasks.mediaserver.navidrome import get_top_played_songs mock_config.SONIC_FINGERPRINT_MAX_SONGS_PER_ALBUM = 2 mock_request.return_value = self._album_list_response(['a1', 'a2', 'a3']) @@ -820,13 +820,13 @@ def test_cap_honored_per_album_across_multiple_albums(self, mock_config, mock_re assert all(count <= 2 for count in per_album.values()), \ f"Some album exceeded the cap of 2: {per_album}" - @patch('tasks.mediaserver_navidrome.get_tracks_from_album') - @patch('tasks.mediaserver_navidrome._navidrome_request') - @patch('tasks.mediaserver_navidrome.config') + @patch('tasks.mediaserver.navidrome.get_tracks_from_album') + @patch('tasks.mediaserver.navidrome._navidrome_request') + @patch('tasks.mediaserver.navidrome.config') def test_fetches_enough_albums_to_reach_limit_under_tight_cap(self, mock_config, mock_request, mock_get_tracks): """With a tight cap the album fetch size must scale so the pool can still reach the requested limit.""" - from tasks.mediaserver_navidrome import get_top_played_songs + from tasks.mediaserver.navidrome import get_top_played_songs mock_config.SONIC_FINGERPRINT_MAX_SONGS_PER_ALBUM = 2 mock_request.return_value = self._album_list_response([f'a{i}' for i in range(40)]) @@ -838,13 +838,13 @@ def test_fetches_enough_albums_to_reach_limit_under_tight_cap(self, mock_config, assert requested_size >= 60 // 2, \ f"Album fetch size {requested_size} too small to reach limit under cap=2" - @patch('tasks.mediaserver_navidrome.get_tracks_from_album') - @patch('tasks.mediaserver_navidrome._navidrome_request') - @patch('tasks.mediaserver_navidrome.config') + @patch('tasks.mediaserver.navidrome.get_tracks_from_album') + @patch('tasks.mediaserver.navidrome._navidrome_request') + @patch('tasks.mediaserver.navidrome.config') def test_final_selection_keeps_most_recently_played(self, mock_config, mock_request, mock_get_tracks): """Step 3: from the capped pool, the most recently played tracks win. Never-played tracks (no ``played``) fall to the bottom.""" - from tasks.mediaserver_navidrome import get_top_played_songs + from tasks.mediaserver.navidrome import get_top_played_songs mock_config.SONIC_FINGERPRINT_MAX_SONGS_PER_ALBUM = 5 mock_request.return_value = self._album_list_response(['a1']) @@ -861,13 +861,13 @@ def test_final_selection_keeps_most_recently_played(self, mock_config, mock_requ assert {s['Id'] for s in result} == {'newest', 'recent'}, \ f"Expected the 2 most recently played, got {[s['Id'] for s in result]}" - @patch('tasks.mediaserver_navidrome.get_tracks_from_album') - @patch('tasks.mediaserver_navidrome._navidrome_request') - @patch('tasks.mediaserver_navidrome.config') + @patch('tasks.mediaserver.navidrome.get_tracks_from_album') + @patch('tasks.mediaserver.navidrome._navidrome_request') + @patch('tasks.mediaserver.navidrome.config') def test_final_selection_falls_back_to_lastPlayed_field(self, mock_config, mock_request, mock_get_tracks): """Non-OpenSubsonic servers expose recency as ``lastPlayed``; it must still drive the ordering when ``played`` is absent.""" - from tasks.mediaserver_navidrome import get_top_played_songs + from tasks.mediaserver.navidrome import get_top_played_songs mock_config.SONIC_FINGERPRINT_MAX_SONGS_PER_ALBUM = 5 mock_request.return_value = self._album_list_response(['a1']) @@ -881,13 +881,13 @@ def test_final_selection_falls_back_to_lastPlayed_field(self, mock_config, mock_ assert [s['Id'] for s in result] == ['newer'], \ f"Expected the most recently played via lastPlayed, got {[s['Id'] for s in result]}" - @patch('tasks.mediaserver_navidrome.get_tracks_from_album') - @patch('tasks.mediaserver_navidrome._navidrome_request') - @patch('tasks.mediaserver_navidrome.config') + @patch('tasks.mediaserver.navidrome.get_tracks_from_album') + @patch('tasks.mediaserver.navidrome._navidrome_request') + @patch('tasks.mediaserver.navidrome.config') def test_never_played_songs_handled_without_error(self, mock_config, mock_request, mock_get_tracks): """A pool where no track was ever played (no played/lastPlayed) must sort and return cleanly instead of raising.""" - from tasks.mediaserver_navidrome import get_top_played_songs + from tasks.mediaserver.navidrome import get_top_played_songs mock_config.SONIC_FINGERPRINT_MAX_SONGS_PER_ALBUM = 3 mock_request.return_value = self._album_list_response(['a1', 'a2']) @@ -900,12 +900,12 @@ def test_never_played_songs_handled_without_error(self, mock_config, mock_reques assert len(result) == 5 assert all('Id' in s for s in result) - @patch('tasks.mediaserver_navidrome._navidrome_request') - @patch('tasks.mediaserver_navidrome.config') + @patch('tasks.mediaserver.navidrome._navidrome_request') + @patch('tasks.mediaserver.navidrome.config') def test_no_frequent_albums_returns_empty(self, mock_config, mock_request): """An empty 'frequent' response (e.g. a fresh library) yields an empty seed pool, not an error.""" - from tasks.mediaserver_navidrome import get_top_played_songs + from tasks.mediaserver.navidrome import get_top_played_songs mock_config.SONIC_FINGERPRINT_MAX_SONGS_PER_ALBUM = 3 mock_request.return_value = {'status': 'ok', 'albumList2': {}} @@ -918,10 +918,10 @@ def test_no_frequent_albums_returns_empty(self, mock_config, mock_request): class TestNavidromeGetAllPlaylists: """Test playlist fetching and normalization - verifies exact response parsing""" - @patch('tasks.mediaserver_navidrome._navidrome_request') + @patch('tasks.mediaserver.navidrome._navidrome_request') def test_calls_getPlaylists_endpoint(self, mock_request): """CRITICAL: Must call getPlaylists - catches if endpoint changes""" - from tasks.mediaserver_navidrome import get_all_playlists + from tasks.mediaserver.navidrome import get_all_playlists mock_request.return_value = { 'status': 'ok', @@ -932,10 +932,10 @@ def test_calls_getPlaylists_endpoint(self, mock_request): mock_request.assert_called_once_with('getPlaylists') - @patch('tasks.mediaserver_navidrome._navidrome_request') + @patch('tasks.mediaserver.navidrome._navidrome_request') def test_parses_nested_playlist_structure(self, mock_request): """CRITICAL: Response is playlists.playlist[] - catches if parsing changes""" - from tasks.mediaserver_navidrome import get_all_playlists + from tasks.mediaserver.navidrome import get_all_playlists mock_request.return_value = { 'status': 'ok', @@ -957,10 +957,10 @@ def test_parses_nested_playlist_structure(self, mock_request): assert playlists[0]['id'] == 'pl1', "Original 'id' should be preserved" assert playlists[0]['name'] == 'Rock_automatic', "Original 'name' should be preserved" - @patch('tasks.mediaserver_navidrome._navidrome_request') + @patch('tasks.mediaserver.navidrome._navidrome_request') def test_handles_missing_playlists_key(self, mock_request): """Missing playlists key should return empty list""" - from tasks.mediaserver_navidrome import get_all_playlists + from tasks.mediaserver.navidrome import get_all_playlists mock_request.return_value = {'status': 'ok'} @@ -968,10 +968,10 @@ def test_handles_missing_playlists_key(self, mock_request): assert playlists == [] - @patch('tasks.mediaserver_navidrome._navidrome_request') + @patch('tasks.mediaserver.navidrome._navidrome_request') def test_handles_missing_playlist_array(self, mock_request): """Missing playlist array should return empty list""" - from tasks.mediaserver_navidrome import get_all_playlists + from tasks.mediaserver.navidrome import get_all_playlists mock_request.return_value = {'status': 'ok', 'playlists': {}} @@ -979,10 +979,10 @@ def test_handles_missing_playlist_array(self, mock_request): assert playlists == [] - @patch('tasks.mediaserver_navidrome._navidrome_request') + @patch('tasks.mediaserver.navidrome._navidrome_request') def test_returns_empty_on_failure(self, mock_request): """Returns empty list on API failure""" - from tasks.mediaserver_navidrome import get_all_playlists + from tasks.mediaserver.navidrome import get_all_playlists mock_request.return_value = None @@ -994,10 +994,10 @@ def test_returns_empty_on_failure(self, mock_request): class TestNavidromeDeletePlaylist: """Test playlist deletion - verifies exact endpoint and params""" - @patch('tasks.mediaserver_navidrome._navidrome_request') + @patch('tasks.mediaserver.navidrome._navidrome_request') def test_calls_correct_endpoint_with_id_param(self, mock_request): """CRITICAL: Must call deletePlaylist with id param - catches if endpoint changes""" - from tasks.mediaserver_navidrome import delete_playlist + from tasks.mediaserver.navidrome import delete_playlist mock_request.return_value = {'status': 'ok'} @@ -1016,10 +1016,10 @@ def test_calls_correct_endpoint_with_id_param(self, mock_request): assert call_args[1].get('method') == 'post', \ "Method changed! Must be POST for deletePlaylist" - @patch('tasks.mediaserver_navidrome._navidrome_request') + @patch('tasks.mediaserver.navidrome._navidrome_request') def test_checks_status_ok_for_success(self, mock_request): """CRITICAL: Must check status=='ok' - catches if success detection changes""" - from tasks.mediaserver_navidrome import delete_playlist + from tasks.mediaserver.navidrome import delete_playlist # Return response without 'ok' status mock_request.return_value = {'status': 'something_else'} @@ -1029,10 +1029,10 @@ def test_checks_status_ok_for_success(self, mock_request): # Should return False because status is not 'ok' assert result is False - @patch('tasks.mediaserver_navidrome._navidrome_request') + @patch('tasks.mediaserver.navidrome._navidrome_request') def test_returns_false_on_none_response(self, mock_request): """None response (API failure) returns False""" - from tasks.mediaserver_navidrome import delete_playlist + from tasks.mediaserver.navidrome import delete_playlist mock_request.return_value = None @@ -1044,10 +1044,10 @@ def test_returns_false_on_none_response(self, mock_request): class TestNavidromeGetPlaylistByName: """Test playlist lookup by name""" - @patch('tasks.mediaserver_navidrome._navidrome_request') + @patch('tasks.mediaserver.navidrome._navidrome_request') def test_finds_playlist_by_exact_name(self, mock_request): """Should find playlist with exact name match""" - from tasks.mediaserver_navidrome import get_playlist_by_name + from tasks.mediaserver.navidrome import get_playlist_by_name mock_request.return_value = { 'status': 'ok', @@ -1066,10 +1066,10 @@ def test_finds_playlist_by_exact_name(self, mock_request): assert result['id'] == 'pl2' assert result['name'] == 'Jazz Favorites' - @patch('tasks.mediaserver_navidrome._navidrome_request') + @patch('tasks.mediaserver.navidrome._navidrome_request') def test_returns_none_if_not_found(self, mock_request): """Returns None if no matching playlist""" - from tasks.mediaserver_navidrome import get_playlist_by_name + from tasks.mediaserver.navidrome import get_playlist_by_name mock_request.return_value = { 'status': 'ok', @@ -1088,10 +1088,10 @@ def test_returns_none_if_not_found(self, mock_request): class TestNavidromeCreatePlaylist: """Test playlist creation with batching""" - @patch('tasks.mediaserver_navidrome._navidrome_request') + @patch('tasks.mediaserver.navidrome._navidrome_request') def test_extracts_playlist_id_from_response(self, mock_request): """Should extract playlist ID from creation response""" - from tasks.mediaserver_navidrome import _create_playlist_batched + from tasks.mediaserver.navidrome import _create_playlist_batched mock_request.return_value = { 'status': 'ok', @@ -1109,10 +1109,10 @@ def test_extracts_playlist_id_from_response(self, mock_request): assert result['Id'] == 'new-pl-123' # Normalized key assert result['Name'] == 'Test Playlist' - @patch('tasks.mediaserver_navidrome._navidrome_request') + @patch('tasks.mediaserver.navidrome._navidrome_request') def test_create_playlist_sets_public_after_creation(self, mock_request): """Should call updatePlaylist(public=true) right after createPlaylist""" - from tasks.mediaserver_navidrome import _create_playlist_batched + from tasks.mediaserver.navidrome import _create_playlist_batched mock_request.return_value = { 'status': 'ok', @@ -1134,10 +1134,10 @@ def test_create_playlist_sets_public_after_creation(self, mock_request): assert update_params.get('playlistId') == 'new-pl-456' assert update_params.get('public') == 'true' - @patch('tasks.mediaserver_navidrome._navidrome_request') + @patch('tasks.mediaserver.navidrome._navidrome_request') def test_returns_none_on_creation_failure(self, mock_request): """Returns None when creation fails""" - from tasks.mediaserver_navidrome import _create_playlist_batched + from tasks.mediaserver.navidrome import _create_playlist_batched mock_request.return_value = None @@ -1145,10 +1145,10 @@ def test_returns_none_on_creation_failure(self, mock_request): assert result is None - @patch('tasks.mediaserver_navidrome._navidrome_request') + @patch('tasks.mediaserver.navidrome._navidrome_request') def test_handles_malformed_response(self, mock_request): """Returns None on malformed response""" - from tasks.mediaserver_navidrome import _create_playlist_batched + from tasks.mediaserver.navidrome import _create_playlist_batched mock_request.return_value = {'status': 'ok'} # Missing playlist key @@ -1160,10 +1160,10 @@ def test_handles_malformed_response(self, mock_request): class TestNavidromeGetLastPlayedTime: """Test last played time extraction""" - @patch('tasks.mediaserver_navidrome._navidrome_request') + @patch('tasks.mediaserver.navidrome._navidrome_request') def test_extracts_last_played(self, mock_request): """Should extract lastPlayed from song response""" - from tasks.mediaserver_navidrome import get_last_played_time + from tasks.mediaserver.navidrome import get_last_played_time mock_request.return_value = { 'status': 'ok', @@ -1178,10 +1178,10 @@ def test_extracts_last_played(self, mock_request): assert result == '2024-01-15T10:30:00Z' - @patch('tasks.mediaserver_navidrome._navidrome_request') + @patch('tasks.mediaserver.navidrome._navidrome_request') def test_returns_none_if_never_played(self, mock_request): """Returns None if no lastPlayed field""" - from tasks.mediaserver_navidrome import get_last_played_time + from tasks.mediaserver.navidrome import get_last_played_time mock_request.return_value = { 'status': 'ok', @@ -1199,11 +1199,11 @@ def test_returns_none_if_never_played(self, mock_request): class TestNavidromeGetRecentAlbums: """Test recent albums parsing""" - @patch('tasks.mediaserver_navidrome._get_target_music_folder_ids') - @patch('tasks.mediaserver_navidrome._navidrome_request') + @patch('tasks.mediaserver.navidrome._get_target_music_folder_ids') + @patch('tasks.mediaserver.navidrome._navidrome_request') def test_normalizes_album_keys(self, mock_request, mock_folders): """Albums should have Id and Name normalized""" - from tasks.mediaserver_navidrome import get_recent_albums + from tasks.mediaserver.navidrome import get_recent_albums mock_folders.return_value = None # No folder filtering mock_request.return_value = { @@ -1222,11 +1222,11 @@ def test_normalizes_album_keys(self, mock_request, mock_folders): assert albums[0]['Id'] == 'album1' assert albums[0]['Name'] == 'First Album' - @patch('tasks.mediaserver_navidrome._get_target_music_folder_ids') - @patch('tasks.mediaserver_navidrome._navidrome_request') + @patch('tasks.mediaserver.navidrome._get_target_music_folder_ids') + @patch('tasks.mediaserver.navidrome._navidrome_request') def test_returns_empty_when_no_matching_folders(self, mock_request, mock_folders): """Returns empty list when folder filter matches nothing""" - from tasks.mediaserver_navidrome import get_recent_albums + from tasks.mediaserver.navidrome import get_recent_albums mock_folders.return_value = set() # Empty set = no matches @@ -1252,11 +1252,11 @@ class TestNavidromeGetAllSongsApplyFilter: in code rather than implied from the presence of ``user_creds``. """ - @patch('tasks.mediaserver_navidrome._get_target_music_folder_ids') - @patch('tasks.mediaserver_navidrome._navidrome_request') + @patch('tasks.mediaserver.navidrome._get_target_music_folder_ids') + @patch('tasks.mediaserver.navidrome._navidrome_request') def test_apply_filter_false_skips_folder_lookup(self, mock_request, mock_filter): """apply_filter=False must NOT call _get_target_music_folder_ids.""" - from tasks.mediaserver_navidrome import get_all_songs + from tasks.mediaserver.navidrome import get_all_songs mock_request.return_value = { 'status': 'ok', @@ -1273,11 +1273,11 @@ def test_apply_filter_false_skips_folder_lookup(self, mock_request, mock_filter) for c in mock_request.call_args_list: assert c.kwargs.get('user_creds') == creds - @patch('tasks.mediaserver_navidrome._get_target_music_folder_ids') - @patch('tasks.mediaserver_navidrome._navidrome_request') + @patch('tasks.mediaserver.navidrome._get_target_music_folder_ids') + @patch('tasks.mediaserver.navidrome._navidrome_request') def test_apply_filter_true_default_honors_filter(self, mock_request, mock_filter): """apply_filter defaults to True, preserving live-provider semantics.""" - from tasks.mediaserver_navidrome import get_all_songs + from tasks.mediaserver.navidrome import get_all_songs mock_filter.return_value = set() # Filter active but no matches. mock_request.return_value = {'status': 'ok'} @@ -1288,15 +1288,15 @@ def test_apply_filter_true_default_honors_filter(self, mock_request, mock_filter assert songs == [] mock_request.assert_not_called() - @patch('tasks.mediaserver_navidrome._navidrome_request') + @patch('tasks.mediaserver.navidrome._navidrome_request') def test_get_target_music_folder_ids_forwards_user_creds(self, mock_request): """The folder-lookup helper must thread user_creds through to the API request so callers (e.g. live-provider code paths receiving session creds) hit ``getMusicFolders`` with valid auth instead of falling back to empty config globals.""" - from tasks.mediaserver_navidrome import _get_target_music_folder_ids + from tasks.mediaserver.navidrome import _get_target_music_folder_ids - with patch('tasks.mediaserver_navidrome.config') as mock_config: + with patch('tasks.mediaserver.navidrome.config') as mock_config: mock_config.MUSIC_LIBRARIES = 'Music' mock_request.return_value = { 'musicFolders': {'musicFolder': [{'id': 1, 'name': 'Music'}]} @@ -1357,8 +1357,8 @@ class TestDispatcherAutomaticPlaylistDeletion: """Test the filtering logic in delete_automatic_playlists""" @patch('tasks.mediaserver.config') - @patch('tasks.mediaserver.jellyfin_get_all_playlists') - @patch('tasks.mediaserver.jellyfin_delete_playlist') + @patch('tasks.mediaserver.jellyfin.get_all_playlists') + @patch('tasks.mediaserver.jellyfin.delete_playlist') def test_only_deletes_automatic_suffix_playlists(self, mock_delete, mock_get, mock_config): """Only playlists ending with '_automatic' should be deleted""" from tasks.mediaserver import delete_automatic_playlists @@ -1384,8 +1384,8 @@ def test_only_deletes_automatic_suffix_playlists(self, mock_delete, mock_get, mo assert '3' not in deleted_ids @patch('tasks.mediaserver.config') - @patch('tasks.mediaserver.navidrome_get_all_playlists') - @patch('tasks.mediaserver.navidrome_delete_playlist') + @patch('tasks.mediaserver.navidrome.get_all_playlists') + @patch('tasks.mediaserver.navidrome.delete_playlist') def test_handles_both_id_and_Id_keys(self, mock_delete, mock_get, mock_config): """Should handle both 'id' and 'Id' keys (Navidrome uses lowercase)""" from tasks.mediaserver import delete_automatic_playlists @@ -1417,7 +1417,7 @@ def test_artist_priority_order(self): # The logic is inline in get_tracks_from_album, so we test expected behavior # by examining the field priority order from the code - # Priority fields in order (from examining mediaserver_lyrion.py): + # Priority fields in order (from examining tasks/mediaserver/lyrion.py): priority_fields = ['trackartist', 'contributor', 'artist', 'albumartist', 'band'] # This verifies our understanding of the priority @@ -1428,11 +1428,11 @@ def test_artist_priority_order(self): class TestLyrionJsonRpcRequest: """Test the core JSON-RPC request helper""" - @patch('tasks.mediaserver_lyrion.requests.Session') - @patch('tasks.mediaserver_lyrion.config') + @patch('tasks.mediaserver.lyrion.requests.Session') + @patch('tasks.mediaserver.lyrion.config') def test_constructs_correct_url(self, mock_config, mock_session_class): """CRITICAL: URL must be /jsonrpc.js - catches if endpoint changes""" - from tasks.mediaserver_lyrion import _jsonrpc_request + from tasks.mediaserver.lyrion import _jsonrpc_request mock_config.LYRION_URL = 'http://lyrion:9000' @@ -1454,11 +1454,11 @@ def test_constructs_correct_url(self, mock_config, mock_session_class): assert call_args[0][0] == 'http://lyrion:9000/jsonrpc.js', \ f"URL changed! Expected '/jsonrpc.js', got '{call_args[0][0]}'" - @patch('tasks.mediaserver_lyrion.requests.Session') - @patch('tasks.mediaserver_lyrion.config') + @patch('tasks.mediaserver.lyrion.requests.Session') + @patch('tasks.mediaserver.lyrion.config') def test_uses_slim_request_method(self, mock_config, mock_session_class): """CRITICAL: Must use 'slim.request' method - catches if protocol changes""" - from tasks.mediaserver_lyrion import _jsonrpc_request + from tasks.mediaserver.lyrion import _jsonrpc_request mock_config.LYRION_URL = 'http://lyrion:9000' @@ -1486,11 +1486,11 @@ def test_uses_slim_request_method(self, mock_config, mock_session_class): assert payload.get('params')[1][0] == 'albums', \ "Command not passed correctly" - @patch('tasks.mediaserver_lyrion.requests.Session') - @patch('tasks.mediaserver_lyrion.config') + @patch('tasks.mediaserver.lyrion.requests.Session') + @patch('tasks.mediaserver.lyrion.config') def test_extracts_result_field(self, mock_config, mock_session_class): """CRITICAL: Must return 'result' field - catches if response parsing changes""" - from tasks.mediaserver_lyrion import _jsonrpc_request + from tasks.mediaserver.lyrion import _jsonrpc_request mock_config.LYRION_URL = 'http://lyrion:9000' @@ -1515,11 +1515,11 @@ def test_extracts_result_field(self, mock_config, mock_session_class): assert 'albums_loop' in result assert 'id' not in result # Should not include top-level id - @patch('tasks.mediaserver_lyrion.requests.Session') - @patch('tasks.mediaserver_lyrion.config') + @patch('tasks.mediaserver.lyrion.requests.Session') + @patch('tasks.mediaserver.lyrion.config') def test_raises_on_jsonrpc_error(self, mock_config, mock_session_class): """CRITICAL: Must raise LyrionAPIError on error response""" - from tasks.mediaserver_lyrion import _jsonrpc_request, LyrionAPIError + from tasks.mediaserver.lyrion import _jsonrpc_request, LyrionAPIError mock_config.LYRION_URL = 'http://lyrion:9000' @@ -1543,10 +1543,10 @@ def test_raises_on_jsonrpc_error(self, mock_config, mock_session_class): class TestLyrionGetAllPlaylists: """Test playlist fetching and normalization""" - @patch('tasks.mediaserver_lyrion._jsonrpc_request') + @patch('tasks.mediaserver.lyrion._jsonrpc_request') def test_calls_playlists_command(self, mock_request): """CRITICAL: Must call 'playlists' command - catches if API changes""" - from tasks.mediaserver_lyrion import get_all_playlists + from tasks.mediaserver.lyrion import get_all_playlists mock_request.return_value = {'playlists_loop': []} @@ -1556,10 +1556,10 @@ def test_calls_playlists_command(self, mock_request): assert call_args[0][0] == 'playlists', \ f"Command changed! Expected 'playlists', got '{call_args[0][0]}'" - @patch('tasks.mediaserver_lyrion._jsonrpc_request') + @patch('tasks.mediaserver.lyrion._jsonrpc_request') def test_normalizes_playlist_keys(self, mock_request): """CRITICAL: Must normalize 'id'->Id, 'playlist'->Name""" - from tasks.mediaserver_lyrion import get_all_playlists + from tasks.mediaserver.lyrion import get_all_playlists mock_request.return_value = { 'playlists_loop': [ @@ -1575,10 +1575,10 @@ def test_normalizes_playlist_keys(self, mock_request): assert playlists[0]['Id'] == 'pl1', "Missing 'Id' normalization" assert playlists[0]['Name'] == 'Rock_automatic', "Missing 'Name' normalization from 'playlist' field" - @patch('tasks.mediaserver_lyrion._jsonrpc_request') + @patch('tasks.mediaserver.lyrion._jsonrpc_request') def test_returns_empty_on_no_playlists(self, mock_request): """Returns empty list when no playlists_loop""" - from tasks.mediaserver_lyrion import get_all_playlists + from tasks.mediaserver.lyrion import get_all_playlists mock_request.return_value = {} @@ -1590,10 +1590,10 @@ def test_returns_empty_on_no_playlists(self, mock_request): class TestLyrionDeletePlaylist: """Test playlist deletion""" - @patch('tasks.mediaserver_lyrion._jsonrpc_request') + @patch('tasks.mediaserver.lyrion._jsonrpc_request') def test_calls_playlists_delete_command(self, mock_request): """CRITICAL: Must use 'playlists' with 'delete' param - catches if API changes""" - from tasks.mediaserver_lyrion import delete_playlist + from tasks.mediaserver.lyrion import delete_playlist mock_request.return_value = {'count': 1} @@ -1608,10 +1608,10 @@ def test_calls_playlists_delete_command(self, mock_request): assert 'delete' in params, "Must include 'delete' param" assert 'playlist_id:playlist-123' in params, "Must include playlist_id param" - @patch('tasks.mediaserver_lyrion._jsonrpc_request') + @patch('tasks.mediaserver.lyrion._jsonrpc_request') def test_returns_true_on_success(self, mock_request): """Returns True when deletion succeeds""" - from tasks.mediaserver_lyrion import delete_playlist + from tasks.mediaserver.lyrion import delete_playlist mock_request.return_value = {'count': 1} @@ -1619,10 +1619,10 @@ def test_returns_true_on_success(self, mock_request): assert result is True - @patch('tasks.mediaserver_lyrion._jsonrpc_request') + @patch('tasks.mediaserver.lyrion._jsonrpc_request') def test_returns_false_on_failure(self, mock_request): """Returns False when deletion fails""" - from tasks.mediaserver_lyrion import delete_playlist + from tasks.mediaserver.lyrion import delete_playlist mock_request.return_value = None @@ -1634,10 +1634,10 @@ def test_returns_false_on_failure(self, mock_request): class TestLyrionGetTracksFromAlbum: """Test track fetching from albums""" - @patch('tasks.mediaserver_lyrion._jsonrpc_request') + @patch('tasks.mediaserver.lyrion._jsonrpc_request') def test_calls_titles_command_with_album_id(self, mock_request): """CRITICAL: Must use 'titles' with album_id filter""" - from tasks.mediaserver_lyrion import get_tracks_from_album + from tasks.mediaserver.lyrion import get_tracks_from_album mock_request.return_value = {'titles_loop': []} @@ -1650,10 +1650,10 @@ def test_calls_titles_command_with_album_id(self, mock_request): assert any('album_id:album-123' in str(p) for p in params), \ "Must include album_id filter" - @patch('tasks.mediaserver_lyrion._jsonrpc_request') + @patch('tasks.mediaserver.lyrion._jsonrpc_request') def test_normalizes_track_fields(self, mock_request): """CRITICAL: Must normalize id->Id, title->Name, add AlbumArtist""" - from tasks.mediaserver_lyrion import get_tracks_from_album + from tasks.mediaserver.lyrion import get_tracks_from_album mock_request.return_value = { 'titles_loop': [ @@ -1676,10 +1676,10 @@ def test_normalizes_track_fields(self, mock_request): "trackartist should be prioritized for AlbumArtist" assert tracks[0]['Path'] == '/music/song1.mp3', "Missing 'Path' from 'url'" - @patch('tasks.mediaserver_lyrion._jsonrpc_request') + @patch('tasks.mediaserver.lyrion._jsonrpc_request') def test_artist_fallback_priority(self, mock_request): """Tests artist field fallback: trackartist > contributor > artist > albumartist""" - from tasks.mediaserver_lyrion import get_tracks_from_album + from tasks.mediaserver.lyrion import get_tracks_from_album mock_request.return_value = { 'titles_loop': [ @@ -1708,10 +1708,10 @@ def test_artist_fallback_priority(self, mock_request): assert tracks[1]['AlbumArtist'] == 'Album Artist Only', \ "Should fall back to albumartist when no higher priority fields" - @patch('tasks.mediaserver_lyrion._jsonrpc_request') + @patch('tasks.mediaserver.lyrion._jsonrpc_request') def test_filters_spotify_tracks(self, mock_request): """CRITICAL: Spotify tracks should be filtered out""" - from tasks.mediaserver_lyrion import get_tracks_from_album + from tasks.mediaserver.lyrion import get_tracks_from_album mock_request.return_value = { 'titles_loop': [ @@ -1740,7 +1740,7 @@ class TestEmbySelectBestArtist: def test_prioritizes_artist_items_over_album_artist(self): """ArtistItems should be preferred over AlbumArtist""" - from tasks.mediaserver_emby import _select_best_artist + from tasks.mediaserver.emby import _select_best_artist item = { 'ArtistItems': [{'Name': 'Track Artist', 'Id': 'artist-123'}], @@ -1755,7 +1755,7 @@ def test_prioritizes_artist_items_over_album_artist(self): def test_falls_back_to_artists_array(self): """If no ArtistItems, use Artists array""" - from tasks.mediaserver_emby import _select_best_artist + from tasks.mediaserver.emby import _select_best_artist item = { 'ArtistItems': [], @@ -1770,7 +1770,7 @@ def test_falls_back_to_artists_array(self): def test_falls_back_to_album_artist(self): """If no Artists, use AlbumArtist""" - from tasks.mediaserver_emby import _select_best_artist + from tasks.mediaserver.emby import _select_best_artist item = { 'AlbumArtist': 'The Album Artist' @@ -1783,7 +1783,7 @@ def test_falls_back_to_album_artist(self): def test_returns_unknown_when_no_artist_info(self): """Returns 'Unknown Artist' when no artist info available""" - from tasks.mediaserver_emby import _select_best_artist + from tasks.mediaserver.emby import _select_best_artist item = {} @@ -1796,11 +1796,11 @@ def test_returns_unknown_when_no_artist_info(self): class TestEmbyGetAllPlaylists: """Test playlist fetching - verifies URL and response parsing""" - @patch('tasks.mediaserver_emby.requests.get') - @patch('tasks.mediaserver_emby.config') + @patch('tasks.mediaserver.emby.requests.get') + @patch('tasks.mediaserver.emby.config') def test_uses_correct_url_with_emby_prefix(self, mock_config, mock_get): """CRITICAL: URL must include /emby/ prefix - catches if path changes""" - from tasks.mediaserver_emby import get_all_playlists + from tasks.mediaserver.emby import get_all_playlists mock_config.EMBY_URL = 'http://emby:8096' mock_config.EMBY_USER_ID = 'user123' @@ -1818,11 +1818,11 @@ def test_uses_correct_url_with_emby_prefix(self, mock_config, mock_get): assert call_url == 'http://emby:8096/emby/Users/user123/Items', \ f"URL changed! Got '{call_url}'" - @patch('tasks.mediaserver_emby.requests.get') - @patch('tasks.mediaserver_emby.config') + @patch('tasks.mediaserver.emby.requests.get') + @patch('tasks.mediaserver.emby.config') def test_includes_playlist_item_type(self, mock_config, mock_get): """CRITICAL: Must filter by IncludeItemTypes=Playlist""" - from tasks.mediaserver_emby import get_all_playlists + from tasks.mediaserver.emby import get_all_playlists mock_config.EMBY_URL = 'http://emby:8096' mock_config.EMBY_USER_ID = 'user123' @@ -1839,11 +1839,11 @@ def test_includes_playlist_item_type(self, mock_config, mock_get): assert call_params.get('IncludeItemTypes') == 'Playlist', \ "Must filter by Playlist item type" - @patch('tasks.mediaserver_emby.requests.get') - @patch('tasks.mediaserver_emby.config') + @patch('tasks.mediaserver.emby.requests.get') + @patch('tasks.mediaserver.emby.config') def test_parses_items_array(self, mock_config, mock_get): """CRITICAL: Must extract Items[] from response""" - from tasks.mediaserver_emby import get_all_playlists + from tasks.mediaserver.emby import get_all_playlists mock_config.EMBY_URL = 'http://emby:8096' mock_config.EMBY_USER_ID = 'user123' @@ -1869,11 +1869,11 @@ def test_parses_items_array(self, mock_config, mock_get): class TestEmbyDeletePlaylist: """Test playlist deletion - Emby uses different endpoint than Jellyfin!""" - @patch('tasks.mediaserver_emby.requests.post') - @patch('tasks.mediaserver_emby.config') + @patch('tasks.mediaserver.emby.requests.post') + @patch('tasks.mediaserver.emby.config') def test_uses_items_delete_endpoint(self, mock_config, mock_post): """CRITICAL: Emby uses /Items/Delete with POST, not DELETE to /Items/{id}""" - from tasks.mediaserver_emby import delete_playlist + from tasks.mediaserver.emby import delete_playlist mock_config.EMBY_URL = 'http://emby:8096' mock_config.HEADERS = {'X-Emby-Token': 'token'} @@ -1890,11 +1890,11 @@ def test_uses_items_delete_endpoint(self, mock_config, mock_post): assert call_url == 'http://emby:8096/emby/Items/Delete', \ f"Emby deletion URL changed! Expected '/emby/Items/Delete', got '{call_url}'" - @patch('tasks.mediaserver_emby.requests.post') - @patch('tasks.mediaserver_emby.config') + @patch('tasks.mediaserver.emby.requests.post') + @patch('tasks.mediaserver.emby.config') def test_passes_id_as_query_param(self, mock_config, mock_post): """CRITICAL: Playlist ID must be in 'Ids' query param""" - from tasks.mediaserver_emby import delete_playlist + from tasks.mediaserver.emby import delete_playlist mock_config.EMBY_URL = 'http://emby:8096' mock_config.HEADERS = {} @@ -1909,11 +1909,11 @@ def test_passes_id_as_query_param(self, mock_config, mock_post): assert call_params.get('Ids') == 'playlist-xyz', \ "Playlist ID must be passed as 'Ids' query param" - @patch('tasks.mediaserver_emby.requests.post') - @patch('tasks.mediaserver_emby.config') + @patch('tasks.mediaserver.emby.requests.post') + @patch('tasks.mediaserver.emby.config') def test_returns_false_on_error(self, mock_config, mock_post): """HTTP error returns False""" - from tasks.mediaserver_emby import delete_playlist + from tasks.mediaserver.emby import delete_playlist mock_config.EMBY_URL = 'http://emby:8096' mock_config.HEADERS = {} @@ -1927,11 +1927,11 @@ def test_returns_false_on_error(self, mock_config, mock_post): class TestEmbyGetTracksFromAlbum: """Test track fetching with artist enrichment""" - @patch('tasks.mediaserver_emby.requests.get') - @patch('tasks.mediaserver_emby.config') + @patch('tasks.mediaserver.emby.requests.get') + @patch('tasks.mediaserver.emby.config') def test_uses_emby_url_prefix(self, mock_config, mock_get): """CRITICAL: URL must include /emby/ prefix""" - from tasks.mediaserver_emby import get_tracks_from_album + from tasks.mediaserver.emby import get_tracks_from_album mock_config.EMBY_URL = 'http://emby:8096' mock_config.EMBY_USER_ID = 'user123' @@ -1947,11 +1947,11 @@ def test_uses_emby_url_prefix(self, mock_config, mock_get): call_url = mock_get.call_args[0][0] assert '/emby/' in call_url, "URL must include /emby/ prefix" - @patch('tasks.mediaserver_emby.requests.get') - @patch('tasks.mediaserver_emby.config') + @patch('tasks.mediaserver.emby.requests.get') + @patch('tasks.mediaserver.emby.config') def test_enriches_tracks_with_artist(self, mock_config, mock_get): """CRITICAL: Must add AlbumArtist and ArtistId fields""" - from tasks.mediaserver_emby import get_tracks_from_album + from tasks.mediaserver.emby import get_tracks_from_album mock_config.EMBY_URL = 'http://emby:8096' mock_config.EMBY_USER_ID = 'user123' @@ -1978,11 +1978,11 @@ def test_enriches_tracks_with_artist(self, mock_config, mock_get): assert tracks[0]['AlbumArtist'] == 'Artist A' assert tracks[0]['ArtistId'] == 'artist-a' - @patch('tasks.mediaserver_emby.requests.get') - @patch('tasks.mediaserver_emby.config') + @patch('tasks.mediaserver.emby.requests.get') + @patch('tasks.mediaserver.emby.config') def test_handles_standalone_track_pseudo_albums(self, mock_config, mock_get): """CRITICAL: Must handle standalone_ prefix for pseudo-albums""" - from tasks.mediaserver_emby import get_tracks_from_album + from tasks.mediaserver.emby import get_tracks_from_album mock_config.EMBY_URL = 'http://emby:8096' mock_config.EMBY_USER_ID = 'user123' @@ -2009,11 +2009,11 @@ def test_handles_standalone_track_pseudo_albums(self, mock_config, mock_get): class TestEmbyCreatePlaylist: """Test playlist creation - Emby uses query params, not JSON body!""" - @patch('tasks.mediaserver_emby.requests.post') - @patch('tasks.mediaserver_emby.config') + @patch('tasks.mediaserver.emby.requests.post') + @patch('tasks.mediaserver.emby.config') def test_uses_query_params_not_json_body(self, mock_config, mock_post): """CRITICAL: Emby expects query params, not JSON body""" - from tasks.mediaserver_emby import create_playlist + from tasks.mediaserver.emby import create_playlist mock_config.EMBY_URL = 'http://emby:8096' mock_config.EMBY_USER_ID = 'user123' @@ -2037,11 +2037,11 @@ def test_uses_query_params_not_json_body(self, mock_config, mock_post): call_kwargs = mock_post.call_args[1] assert 'json' not in call_kwargs, "Emby should NOT receive JSON body" - @patch('tasks.mediaserver_emby.requests.post') - @patch('tasks.mediaserver_emby.config') + @patch('tasks.mediaserver.emby.requests.post') + @patch('tasks.mediaserver.emby.config') def test_url_encodes_playlist_name(self, mock_config, mock_post): """Playlist names with special chars must be URL encoded""" - from tasks.mediaserver_emby import create_playlist + from tasks.mediaserver.emby import create_playlist mock_config.EMBY_URL = 'http://emby:8096' mock_config.EMBY_USER_ID = 'user123' @@ -2070,10 +2070,10 @@ def test_url_encodes_playlist_name(self, mock_config, mock_post): # ============================================================================= class TestJellyfinListLibraries: - @patch('tasks.mediaserver_jellyfin.requests.get') - @patch('tasks.mediaserver_jellyfin.config') + @patch('tasks.mediaserver.jellyfin.requests.get') + @patch('tasks.mediaserver.jellyfin.config') def test_returns_music_libraries_with_id_and_name(self, mock_config, mock_get): - from tasks.mediaserver_jellyfin import list_libraries + from tasks.mediaserver.jellyfin import list_libraries mock_config.JELLYFIN_URL = 'http://jelly:8096' mock_config.JELLYFIN_TOKEN = 'admin-token' @@ -2095,11 +2095,11 @@ def test_returns_music_libraries_with_id_and_name(self, mock_config, mock_get): {'id': 'lib-3', 'name': 'Podcasts'}, ] - @patch('tasks.mediaserver_jellyfin.requests.get') - @patch('tasks.mediaserver_jellyfin.config') + @patch('tasks.mediaserver.jellyfin.requests.get') + @patch('tasks.mediaserver.jellyfin.config') def test_forwards_user_creds_to_url_and_token(self, mock_config, mock_get): """Migration target probe must use session creds, not config globals.""" - from tasks.mediaserver_jellyfin import list_libraries + from tasks.mediaserver.jellyfin import list_libraries mock_config.JELLYFIN_URL = 'http://SHOULD-NOT-BE-USED:0000' mock_config.JELLYFIN_TOKEN = 'SHOULD-NOT-BE-USED' @@ -2122,10 +2122,10 @@ def test_forwards_user_creds_to_url_and_token(self, mock_config, mock_get): class TestEmbyListLibraries: - @patch('tasks.mediaserver_emby.requests.get') - @patch('tasks.mediaserver_emby.config') + @patch('tasks.mediaserver.emby.requests.get') + @patch('tasks.mediaserver.emby.config') def test_returns_music_libraries_only(self, mock_config, mock_get): - from tasks.mediaserver_emby import list_libraries + from tasks.mediaserver.emby import list_libraries mock_config.EMBY_URL = 'http://emby:8096' mock_config.EMBY_TOKEN = 'admin-token' @@ -2143,10 +2143,10 @@ def test_returns_music_libraries_only(self, mock_config, mock_get): assert result == [{'id': 'e1', 'name': 'Music'}] - @patch('tasks.mediaserver_emby.requests.get') - @patch('tasks.mediaserver_emby.config') + @patch('tasks.mediaserver.emby.requests.get') + @patch('tasks.mediaserver.emby.config') def test_forwards_user_creds(self, mock_config, mock_get): - from tasks.mediaserver_emby import list_libraries + from tasks.mediaserver.emby import list_libraries mock_config.EMBY_URL = 'http://SHOULD-NOT-BE-USED:0000' mock_config.EMBY_TOKEN = 'SHOULD-NOT-BE-USED' @@ -2169,14 +2169,14 @@ def test_forwards_user_creds(self, mock_config, mock_get): class TestNavidromeListLibraries: - @patch('tasks.mediaserver_navidrome._navidrome_request') + @patch('tasks.mediaserver.navidrome._navidrome_request') def test_returns_every_folder_without_reading_music_libraries(self, mock_req): """ Does NOT call _get_target_music_folder_ids (which would read config.MUSIC_LIBRARIES and break when the filter is set for the source provider but doesn't apply to the target). Returns every folder. """ - from tasks.mediaserver_navidrome import list_libraries + from tasks.mediaserver.navidrome import list_libraries mock_req.return_value = { 'musicFolders': { @@ -2199,11 +2199,11 @@ def test_returns_every_folder_without_reading_music_libraries(self, mock_req): assert args[0] == 'getMusicFolders' assert kwargs.get('user_creds') is None - @patch('tasks.mediaserver_navidrome._navidrome_request') + @patch('tasks.mediaserver.navidrome._navidrome_request') def test_handles_single_dict_response(self, mock_req): """Some Subsonic-compatible servers return a single dict (not a list) when only one folder exists. The function must coerce to a list.""" - from tasks.mediaserver_navidrome import list_libraries + from tasks.mediaserver.navidrome import list_libraries mock_req.return_value = { 'musicFolders': { @@ -2215,14 +2215,14 @@ def test_handles_single_dict_response(self, mock_req): assert result == [{'id': '1', 'name': 'OnlyFolder'}] - @patch('tasks.mediaserver_navidrome._navidrome_request') + @patch('tasks.mediaserver.navidrome._navidrome_request') def test_forwards_user_creds_to_getmusicfolders(self, mock_req): """ Migration-target path: user_creds must reach _navidrome_request so the request uses the session's URL/username/password rather than config.NAVIDROME_* (which are empty for a target that isn't live yet). """ - from tasks.mediaserver_navidrome import list_libraries + from tasks.mediaserver.navidrome import list_libraries mock_req.return_value = {'musicFolders': {'musicFolder': []}} @@ -2235,9 +2235,9 @@ def test_forwards_user_creds_to_getmusicfolders(self, mock_req): class TestLyrionListLibraries: - @patch('tasks.mediaserver_lyrion._jsonrpc_request') + @patch('tasks.mediaserver.lyrion._jsonrpc_request') def test_returns_every_folder(self, mock_rpc): - from tasks.mediaserver_lyrion import list_libraries + from tasks.mediaserver.lyrion import list_libraries mock_rpc.return_value = { 'folder_loop': [ @@ -2258,11 +2258,11 @@ def test_returns_every_folder(self, mock_rpc): assert args[0] == 'musicfolder' assert kwargs.get('user_creds') is None - @patch('tasks.mediaserver_lyrion._jsonrpc_request') + @patch('tasks.mediaserver.lyrion._jsonrpc_request') def test_handles_lyrion_9_x_filename_field(self, mock_rpc): """Lyrion 9.0.x returns folder entries with ``filename`` (not ``name``). Older versions used ``name`` / ``folder``. Accept all three.""" - from tasks.mediaserver_lyrion import list_libraries + from tasks.mediaserver.lyrion import list_libraries mock_rpc.return_value = { 'folder_loop': [ @@ -2279,14 +2279,14 @@ def test_handles_lyrion_9_x_filename_field(self, mock_rpc): {'id': '686', 'name': 'Library_B'}, ] - @patch('tasks.mediaserver_lyrion._jsonrpc_request') + @patch('tasks.mediaserver.lyrion._jsonrpc_request') def test_prefers_path_over_name_when_available(self, mock_rpc): """Lyrion's scan-time filter (_get_target_paths_for_filtering) does a substring match against album file URLs, so when the server reports a real path we persist it (more deterministic match). Otherwise we fall back to the folder display name (which Lyrion treats as a path substring at scan time on standard layouts).""" - from tasks.mediaserver_lyrion import list_libraries + from tasks.mediaserver.lyrion import list_libraries mock_rpc.return_value = { 'folder_loop': [ @@ -2304,9 +2304,9 @@ def test_prefers_path_over_name_when_available(self, mock_rpc): {'id': '12', 'name': 'NoPath'}, # falls back when path absent ] - @patch('tasks.mediaserver_lyrion._jsonrpc_request') + @patch('tasks.mediaserver.lyrion._jsonrpc_request') def test_forwards_user_creds(self, mock_rpc): - from tasks.mediaserver_lyrion import list_libraries + from tasks.mediaserver.lyrion import list_libraries mock_rpc.return_value = {'folder_loop': []} @@ -2325,10 +2325,10 @@ def test_forwards_user_creds(self, mock_rpc): class TestNavidromeCreateOrReplacePlaylist: """Navidrome upsert: create when missing, clear+add when existing (preserve ID).""" - @patch('tasks.mediaserver_navidrome._create_playlist_batched') - @patch('tasks.mediaserver_navidrome.get_playlist_by_name') + @patch('tasks.mediaserver.navidrome._create_playlist_batched') + @patch('tasks.mediaserver.navidrome.get_playlist_by_name') def test_missing_playlist_creates_via_batched(self, mock_get, mock_create): - from tasks.mediaserver_navidrome import create_or_replace_playlist + from tasks.mediaserver.navidrome import create_or_replace_playlist mock_get.return_value = None mock_create.return_value = {'Id': 'new-pl-1', 'Name': 'SF', 'id': 'new-pl-1'} @@ -2338,11 +2338,11 @@ def test_missing_playlist_creates_via_batched(self, mock_get, mock_create): mock_create.assert_called_once_with('SF', ['s1', 's2'], user_creds=None) assert result['Id'] == 'new-pl-1' - @patch('tasks.mediaserver_navidrome._add_to_playlist') - @patch('tasks.mediaserver_navidrome._navidrome_request') - @patch('tasks.mediaserver_navidrome.get_playlist_by_name') + @patch('tasks.mediaserver.navidrome._add_to_playlist') + @patch('tasks.mediaserver.navidrome._navidrome_request') + @patch('tasks.mediaserver.navidrome.get_playlist_by_name') def test_existing_playlist_preserves_id(self, mock_get, mock_request, mock_add): - from tasks.mediaserver_navidrome import create_or_replace_playlist + from tasks.mediaserver.navidrome import create_or_replace_playlist mock_get.return_value = {'id': 'pl-existing', 'name': 'SF'} # 1st call: getPlaylist returns 3 songs. 2nd+: updatePlaylist returns ok. @@ -2371,11 +2371,11 @@ def test_existing_playlist_preserves_id(self, mock_get, mock_request, mock_add): # Returned dict carries the existing ID assert result['Id'] == 'pl-existing' - @patch('tasks.mediaserver_navidrome._add_to_playlist') - @patch('tasks.mediaserver_navidrome._navidrome_request') - @patch('tasks.mediaserver_navidrome.get_playlist_by_name') + @patch('tasks.mediaserver.navidrome._add_to_playlist') + @patch('tasks.mediaserver.navidrome._navidrome_request') + @patch('tasks.mediaserver.navidrome.get_playlist_by_name') def test_clear_batches_above_40(self, mock_get, mock_request, mock_add): - from tasks.mediaserver_navidrome import create_or_replace_playlist + from tasks.mediaserver.navidrome import create_or_replace_playlist mock_get.return_value = {'id': 'pl-100', 'name': 'SF'} mock_request.side_effect = [ @@ -2393,10 +2393,10 @@ def test_clear_batches_above_40(self, mock_get, mock_request, mock_add): assert len(update_calls[1][0][1]['songIndexToRemove']) == 40 assert len(update_calls[2][0][1]['songIndexToRemove']) == 20 - @patch('tasks.mediaserver_navidrome._navidrome_request') - @patch('tasks.mediaserver_navidrome.get_playlist_by_name') + @patch('tasks.mediaserver.navidrome._navidrome_request') + @patch('tasks.mediaserver.navidrome.get_playlist_by_name') def test_empty_item_ids_returns_none_without_calls(self, mock_get, mock_request): - from tasks.mediaserver_navidrome import create_or_replace_playlist + from tasks.mediaserver.navidrome import create_or_replace_playlist result = create_or_replace_playlist('SF', []) @@ -2404,12 +2404,12 @@ def test_empty_item_ids_returns_none_without_calls(self, mock_get, mock_request) mock_get.assert_not_called() mock_request.assert_not_called() - @patch('tasks.mediaserver_navidrome._add_to_playlist') - @patch('tasks.mediaserver_navidrome._navidrome_request') - @patch('tasks.mediaserver_navidrome.get_playlist_by_name') + @patch('tasks.mediaserver.navidrome._add_to_playlist') + @patch('tasks.mediaserver.navidrome._navidrome_request') + @patch('tasks.mediaserver.navidrome.get_playlist_by_name') def test_returns_none_when_add_fails_after_clear(self, mock_get, mock_request, mock_add): """If clear succeeds but add fails, return None so the cron handler doesn't log success.""" - from tasks.mediaserver_navidrome import create_or_replace_playlist + from tasks.mediaserver.navidrome import create_or_replace_playlist mock_get.return_value = {'id': 'pl-1', 'name': 'SF'} mock_request.side_effect = [ @@ -2426,11 +2426,11 @@ def test_returns_none_when_add_fails_after_clear(self, mock_get, mock_request, m class TestJellyfinCreateOrReplacePlaylist: """Jellyfin upsert via /Playlists/{Id}/Items POST/DELETE/GET.""" - @patch('tasks.mediaserver_jellyfin.requests') - @patch('tasks.mediaserver_jellyfin.get_playlist_by_name') - @patch('tasks.mediaserver_jellyfin.config') + @patch('tasks.mediaserver.jellyfin.requests') + @patch('tasks.mediaserver.jellyfin.get_playlist_by_name') + @patch('tasks.mediaserver.jellyfin.config') def test_missing_playlist_creates_and_returns_id(self, mock_config, mock_get, mock_requests): - from tasks.mediaserver_jellyfin import create_or_replace_playlist + from tasks.mediaserver.jellyfin import create_or_replace_playlist mock_config.JELLYFIN_URL = 'http://jf' mock_config.JELLYFIN_USER_ID = 'admin-user' @@ -2450,11 +2450,11 @@ def test_missing_playlist_creates_and_returns_id(self, mock_config, mock_get, mo assert post_call[1]['json'] == {'Name': 'SF', 'Ids': ['s1', 's2'], 'UserId': 'admin-user'} assert result['Id'] == 'new-jf-1' - @patch('tasks.mediaserver_jellyfin.requests') - @patch('tasks.mediaserver_jellyfin.get_playlist_by_name') - @patch('tasks.mediaserver_jellyfin.config') + @patch('tasks.mediaserver.jellyfin.requests') + @patch('tasks.mediaserver.jellyfin.get_playlist_by_name') + @patch('tasks.mediaserver.jellyfin.config') def test_existing_playlist_clears_and_adds_preserving_id(self, mock_config, mock_get, mock_requests): - from tasks.mediaserver_jellyfin import create_or_replace_playlist + from tasks.mediaserver.jellyfin import create_or_replace_playlist mock_config.JELLYFIN_URL = 'http://jf' mock_config.JELLYFIN_USER_ID = 'admin-user' @@ -2489,25 +2489,25 @@ def test_existing_playlist_clears_and_adds_preserving_id(self, mock_config, mock # Same ID preserved assert result['Id'] == 'pl-existing' - @patch('tasks.mediaserver_jellyfin.get_playlist_by_name') + @patch('tasks.mediaserver.jellyfin.get_playlist_by_name') def test_empty_item_ids_returns_none(self, mock_get): - from tasks.mediaserver_jellyfin import create_or_replace_playlist + from tasks.mediaserver.jellyfin import create_or_replace_playlist result = create_or_replace_playlist('SF', []) assert result is None mock_get.assert_not_called() - @patch('tasks.mediaserver_jellyfin._add_items_to_playlist') - @patch('tasks.mediaserver_jellyfin._remove_playlist_entries') - @patch('tasks.mediaserver_jellyfin._get_playlist_entry_ids') - @patch('tasks.mediaserver_jellyfin.get_playlist_by_name') - @patch('tasks.mediaserver_jellyfin.config') + @patch('tasks.mediaserver.jellyfin._add_items_to_playlist') + @patch('tasks.mediaserver.jellyfin._remove_playlist_entries') + @patch('tasks.mediaserver.jellyfin._get_playlist_entry_ids') + @patch('tasks.mediaserver.jellyfin.get_playlist_by_name') + @patch('tasks.mediaserver.jellyfin.config') def test_returns_none_when_add_fails_after_clear( self, mock_config, mock_get, mock_get_entries, mock_remove, mock_add ): """If clear succeeds but add fails, return None instead of misreporting success.""" - from tasks.mediaserver_jellyfin import create_or_replace_playlist + from tasks.mediaserver.jellyfin import create_or_replace_playlist mock_config.JELLYFIN_URL = 'http://jf' mock_config.JELLYFIN_USER_ID = 'admin-user' @@ -2521,18 +2521,18 @@ def test_returns_none_when_add_fails_after_clear( assert result is None - @patch('tasks.mediaserver_jellyfin._create_fresh_playlist') - @patch('tasks.mediaserver_jellyfin.delete_playlist') - @patch('tasks.mediaserver_jellyfin._remove_playlist_entries') - @patch('tasks.mediaserver_jellyfin._get_playlist_entry_ids') - @patch('tasks.mediaserver_jellyfin.get_playlist_by_name') - @patch('tasks.mediaserver_jellyfin.config') + @patch('tasks.mediaserver.jellyfin._create_fresh_playlist') + @patch('tasks.mediaserver.jellyfin.delete_playlist') + @patch('tasks.mediaserver.jellyfin._remove_playlist_entries') + @patch('tasks.mediaserver.jellyfin._get_playlist_entry_ids') + @patch('tasks.mediaserver.jellyfin.get_playlist_by_name') + @patch('tasks.mediaserver.jellyfin.config') def test_falls_back_to_recreate_when_remove_fails( self, mock_config, mock_get, mock_get_entries, mock_remove, mock_delete, mock_create ): """Jellyfin <10.11 + API token rejects DELETE /Playlists/{Id}/Items (jellyfin#13476). When that happens, delete the whole playlist and recreate. Id will change.""" - from tasks.mediaserver_jellyfin import create_or_replace_playlist + from tasks.mediaserver.jellyfin import create_or_replace_playlist mock_config.JELLYFIN_URL = 'http://jf' mock_config.JELLYFIN_USER_ID = 'admin-user' @@ -2549,17 +2549,17 @@ def test_falls_back_to_recreate_when_remove_fails( mock_create.assert_called_once_with('SF', ['n1', 'n2']) assert result['Id'] == 'new-pl' - @patch('tasks.mediaserver_jellyfin._create_fresh_playlist') - @patch('tasks.mediaserver_jellyfin.delete_playlist') - @patch('tasks.mediaserver_jellyfin._remove_playlist_entries') - @patch('tasks.mediaserver_jellyfin._get_playlist_entry_ids') - @patch('tasks.mediaserver_jellyfin.get_playlist_by_name') - @patch('tasks.mediaserver_jellyfin.config') + @patch('tasks.mediaserver.jellyfin._create_fresh_playlist') + @patch('tasks.mediaserver.jellyfin.delete_playlist') + @patch('tasks.mediaserver.jellyfin._remove_playlist_entries') + @patch('tasks.mediaserver.jellyfin._get_playlist_entry_ids') + @patch('tasks.mediaserver.jellyfin.get_playlist_by_name') + @patch('tasks.mediaserver.jellyfin.config') def test_fallback_returns_none_when_delete_playlist_fails( self, mock_config, mock_get, mock_get_entries, mock_remove, mock_delete, mock_create ): """If the fallback delete itself fails, don't try to recreate — bail out.""" - from tasks.mediaserver_jellyfin import create_or_replace_playlist + from tasks.mediaserver.jellyfin import create_or_replace_playlist mock_config.JELLYFIN_URL = 'http://jf' mock_config.JELLYFIN_USER_ID = 'admin-user' @@ -2578,11 +2578,11 @@ def test_fallback_returns_none_when_delete_playlist_fails( class TestEmbyCreateOrReplacePlaylist: """Emby upsert under /emby/ prefix with uppercase params.""" - @patch('tasks.mediaserver_emby.requests') - @patch('tasks.mediaserver_emby.get_playlist_by_name') - @patch('tasks.mediaserver_emby.config') + @patch('tasks.mediaserver.emby.requests') + @patch('tasks.mediaserver.emby.get_playlist_by_name') + @patch('tasks.mediaserver.emby.config') def test_existing_playlist_clears_and_adds_preserving_id(self, mock_config, mock_get, mock_requests): - from tasks.mediaserver_emby import create_or_replace_playlist + from tasks.mediaserver.emby import create_or_replace_playlist mock_config.EMBY_URL = 'http://emby' mock_config.EMBY_USER_ID = 'admin-emby' @@ -2609,15 +2609,15 @@ def test_existing_playlist_clears_and_adds_preserving_id(self, mock_config, mock assert post_call[1]['params']['UserId'] == 'admin-emby' assert result['Id'] == 'emby-pl' - @patch('tasks.mediaserver_emby._add_items_to_playlist') - @patch('tasks.mediaserver_emby._remove_playlist_entries') - @patch('tasks.mediaserver_emby._get_playlist_entry_ids') - @patch('tasks.mediaserver_emby.get_playlist_by_name') - @patch('tasks.mediaserver_emby.config') + @patch('tasks.mediaserver.emby._add_items_to_playlist') + @patch('tasks.mediaserver.emby._remove_playlist_entries') + @patch('tasks.mediaserver.emby._get_playlist_entry_ids') + @patch('tasks.mediaserver.emby.get_playlist_by_name') + @patch('tasks.mediaserver.emby.config') def test_returns_none_when_add_fails_after_clear( self, mock_config, mock_get, mock_get_entries, mock_remove, mock_add ): - from tasks.mediaserver_emby import create_or_replace_playlist + from tasks.mediaserver.emby import create_or_replace_playlist mock_config.EMBY_URL = 'http://emby' mock_config.EMBY_USER_ID = 'admin-emby' @@ -2635,11 +2635,11 @@ def test_returns_none_when_add_fails_after_clear( class TestLyrionCreateOrReplacePlaylist: """Lyrion upsert: deletes existing then creates fresh (ID may change — known limitation).""" - @patch('tasks.mediaserver_lyrion._create_playlist_batched') - @patch('tasks.mediaserver_lyrion.delete_playlist') - @patch('tasks.mediaserver_lyrion.get_playlist_by_name') + @patch('tasks.mediaserver.lyrion._create_playlist_batched') + @patch('tasks.mediaserver.lyrion.delete_playlist') + @patch('tasks.mediaserver.lyrion.get_playlist_by_name') def test_existing_deletes_then_creates(self, mock_get, mock_delete, mock_create): - from tasks.mediaserver_lyrion import create_or_replace_playlist + from tasks.mediaserver.lyrion import create_or_replace_playlist mock_get.return_value = {'Id': 99, 'Name': 'SF'} mock_delete.return_value = True @@ -2651,11 +2651,11 @@ def test_existing_deletes_then_creates(self, mock_get, mock_delete, mock_create) mock_create.assert_called_once_with('SF', ['t1']) assert result['Name'] == 'SF' - @patch('tasks.mediaserver_lyrion._create_playlist_batched') - @patch('tasks.mediaserver_lyrion.delete_playlist') - @patch('tasks.mediaserver_lyrion.get_playlist_by_name') + @patch('tasks.mediaserver.lyrion._create_playlist_batched') + @patch('tasks.mediaserver.lyrion.delete_playlist') + @patch('tasks.mediaserver.lyrion.get_playlist_by_name') def test_missing_creates_without_delete(self, mock_get, mock_delete, mock_create): - from tasks.mediaserver_lyrion import create_or_replace_playlist + from tasks.mediaserver.lyrion import create_or_replace_playlist mock_get.return_value = None mock_create.return_value = {'Id': 50, 'Name': 'SF'} @@ -2665,12 +2665,12 @@ def test_missing_creates_without_delete(self, mock_get, mock_delete, mock_create mock_delete.assert_not_called() mock_create.assert_called_once_with('SF', ['t1']) - @patch('tasks.mediaserver_lyrion._create_playlist_batched') - @patch('tasks.mediaserver_lyrion.delete_playlist') - @patch('tasks.mediaserver_lyrion.get_playlist_by_name') + @patch('tasks.mediaserver.lyrion._create_playlist_batched') + @patch('tasks.mediaserver.lyrion.delete_playlist') + @patch('tasks.mediaserver.lyrion.get_playlist_by_name') def test_aborts_when_delete_fails(self, mock_get, mock_delete, mock_create): """If delete fails, return None instead of creating a duplicate playlist.""" - from tasks.mediaserver_lyrion import create_or_replace_playlist + from tasks.mediaserver.lyrion import create_or_replace_playlist mock_get.return_value = {'Id': 99, 'Name': 'SF'} mock_delete.return_value = False @@ -2695,15 +2695,15 @@ def test_requires_name_and_ids(self, mock_config): create_or_replace_playlist('Name', []) @patch('tasks.mediaserver.config') - def test_mpd_raises_not_implemented(self, mock_config): + def test_unsupported_backend_raises_not_implemented(self, mock_config): from tasks.mediaserver import create_or_replace_playlist - mock_config.MEDIASERVER_TYPE = 'mpd' + mock_config.MEDIASERVER_TYPE = 'unsupported' with pytest.raises(NotImplementedError): create_or_replace_playlist('SF', ['s1']) - @patch('tasks.mediaserver.navidrome_create_or_replace_playlist') + @patch('tasks.mediaserver.navidrome.create_or_replace_playlist') @patch('tasks.mediaserver.config') def test_dispatches_to_navidrome(self, mock_config, mock_provider): from tasks.mediaserver import create_or_replace_playlist @@ -2716,7 +2716,7 @@ def test_dispatches_to_navidrome(self, mock_config, mock_provider): mock_provider.assert_called_once_with('SF', ['s1'], None) assert result['Id'] == 'pl-1' - @patch('tasks.mediaserver.jellyfin_create_or_replace_playlist') + @patch('tasks.mediaserver.jellyfin.create_or_replace_playlist') @patch('tasks.mediaserver.config') def test_dispatches_to_jellyfin(self, mock_config, mock_provider): from tasks.mediaserver import create_or_replace_playlist @@ -2728,7 +2728,7 @@ def test_dispatches_to_jellyfin(self, mock_config, mock_provider): mock_provider.assert_called_once() - @patch('tasks.mediaserver.emby_create_or_replace_playlist') + @patch('tasks.mediaserver.emby.create_or_replace_playlist') @patch('tasks.mediaserver.config') def test_dispatches_to_emby(self, mock_config, mock_provider): from tasks.mediaserver import create_or_replace_playlist @@ -2740,7 +2740,7 @@ def test_dispatches_to_emby(self, mock_config, mock_provider): mock_provider.assert_called_once() - @patch('tasks.mediaserver.lyrion_create_or_replace_playlist') + @patch('tasks.mediaserver.lyrion.create_or_replace_playlist') @patch('tasks.mediaserver.config') def test_dispatches_to_lyrion(self, mock_config, mock_provider): from tasks.mediaserver import create_or_replace_playlist @@ -2782,11 +2782,11 @@ def _audio_page(n_items, start=0): class TestJellyfinGetAllSongsPagination: """Jellyfin get_all_songs: paginate, and raise rather than truncate.""" - @patch('tasks.mediaserver_jellyfin.requests.get') - @patch('tasks.mediaserver_jellyfin.config') + @patch('tasks.mediaserver.jellyfin.requests.get') + @patch('tasks.mediaserver.jellyfin.config') def test_paginates_until_short_page(self, mock_config, mock_get): """A full page (== limit) triggers another request; a short page stops it.""" - from tasks.mediaserver_jellyfin import get_all_songs + from tasks.mediaserver.jellyfin import get_all_songs mock_config.JELLYFIN_URL = 'http://jellyfin:8096' mock_config.JELLYFIN_USER_ID = 'user123' @@ -2803,11 +2803,11 @@ def test_paginates_until_short_page(self, mock_config, mock_get): assert page2_params['StartIndex'] == 500 assert page2_params['Limit'] == 500 - @patch('tasks.mediaserver_jellyfin.requests.get') - @patch('tasks.mediaserver_jellyfin.config') + @patch('tasks.mediaserver.jellyfin.requests.get') + @patch('tasks.mediaserver.jellyfin.config') def test_raises_on_midscan_failure_instead_of_truncating(self, mock_config, mock_get): """A timeout on a later page must propagate, NOT return the partial list.""" - from tasks.mediaserver_jellyfin import get_all_songs + from tasks.mediaserver.jellyfin import get_all_songs mock_config.JELLYFIN_URL = 'http://jellyfin:8096' mock_config.JELLYFIN_USER_ID = 'user123' @@ -2821,11 +2821,11 @@ def test_raises_on_midscan_failure_instead_of_truncating(self, mock_config, mock with pytest.raises(requests.exceptions.ReadTimeout): get_all_songs() - @patch('tasks.mediaserver_jellyfin.requests.get') - @patch('tasks.mediaserver_jellyfin.config') + @patch('tasks.mediaserver.jellyfin.requests.get') + @patch('tasks.mediaserver.jellyfin.config') def test_empty_library_returns_empty_without_raising(self, mock_config, mock_get): """A genuinely empty library (first page empty) returns [], does not raise.""" - from tasks.mediaserver_jellyfin import get_all_songs + from tasks.mediaserver.jellyfin import get_all_songs mock_config.JELLYFIN_URL = 'http://jellyfin:8096' mock_config.JELLYFIN_USER_ID = 'user123' @@ -2839,10 +2839,10 @@ class TestEmbyGetAllSongsRaisesOnFailure: """Emby is already paginated (limit=1000); on a page error it must raise, not return the partial list it had accumulated.""" - @patch('tasks.mediaserver_emby.requests.get') - @patch('tasks.mediaserver_emby.config') + @patch('tasks.mediaserver.emby.requests.get') + @patch('tasks.mediaserver.emby.config') def test_raises_on_midscan_failure_instead_of_truncating(self, mock_config, mock_get): - from tasks.mediaserver_emby import get_all_songs + from tasks.mediaserver.emby import get_all_songs mock_config.EMBY_URL = 'http://emby:8096' mock_config.EMBY_USER_ID = 'user123' @@ -2856,10 +2856,10 @@ def test_raises_on_midscan_failure_instead_of_truncating(self, mock_config, mock with pytest.raises(requests.exceptions.ReadTimeout): get_all_songs() - @patch('tasks.mediaserver_emby.requests.get') - @patch('tasks.mediaserver_emby.config') + @patch('tasks.mediaserver.emby.requests.get') + @patch('tasks.mediaserver.emby.config') def test_empty_library_returns_empty_without_raising(self, mock_config, mock_get): - from tasks.mediaserver_emby import get_all_songs + from tasks.mediaserver.emby import get_all_songs mock_config.EMBY_URL = 'http://emby:8096' mock_config.EMBY_USER_ID = 'user123' diff --git a/tests/unit/test_memory_cleanup.py b/test/unit/test_memory_cleanup.py similarity index 100% rename from tests/unit/test_memory_cleanup.py rename to test/unit/test_memory_cleanup.py diff --git a/tests/unit/test_memory_utils.py b/test/unit/test_memory_utils.py similarity index 100% rename from tests/unit/test_memory_utils.py rename to test/unit/test_memory_utils.py diff --git a/tests/unit/test_numeric_bootstrap.py b/test/unit/test_numeric_bootstrap.py similarity index 100% rename from tests/unit/test_numeric_bootstrap.py rename to test/unit/test_numeric_bootstrap.py diff --git a/tests/unit/test_path_manager.py b/test/unit/test_path_manager.py similarity index 100% rename from tests/unit/test_path_manager.py rename to test/unit/test_path_manager.py diff --git a/tests/unit/test_playlist_ordering.py b/test/unit/test_playlist_ordering.py similarity index 99% rename from tests/unit/test_playlist_ordering.py rename to test/unit/test_playlist_ordering.py index c041321f..a8106f17 100644 --- a/tests/unit/test_playlist_ordering.py +++ b/test/unit/test_playlist_ordering.py @@ -9,7 +9,7 @@ - Handling of songs missing from database """ -from tests.conftest import _import_module +from test.unit.conftest import _import_module def _load_playlist_ordering(): diff --git a/tests/unit/test_provider_migration_blueprint.py b/test/unit/test_provider_migration_blueprint.py similarity index 98% rename from tests/unit/test_provider_migration_blueprint.py rename to test/unit/test_provider_migration_blueprint.py index 0fd3e22f..fd3181ca 100644 --- a/tests/unit/test_provider_migration_blueprint.py +++ b/test/unit/test_provider_migration_blueprint.py @@ -213,7 +213,7 @@ def test_returns_warning_when_still_not_absolute(self, bp_mod, client): def test_rejects_unsupported_current_provider(self, bp_mod, client): import config - config.MEDIASERVER_TYPE = 'mpd' + config.MEDIASERVER_TYPE = 'plex' resp = client.post('/api/migration/source-paths/refresh', json={'session_id': 1}) assert resp.status_code == 400 @@ -389,8 +389,8 @@ def test_rejects_unsafe_urls(self, bp_mod, url): assert isinstance(reason, str) and reason # a human-readable reason @pytest.mark.parametrize('creds', [{}, {'url': ''}, {'url': None}]) - def test_missing_url_is_allowed_for_mpd(self, bp_mod, creds): - # MPD targets carry no URL; the wrapper lets the downstream probe handle it. + def test_missing_url_is_allowed(self, bp_mod, creds): + # The wrapper lets the downstream probe handle a missing URL. ok, reason = bp_mod._validate_probe_url(creds) assert ok is True assert reason is None diff --git a/tests/unit/test_provider_migration_execute.py b/test/unit/test_provider_migration_execute.py similarity index 100% rename from tests/unit/test_provider_migration_execute.py rename to test/unit/test_provider_migration_execute.py diff --git a/tests/unit/test_provider_migration_matcher.py b/test/unit/test_provider_migration_matcher.py similarity index 100% rename from tests/unit/test_provider_migration_matcher.py rename to test/unit/test_provider_migration_matcher.py diff --git a/tests/unit/test_provider_probe.py b/test/unit/test_provider_probe.py similarity index 97% rename from tests/unit/test_provider_probe.py rename to test/unit/test_provider_probe.py index eb50ed19..83a240c9 100644 --- a/tests/unit/test_provider_probe.py +++ b/test/unit/test_provider_probe.py @@ -116,7 +116,7 @@ def test_supported_providers_normalized_lowercase(self, probe): def test_unsupported_provider_raises(self, probe): with pytest.raises(ValueError) as ei: - probe._normalize_provider_type('mpd') + probe._normalize_provider_type('plex') assert 'not supported' in str(ei.value) def test_empty_or_none_raises(self, probe): @@ -158,7 +158,7 @@ def test_empty_result_is_handled(self, probe): def test_unsupported_provider_raises_before_call(self, probe): with patch.object(probe.mediaserver, 'get_all_songs') as m: with pytest.raises(ValueError): - probe.fetch_all_tracks('mpd', self.CREDS) + probe.fetch_all_tracks('plex', self.CREDS) m.assert_not_called() @@ -179,7 +179,7 @@ def test_delegates_to_mediaserver(self, probe): def test_unsupported_provider_raises(self, probe): with patch.object(probe.mediaserver, 'search_albums') as m: with pytest.raises(ValueError): - probe.search_albums('mpd', self.CREDS, 'q') + probe.search_albums('plex', self.CREDS, 'q') m.assert_not_called() @@ -224,5 +224,5 @@ def test_delegates_to_mediaserver(self, probe): def test_unsupported_provider_raises(self, probe): with patch.object(probe.mediaserver, 'test_connection') as m: with pytest.raises(ValueError): - probe.test_connection('mpd', self.CREDS) + probe.test_connection('plex', self.CREDS) m.assert_not_called() diff --git a/tests/unit/test_radius_walk_helper.py b/test/unit/test_radius_walk_helper.py similarity index 100% rename from tests/unit/test_radius_walk_helper.py rename to test/unit/test_radius_walk_helper.py diff --git a/tests/unit/test_restart_manager.py b/test/unit/test_restart_manager.py similarity index 100% rename from tests/unit/test_restart_manager.py rename to test/unit/test_restart_manager.py diff --git a/tests/unit/test_security_ssrf.py b/test/unit/test_security_ssrf.py similarity index 100% rename from tests/unit/test_security_ssrf.py rename to test/unit/test_security_ssrf.py diff --git a/tests/unit/test_sem_grove_manager.py b/test/unit/test_sem_grove_manager.py similarity index 99% rename from tests/unit/test_sem_grove_manager.py rename to test/unit/test_sem_grove_manager.py index 8c6dbf92..c35d8348 100644 --- a/tests/unit/test_sem_grove_manager.py +++ b/test/unit/test_sem_grove_manager.py @@ -1,4 +1,4 @@ -# tests/unit/test_sem_grove_manager.py +# test/unit/test_sem_grove_manager.py """ Unit tests for tasks/sem_grove_manager.py diff --git a/tests/unit/test_setup_manager.py b/test/unit/test_setup_manager.py similarity index 100% rename from tests/unit/test_setup_manager.py rename to test/unit/test_setup_manager.py diff --git a/tests/unit/test_song_alchemy.py b/test/unit/test_song_alchemy.py similarity index 100% rename from tests/unit/test_song_alchemy.py rename to test/unit/test_song_alchemy.py diff --git a/tests/unit/test_sonic_fingerprint_manager.py b/test/unit/test_sonic_fingerprint_manager.py similarity index 100% rename from tests/unit/test_sonic_fingerprint_manager.py rename to test/unit/test_sonic_fingerprint_manager.py diff --git a/tests/unit/test_sql_injection_params.py b/test/unit/test_sql_injection_params.py similarity index 100% rename from tests/unit/test_sql_injection_params.py rename to test/unit/test_sql_injection_params.py diff --git a/tests/unit/test_string_sanitization.py b/test/unit/test_string_sanitization.py similarity index 100% rename from tests/unit/test_string_sanitization.py rename to test/unit/test_string_sanitization.py diff --git a/tests/unit/test_taskqueue.py b/test/unit/test_taskqueue.py similarity index 100% rename from tests/unit/test_taskqueue.py rename to test/unit/test_taskqueue.py diff --git a/tests/unit/test_tool_plan.py b/test/unit/test_tool_plan.py similarity index 100% rename from tests/unit/test_tool_plan.py rename to test/unit/test_tool_plan.py diff --git a/tests/unit/test_tz_helper.py b/test/unit/test_tz_helper.py similarity index 100% rename from tests/unit/test_tz_helper.py rename to test/unit/test_tz_helper.py diff --git a/tests/unit/test_voyager_manager.py b/test/unit/test_voyager_manager.py similarity index 99% rename from tests/unit/test_voyager_manager.py rename to test/unit/test_voyager_manager.py index da1fa985..ff809c4e 100644 --- a/tests/unit/test_voyager_manager.py +++ b/test/unit/test_voyager_manager.py @@ -1,4 +1,4 @@ -# tests/unit/test_voyager_manager.py +# test/unit/test_voyager_manager.py """ Unit tests for tasks/voyager_manager.py @@ -479,7 +479,7 @@ def test_raises_when_id_map_not_loaded(self): class TestCreatePlaylistFromIds: """Test playlist creation functionality""" - @patch('tasks.voyager_manager.create_instant_playlist') + @patch('tasks.mediaserver.create_instant_playlist') def test_calls_mediaserver_create_playlist(self, mock_create): """Should call mediaserver create_instant_playlist""" from tasks.voyager_manager import create_playlist_from_ids @@ -491,7 +491,7 @@ def test_calls_mediaserver_create_playlist(self, mock_create): assert result == 'playlist-123' mock_create.assert_called_once_with('Test Playlist', ['track-1', 'track-2'], user_creds=None) - @patch('tasks.voyager_manager.create_instant_playlist') + @patch('tasks.mediaserver.create_instant_playlist') def test_raises_on_creation_failure(self, mock_create): """Should raise exception if playlist creation fails""" from tasks.voyager_manager import create_playlist_from_ids @@ -501,7 +501,7 @@ def test_raises_on_creation_failure(self, mock_create): with pytest.raises(Exception, match="Playlist creation failed"): create_playlist_from_ids('Test Playlist', ['track-1']) - @patch('tasks.voyager_manager.create_instant_playlist') + @patch('tasks.mediaserver.create_instant_playlist') def test_raises_on_missing_playlist_id(self, mock_create): """Should raise exception if response has no Id""" from tasks.voyager_manager import create_playlist_from_ids @@ -511,7 +511,7 @@ def test_raises_on_missing_playlist_id(self, mock_create): with pytest.raises(Exception, match="did not include a playlist ID"): create_playlist_from_ids('Test Playlist', ['track-1']) - @patch('tasks.voyager_manager.create_instant_playlist') + @patch('tasks.mediaserver.create_instant_playlist') def test_passes_user_credentials(self, mock_create): """Should pass user credentials to mediaserver""" from tasks.voyager_manager import create_playlist_from_ids diff --git a/tests/unit/test_windows_paths.py b/test/unit/test_windows_paths.py similarity index 95% rename from tests/unit/test_windows_paths.py rename to test/unit/test_windows_paths.py index b6022ae7..5ffd9305 100644 --- a/tests/unit/test_windows_paths.py +++ b/test/unit/test_windows_paths.py @@ -1,11 +1,11 @@ import os from unittest.mock import patch -from tests.conftest import _import_module +from test.unit.conftest import _import_module def _load_paths(): - return _import_module('windows.paths', 'windows/paths.py') + return _import_module('windows.paths', 'native-build/windows/paths.py') def _norm(path): diff --git a/testing_suite/instant_playlist_optimize_config.yaml b/testing_suite/instant_playlist_optimize_config.yaml deleted file mode 100644 index 7c5c49c5..00000000 --- a/testing_suite/instant_playlist_optimize_config.yaml +++ /dev/null @@ -1,252 +0,0 @@ -# Instant Playlist - Optimization Test Config -# Single instance, iterative testing to improve prompt + code quality - -instance: - api_url: "http://localhost:8000" - -test_config: - timeout_per_request: 300 - retry_on_error: 1 - retry_delay: 5 - -models: - # --- OpenRouter models --- - - provider: "openrouter" - name: "Claude Sonnet 4.6" - model_id: "anthropic/claude-sonnet-4.6" - enabled: false - - - provider: "openrouter" - name: "Claude 4.5 Haiku" - model_id: "anthropic/claude-haiku-4.5" - enabled: false - - - provider: "openrouter" - name: "Gemini 3 Flash" - model_id: "google/gemini-3-flash-preview" - enabled: false - - - provider: "openrouter" - name: "GPT-4o Mini" - model_id: "openai/gpt-4o-mini" - enabled: false - - # --- Ollama models --- - # Benchmark #1: 0.960 agent score, perfect restraint - - provider: "ollama" - name: "Qwen 3 1.7B" - model_id: "qwen3:1.7b" - url: "http://192.168.1.71:11434/api/generate" - enabled: false - - # Benchmark #2: 0.920, fastest model (1.6s), perfect restraint - - provider: "ollama" - name: "LFM 2.5 1.2B" - model_id: "lfm2.5-thinking:1.2b" - url: "http://192.168.1.71:11434/api/generate" - enabled: false - - # Benchmark #4: 0.800, perfect restraint - - provider: "ollama" - name: "Qwen 2.5 1.5B" - model_id: "qwen2.5:1.5b" - url: "http://192.168.1.71:11434/api/generate" - enabled: false - - # Rank 3: avg 2.7, 21.7s avg response - - provider: "ollama" - name: "Ministral 3 3B" - model_id: "ministral-3:3b" - url: "http://192.168.1.71:11434/api/generate" - enabled: true - - # Benchmark #7: 0.780, perfect restraint - - provider: "ollama" - name: "Phi 4 Mini 3.8B" - model_id: "phi4-mini:3.8b" - url: "http://192.168.1.71:11434/api/generate" - enabled: false - - # Benchmark #10: 0.660, high action but 0.000 restraint - - provider: "ollama" - name: "Llama 3.2 3B" - model_id: "llama3.2:3b" - url: "http://192.168.1.71:11434/api/generate" - enabled: false - - # Rank 1: avg 2.9, 6.3s avg response (fastest top model) - - provider: "ollama" - name: "Gemma 3 4B" - model_id: "gemma3:4b" - url: "http://192.168.1.71:11434/api/generate" - enabled: true - - - provider: "ollama" - name: "Qwen 3.5 0.8B" - model_id: "qwen3.5:0.8b" - url: "http://192.168.1.71:11434/api/generate" - enabled: false - - - provider: "ollama" - name: "Qwen 3.5 2B" - model_id: "qwen3.5:2b" - url: "http://192.168.1.71:11434/api/generate" - enabled: false - - # Rank 3: avg 2.7, 57.4s avg response - - provider: "ollama" - name: "Qwen 3.5 4B" - model_id: "qwen3.5:4b" - url: "http://192.168.1.71:11434/api/generate" - enabled: true - - # Rank 1: avg 2.9, 46.7s avg response - - provider: "ollama" - name: "Qwen 3.5 9B" - model_id: "qwen3.5:9b" - url: "http://192.168.1.71:11434/api/generate" - enabled: true - -test_prompts: - # ===== BUG FIX VALIDATION ===== - # These test the 4 bugs we just fixed - - # Bug 1: Year filter - was setting year_min=1, year_max=2026 - - prompt: "2026 songs" - category: "year_filter" - expected: - min_songs: 50 - expected_tools: ["search_database"] - must_have_filter: ["year="] - no_extra_filters: true - allowed_filters: ["year_min", "year_max"] - - - prompt: "give me 100 songs from 2026" - category: "year_filter" - expected: - min_songs: 80 - expected_tools: ["search_database"] - must_have_filter: ["year="] - no_extra_filters: true - allowed_filters: ["year_min", "year_max"] - - - prompt: "songs from 2024" - category: "year_filter" - expected: - min_songs: 30 - expected_tools: ["search_database"] - must_have_filter: ["year="] - - - prompt: "90s rock" - category: "year_filter" - expected: - expected_tools: ["search_database"] - must_have_filter: ["year="] - - # Bug 2: Random rating filter added - - prompt: "electronic music" - category: "no_extra_filter" - expected: - expected_tools: ["search_database"] - no_extra_filters: true - allowed_filters: ["genres"] - - # Bug 3: Random genre filter added - - prompt: "songs from 2020-2025" - category: "no_extra_filter" - expected: - expected_tools: ["search_database"] - no_extra_filters: true - allowed_filters: ["year_min", "year_max"] - - # Bug 4: Per-artist cap reducing results below 100 - - prompt: "2026 songs" - category: "artist_cap" - expected: - min_songs: 80 - - # ===== TOOL SELECTION ===== - - - prompt: "Songs similar to By the Way by Red Hot Chili Peppers" - category: "song_similarity" - expected: - expected_tools: ["song_similarity"] - min_songs: 50 - - - prompt: "calm piano music" - category: "text_search" - expected: - expected_tools: ["text_search"] - min_songs: 50 - - - prompt: "songs like AC/DC" - category: "artist_similarity" - expected: - expected_tools: ["artist_similarity"] - min_songs: 50 - - - prompt: "songs from blink-182" - category: "artist_filter" - expected: - expected_tools: ["search_database"] - - - prompt: "top songs of Madonna" - category: "ai_brainstorm" - expected: - expected_tools: ["ai_brainstorm"] - - - prompt: "sounds like Iron Maiden and Metallica combined" - category: "song_alchemy" - expected: - expected_tools: ["song_alchemy"] - min_songs: 50 - - # ===== FILTER COMBINATIONS ===== - - - prompt: "rock 5 star songs" - category: "combined_filter" - expected: - expected_tools: ["search_database"] - must_have_filter: ["min_rating"] - - - prompt: "sad jazz songs" - category: "combined_filter" - expected: - expected_tools: ["search_database"] - - - prompt: "fast metal songs" - category: "combined_filter" - expected: - expected_tools: ["search_database"] - - - prompt: "songs in minor key" - category: "scale_filter" - expected: - expected_tools: ["search_database"] - - # ===== COMPLEX / MULTI-TOOL ===== - - - prompt: "energetic rock music for working out" - category: "multi_tool" - expected: - min_songs: 50 - - - prompt: "mix of Daft Punk and Gorillaz" - category: "song_alchemy" - expected: - expected_tools: ["song_alchemy"] - min_songs: 50 - - - prompt: "High-energy metal and hard rock from 2000-2015, in minor scale, between 120-180 BPM" - category: "multi_filter" - expected: - expected_tools: ["search_database"] - must_have_filter: ["year="] - - - prompt: "songs similar to Metallica, I want a huge playlist with lots of variety" - category: "diversity_stress" - expected: - min_songs: 80 - -output: - directory: "testing_suite/reports/optimization" diff --git a/windows/__init__.py b/windows/__init__.py deleted file mode 100644 index 5bc9e002..00000000 --- a/windows/__init__.py +++ /dev/null @@ -1,4 +0,0 @@ -"""Windows standalone build package. - -See windows/README.md for build instructions. -"""