Merge pull request #640 from NeptuneHub/devel

Code refactoring: Import depth cleaning

Author: NeptuneHub

.github/workflows/build-windows.yml

@@ -141,14 +141,18 @@ jobs:
       - name: Create zip for release
         run: |
           python -c "
-          import zipfile, pathlib
-          src = pathlib.Path('artifacts/AudioMuse-AI')
+          import zipfile, pathlib, sys
+          # upload-artifact strips the directory pointed at by 'path:', so the
+          # bundle's files land directly under artifacts/ (not artifacts/AudioMuse-AI/).
+          src = pathlib.Path('artifacts')
           out = 'AudioMuse-AI-amd64-windows.zip'
+          files = [f for f in sorted(src.rglob('*')) if f.is_file()]
+          if not files:
+              sys.exit('No files found under artifacts/ -- the build artifact is missing or empty; refusing to ship an empty zip')
           with zipfile.ZipFile(out, 'w', zipfile.ZIP_DEFLATED, allowZip64=True) as zf:
-              for f in sorted(src.rglob('*')):
-                  if f.is_file():
-                      zf.write(f, f.relative_to(src.parent).as_posix())
-          print(f'Created {out}')
+              for f in files:
+                  zf.write(f, 'AudioMuse-AI/' + f.relative_to(src).as_posix())
+          print(f'Created {out} with {len(files)} files')
           "
 
       - name: Attach packages to the release

.github/workflows/lint-flake8.yml

@@ -28,4 +28,4 @@ jobs:
 
     - name: Run flake8 static analysis
       run: |
-        flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics
+        flake8 . --count --select=E9,F63,F7,F82,F401,F811 --show-source --statistics

app.py

@@ -32,10 +32,13 @@
 
 # Import helper functions
 from app_helper import (
-    init_db, get_db, close_db,
+    get_db, close_db,
     redis_conn,
     get_task_info_from_db,
     cancel_job_and_children_recursive,
+)
+from database import init_db
+from config import (
     TASK_STATUS_PENDING, TASK_STATUS_STARTED, TASK_STATUS_PROGRESS,
     TASK_STATUS_SUCCESS, TASK_STATUS_FAILURE, TASK_STATUS_REVOKED
 )
@@ -46,8 +49,6 @@
     resolve_jwt_secret,
 )
 
-from app_provider_migration import migration_bp
-
 from error import error_manager
 from error.error_manager import AudioMuseError
 from error.error_dictionary import UNKNOWN_ERROR_CODE
@@ -172,7 +173,7 @@ def teardown_db(e=None):
 else:
     app.logger.info("RQ worker mode: skipping startup database schema bootstrap.")
 
-import app_setup
+import app_setup  # noqa: F401
 
 # --- API Endpoints ---
 
@@ -705,7 +706,7 @@ def listen_for_index_reloads():
             load_voyager_index_for_querying(force_reload=True)
             from tasks.artist_gmm_manager import load_artist_index_for_querying
             load_artist_index_for_querying(force_reload=True)
-            from app_helper import load_map_projection, load_artist_projection
+            from database import load_map_projection, load_artist_projection
             load_map_projection('main_map', force_reload=True)
             load_artist_projection('artist_map', force_reload=True)
             # Rebuild the map JSON cache used by the /api/map endpoint
@@ -757,48 +758,56 @@ def listen_for_index_reloads():
 
 
 
-# --- Import and Register Blueprints ---
-# This is the original, working structure.
-from app_chat import chat_bp
-from app_clustering import clustering_bp
-from app_analysis import analysis_bp
-from app_cron import cron_bp, run_due_cron_jobs
-from app_voyager import voyager_bp
-from app_sonic_fingerprint import sonic_fingerprint_bp
-from app_path import path_bp
-from app_external import external_bp # --- NEW: Import the external blueprint ---
-from app_alchemy import alchemy_bp
-from app_map import map_bp
-from app_waveform import waveform_bp
-from app_artist_similarity import artist_similarity_bp
-from app_clap_search import clap_search_bp
-from app_lyrics import lyrics_search_bp
-from app_sem_grove import sem_grove_bp
-from app_backup import backup_bp
-from app_dashboard import dashboard_bp
-from app_users import users_bp
-from app_sync import sync_bp
-
-app.register_blueprint(chat_bp, url_prefix='/chat')
-app.register_blueprint(clustering_bp)
-app.register_blueprint(analysis_bp)
-app.register_blueprint(cron_bp)
-app.register_blueprint(voyager_bp)
-app.register_blueprint(sonic_fingerprint_bp)
-app.register_blueprint(path_bp)
-app.register_blueprint(external_bp, url_prefix='/external') # --- NEW: Register the external blueprint ---
-app.register_blueprint(alchemy_bp)
-app.register_blueprint(map_bp)
-app.register_blueprint(waveform_bp)
-app.register_blueprint(artist_similarity_bp)
-app.register_blueprint(clap_search_bp)
-app.register_blueprint(lyrics_search_bp)
-app.register_blueprint(sem_grove_bp)
-app.register_blueprint(backup_bp)
-app.register_blueprint(migration_bp)
-app.register_blueprint(dashboard_bp)
-app.register_blueprint(users_bp)
-app.register_blueprint(sync_bp)
+# --- Blueprint Registration ---
+# Standard Flask factory pattern: blueprint imports are inside
+# this function so the eager import graph stays flat.
+
+
+def _register_blueprints(flask_app):
+    from app_chat import chat_bp
+    from app_clustering import clustering_bp
+    from app_analysis import analysis_bp
+    from app_cron import cron_bp
+    from app_voyager import voyager_bp
+    from app_sonic_fingerprint import sonic_fingerprint_bp
+    from app_path import path_bp
+    from app_external import external_bp
+    from app_alchemy import alchemy_bp
+    from app_map import map_bp
+    from app_waveform import waveform_bp
+    from app_artist_similarity import artist_similarity_bp
+    from app_clap_search import clap_search_bp
+    from app_lyrics import lyrics_search_bp
+    from app_sem_grove import sem_grove_bp
+    from app_backup import backup_bp
+    from app_provider_migration import migration_bp
+    from app_dashboard import dashboard_bp
+    from app_users import users_bp
+    from app_sync import sync_bp
+
+    flask_app.register_blueprint(chat_bp, url_prefix='/chat')
+    flask_app.register_blueprint(clustering_bp)
+    flask_app.register_blueprint(analysis_bp)
+    flask_app.register_blueprint(cron_bp)
+    flask_app.register_blueprint(voyager_bp)
+    flask_app.register_blueprint(sonic_fingerprint_bp)
+    flask_app.register_blueprint(path_bp)
+    flask_app.register_blueprint(external_bp, url_prefix='/external')
+    flask_app.register_blueprint(alchemy_bp)
+    flask_app.register_blueprint(map_bp)
+    flask_app.register_blueprint(waveform_bp)
+    flask_app.register_blueprint(artist_similarity_bp)
+    flask_app.register_blueprint(clap_search_bp)
+    flask_app.register_blueprint(lyrics_search_bp)
+    flask_app.register_blueprint(sem_grove_bp)
+    flask_app.register_blueprint(backup_bp)
+    flask_app.register_blueprint(migration_bp)
+    flask_app.register_blueprint(dashboard_bp)
+    flask_app.register_blueprint(users_bp)
+    flask_app.register_blueprint(sync_bp)
+
+
+_register_blueprints(app)
 
 # --- Startup: Load indexes and caches (Flask server only, NOT RQ workers) ---
 # RQ workers import app.py but should NOT load indexes or start background threads.
@@ -828,7 +837,7 @@ def listen_for_index_reloads():
       logger.debug(f"No precomputed map projection to load at startup or load failed: {e}")
     # Also try to load artist component projection into memory
     try:
-      from app_helper import load_artist_projection
+      from database import load_artist_projection
       load_artist_projection('artist_map')
       logger.info("In-memory artist component projection loaded at startup.")
     except Exception as e:
@@ -895,6 +904,7 @@ def _start_map_init_background():
   def _cron_manager_loop():
     try:
       from time import sleep
+      from app_cron import run_due_cron_jobs
       while True:
         try:
           with app.app_context():

app_alchemy.py

@@ -185,7 +185,7 @@ def list_anchors():
       500:
         description: Database error.
     """
-    from app_helper import get_alchemy_anchors
+    from database import get_alchemy_anchors
     try:
         anchors = get_alchemy_anchors()
         # no centroid returned here (name-only list)
@@ -227,7 +227,7 @@ def create_anchor():
       500:
         description: Database failure.
     """
-    from app_helper import save_alchemy_anchor
+    from database import save_alchemy_anchor
     payload = request.get_json() or {}
     name = (payload.get('name') or '').strip()
     centroid = payload.get('centroid')
@@ -260,7 +260,7 @@ def remove_anchor(anchor_id):
       404:
         description: Anchor not found.
     """
-    from app_helper import delete_alchemy_anchor
+    from database import delete_alchemy_anchor
     ok = delete_alchemy_anchor(anchor_id)
     if not ok:
         return jsonify({'error': 'Anchor not found'}), 404
@@ -298,7 +298,7 @@ def rename_anchor(anchor_id):
       404:
         description: Anchor not found.
     """
-    from app_helper import update_alchemy_anchor_name
+    from database import update_alchemy_anchor_name
     payload = request.get_json() or {}
     name = (payload.get('name') or '').strip()
     if not name:
@@ -371,7 +371,7 @@ def list_radios():
       500:
         description: Database error.
     """
-    from app_helper import get_alchemy_radios
+    from database import get_alchemy_radios
     try:
         radios = get_alchemy_radios()
         return jsonify({'radios': [{
@@ -418,7 +418,7 @@ def create_radio():
       500:
         description: Database failure.
     """
-    from app_helper import create_alchemy_radio
+    from database import create_alchemy_radio
     payload = request.get_json() or {}
     anchor_id = payload.get('anchor_id')
     try:
@@ -471,7 +471,7 @@ def update_radio(radio_id):
       404:
         description: Radio not found.
     """
-    from app_helper import update_alchemy_radio
+    from database import update_alchemy_radio
     payload = request.get_json() or {}
     temperature, n_results, error = _parse_radio_settings(payload)
     if error:
@@ -502,7 +502,7 @@ def remove_radio(radio_id):
       404:
         description: Radio not found.
     """
-    from app_helper import delete_alchemy_radio
+    from database import delete_alchemy_radio
     ok = delete_alchemy_radio(radio_id)
     if not ok:
         return jsonify({'error': 'Radio not found'}), 404
@@ -588,7 +588,7 @@ def artist_projections_api():
       500:
         description: Failure to read cache.
     """
-    from app_helper import ARTIST_PROJECTION_CACHE
+    from database import ARTIST_PROJECTION_CACHE
 
     try:
         if not ARTIST_PROJECTION_CACHE:

app_analysis.py

@@ -1,14 +1,18 @@
 # app_analysis.py
-from flask import Blueprint, jsonify, request
+from flask import Blueprint, jsonify, request, render_template
 import uuid
 import logging
 
 # Import configuration from the main config.py
-from config import NUM_RECENT_ALBUMS, TOP_N_MOODS
+from config import NUM_RECENT_ALBUMS, TOP_N_MOODS, TASK_STATUS_PENDING
 
 # RQ import
 from rq import Retry
 
+# App helper functions
+from app_helper import rq_queue_high, save_task_status
+from database import clean_up_previous_main_tasks, get_active_main_task
+
 logger = logging.getLogger(__name__)
 
 # Create a Blueprint for analysis-related routes
@@ -29,7 +33,6 @@ def cleaning_page():
             schema:
               type: string
     """
-    from flask import render_template
     return render_template('cleaning.html', title = 'AudioMuse-AI - Database Cleaning', active='cleaning')
 
 @analysis_bp.route('/api/analysis/start', methods=['POST'])
@@ -80,9 +83,6 @@ def start_analysis_endpoint():
       500:
         description: Server error during task enqueue.
     """
-    # Local imports to prevent circular dependency at startup
-    from app_helper import rq_queue_high, clean_up_previous_main_tasks, save_task_status, TASK_STATUS_PENDING, get_active_main_task
-
     # Check for any existing active main task to prevent parallel batch runs.
     active_task = get_active_main_task()
     if active_task:
@@ -146,9 +146,6 @@ def start_cleaning_endpoint():
       500:
         description: Server error during task enqueue.
     """
-    # Local imports to prevent circular dependency at startup
-    from app_helper import rq_queue_high, clean_up_previous_main_tasks, save_task_status, TASK_STATUS_PENDING, get_active_main_task
-
     active_task = get_active_main_task()
     if active_task:
         return jsonify({

app_clustering.py

@@ -5,14 +5,32 @@
 import traceback
 
 # Import all necessary configuration variables
-from config import MAX_SONGS_PER_CLUSTER, SCORE_WEIGHT_DIVERSITY, SCORE_WEIGHT_SILHOUETTE, SCORE_WEIGHT_DAVIES_BOULDIN, SCORE_WEIGHT_CALINSKI_HARABASZ, SCORE_WEIGHT_PURITY, SCORE_WEIGHT_OTHER_FEATURE_DIVERSITY, SCORE_WEIGHT_OTHER_FEATURE_PURITY, MIN_SONGS_PER_GENRE_FOR_STRATIFICATION, STRATIFIED_SAMPLING_TARGET_PERCENTILE, CLUSTER_ALGORITHM, NUM_CLUSTERS_MIN, NUM_CLUSTERS_MAX, DBSCAN_EPS_MIN, DBSCAN_EPS_MAX, DBSCAN_MIN_SAMPLES_MIN, DBSCAN_MIN_SAMPLES_MAX, GMM_N_COMPONENTS_MIN, GMM_N_COMPONENTS_MAX, SPECTRAL_N_CLUSTERS_MIN, SPECTRAL_N_CLUSTERS_MAX, ENABLE_CLUSTERING_EMBEDDINGS, PCA_COMPONENTS_MIN, PCA_COMPONENTS_MAX, CLUSTERING_RUNS, TOP_N_MOODS, AI_MODEL_PROVIDER, OLLAMA_SERVER_URL, OLLAMA_MODEL_NAME, OPENAI_SERVER_URL, OPENAI_MODEL_NAME, OPENAI_API_KEY, GEMINI_API_KEY, GEMINI_MODEL_NAME, TOP_N_PLAYLISTS, MISTRAL_API_KEY, MISTRAL_MODEL_NAME
+from config import (
+    MAX_SONGS_PER_CLUSTER, SCORE_WEIGHT_DIVERSITY, SCORE_WEIGHT_SILHOUETTE,
+    SCORE_WEIGHT_DAVIES_BOULDIN, SCORE_WEIGHT_CALINSKI_HARABASZ,
+    SCORE_WEIGHT_PURITY, SCORE_WEIGHT_OTHER_FEATURE_DIVERSITY,
+    SCORE_WEIGHT_OTHER_FEATURE_PURITY, MIN_SONGS_PER_GENRE_FOR_STRATIFICATION,
+    STRATIFIED_SAMPLING_TARGET_PERCENTILE, CLUSTER_ALGORITHM, NUM_CLUSTERS_MIN,
+    NUM_CLUSTERS_MAX, DBSCAN_EPS_MIN, DBSCAN_EPS_MAX, DBSCAN_MIN_SAMPLES_MIN,
+    DBSCAN_MIN_SAMPLES_MAX, GMM_N_COMPONENTS_MIN, GMM_N_COMPONENTS_MAX,
+    SPECTRAL_N_CLUSTERS_MIN, SPECTRAL_N_CLUSTERS_MAX, ENABLE_CLUSTERING_EMBEDDINGS,
+    PCA_COMPONENTS_MIN, PCA_COMPONENTS_MAX, CLUSTERING_RUNS, TOP_N_MOODS,
+    AI_MODEL_PROVIDER, OLLAMA_SERVER_URL, OLLAMA_MODEL_NAME, OPENAI_SERVER_URL,
+    OPENAI_MODEL_NAME, OPENAI_API_KEY, GEMINI_API_KEY, GEMINI_MODEL_NAME,
+    TOP_N_PLAYLISTS, MISTRAL_API_KEY, MISTRAL_MODEL_NAME,
+    TASK_STATUS_PENDING, TASK_STATUS_FAILURE,
+)
 
 # RQ import
 from rq import Retry
 
 from error import error_manager
 from error.error_dictionary import ERR_CLUSTERING_FAILED
 
+# App helper functions
+from app_helper import rq_queue_high, save_task_status
+from database import clean_up_previous_main_tasks, get_active_main_task
+
 
 logger = logging.getLogger(__name__)
 
@@ -22,7 +40,6 @@
 def clustering_task_failure_handler(job, connection, type, value, tb):
     """A failure handler for the main clustering task, executed by the worker."""
     from flask_app import app
-    from app_helper import save_task_status, TASK_STATUS_FAILURE
     with app.app_context():
         task_id = getattr(job, 'id', None) or getattr(job, 'get_id', lambda: None)()
 
@@ -245,10 +262,6 @@ def start_clustering_endpoint():
                         status:
                             type: string
     """
-    # Local imports to prevent circular dependency at startup
-    from app_helper import rq_queue_high, get_active_main_task
-    from app_helper import clean_up_previous_main_tasks, save_task_status, TASK_STATUS_PENDING
-
     # Check for any existing active main task to prevent parallel batch runs
     active_task = get_active_main_task()
     if active_task:

app_cron.py

@@ -1,8 +1,8 @@
 from flask import Blueprint, render_template, jsonify, request
 from psycopg2.extras import DictCursor
-from database import get_db
+from database import get_db, save_task_status
 from taskqueue import rq_queue_high
-from app_helper import save_task_status, TASK_STATUS_PENDING
+from config import TASK_STATUS_PENDING
 import uuid, time, logging
 from config import (
     TOP_N_MOODS,
@@ -240,8 +240,6 @@ def run_due_cron_jobs():
                         "pca_components_max": int(PCA_COMPONENTS_MAX),
                         "num_clustering_runs": int(CLUSTERING_RUNS),
                         "max_songs_per_cluster_val": int(MAX_SONGS_PER_CLUSTER),
-                        "gmm_n_components_min": int(GMM_N_COMPONENTS_MIN),
-                        "gmm_n_components_max": int(GMM_N_COMPONENTS_MAX),
                         "top_n_playlists_param": int(TOP_N_PLAYLISTS),
                         "min_songs_per_genre_for_stratification_param": int(MIN_SONGS_PER_GENRE_FOR_STRATIFICATION),
                         "stratified_sampling_target_percentile_param": int(STRATIFIED_SAMPLING_TARGET_PERCENTILE),