Gemini, Copilot, Sonarqube review and fix

Author: NeptuneHub

app.py

@@ -761,7 +761,6 @@ def listen_for_index_reloads():
 # --- Blueprint Registration ---
 # Standard Flask factory pattern: blueprint imports are inside
 # this function so the eager import graph stays flat.
-from app_cron import run_due_cron_jobs
 
 
 def _register_blueprints(flask_app):
@@ -905,6 +904,7 @@ def _start_map_init_background():
   def _cron_manager_loop():
     try:
       from time import sleep
+      from app_cron import run_due_cron_jobs
       while True:
         try:
           with app.app_context():

app_cron.py

@@ -240,8 +240,6 @@ def run_due_cron_jobs():
                         "pca_components_max": int(PCA_COMPONENTS_MAX),
                         "num_clustering_runs": int(CLUSTERING_RUNS),
                         "max_songs_per_cluster_val": int(MAX_SONGS_PER_CLUSTER),
-                        "gmm_n_components_min": int(GMM_N_COMPONENTS_MIN),
-                        "gmm_n_components_max": int(GMM_N_COMPONENTS_MAX),
                         "top_n_playlists_param": int(TOP_N_PLAYLISTS),
                         "min_songs_per_genre_for_stratification_param": int(MIN_SONGS_PER_GENRE_FOR_STRATIFICATION),
                         "stratified_sampling_target_percentile_param": int(STRATIFIED_SAMPLING_TARGET_PERCENTILE),

app_logging.py

@@ -11,12 +11,17 @@
 ``logger.info(...)`` from task modules fell through to Python's ``lastResort``
 handler — silently dropping INFO-level output during long-running jobs.
 
-Emoji safety: the ``EmojiStrippingFilter`` removes emoji and other non-Latin-1
-symbols from every log record before it reaches a handler.  This prevents
-``UnicodeEncodeError`` / ``UnicodeDecodeError`` crashes on Windows when stdout
-is a pipe (PyInstaller native build) or when the console code-page cannot
-represent the character.  HTML templates and web-UI progress messages are
-unaffected — only the Python ``logging`` pipeline is sanitised.
+Record sanitization: the ``LogSanitizingFilter`` cleans every log record before
+it reaches a handler. It removes emoji / non-Latin-1 symbols (which raise
+``UnicodeEncodeError`` / ``UnicodeDecodeError`` on Windows when stdout is a pipe
+or the console code-page cannot represent the character) and neutralizes CR/LF
+and other control characters so an attacker-controlled value embedded in a
+message cannot forge or split log lines (CWE-117, log injection). This is the
+single, centralized place where log-message sanitization happens — call sites
+log the raw value and the filter cleans it. HTML templates and web-UI progress
+messages are unaffected — only the Python ``logging`` pipeline is sanitised, and
+the traceback ``logger.exception`` appends is left intact (the formatter renders
+it after filtering, so the full error is always visible in the log).
 """
 
 import logging
@@ -26,55 +31,69 @@
 LOG_DATEFMT = "%d-%m-%Y %H-%M-%S"
 
 # ---------------------------------------------------------------------------
-# Emoji / symbol stripping for console-safe logging
+# Console-safe + injection-safe log record sanitization
 # ---------------------------------------------------------------------------
 # Ranges cover all common emoji blocks plus Dingbats, Misc Symbols,
 # Geometric Shapes, Supplemental Symbols, and variation selectors.
-# Characters within Latin-1 (U+0000–U+00FF) are *not* stripped, so
-# European accented letters (e.g. é, ñ, ü) pass through unchanged.
+# Characters within Latin-1 (U+0000-U+00FF) are *not* stripped, so
+# European accented letters pass through unchanged.
 _EMOJI_RE = re.compile(
     "[\U0001F300-\U0001F9FF"   # Misc Symbols, Emoticons, Transport, Supplemental
     "\U0001FA00-\U0001FAFF"    # Chess Symbols, Symbols Extended-A
-    "\U00002190-\U000027BF"    # Arrows (→ ← ↑ ↓ ↔), Misc Technical, Dingbats (✓ ✗ ✕ ★ ☆ ♯ ♭ etc.)
-    "\U000025A0-\U000025FF"    # Geometric Shapes (● ○ ■ □ ◆ ◇ ▲ ▼ etc.)
+    "\U00002190-\U000027BF"    # Arrows, Misc Technical, Dingbats
+    "\U000025A0-\U000025FF"    # Geometric Shapes
     "\U00002B00-\U00002BFF"    # Misc Symbols & Arrows
     "\U0001F000-\U0001F02F"    # Mahjong Tiles
     "\U0001F0A0-\U0001F0FF"    # Playing Cards
-    "\uFE0F\u200D"             # Variation Selector-16, Zero-Width Joiner
+    "\\uFE0F\\u200D"           # Variation Selector-16, Zero-Width Joiner
     "]+"
 )
 
+# C0 control codes plus DEL, EXCEPT tab (0x09). Includes LF (0x0A) and CR (0x0D):
+# replacing these with a space prevents a value with embedded newlines from
+# forging additional log lines.
+_CONTROL_RE = re.compile(r"[\x00-\x08\x0A-\x1F\x7F]")
 
-def _strip_emoji(text: str) -> str:
-    """Remove emoji and symbol characters from *text*, returning a plain string."""
+
+def _sanitize_log_text(text: str) -> str:
+    """Make *text* safe for a single console log line.
+
+    Strips emoji/symbol characters (Windows code-page safety) and replaces CR/LF
+    and other C0 control codes with a space so an attacker-controlled value
+    cannot forge or split log lines (CWE-117). Tabs are preserved.
+    """
     if not isinstance(text, str):
         return text
     cleaned = _EMOJI_RE.sub("", text)
-    # Collapse multiple spaces that may result from removing a symbol
+    cleaned = _CONTROL_RE.sub(" ", cleaned)
+    # Collapse runs of spaces left behind by removed symbols / control codes.
     return re.sub(r" {2,}", " ", cleaned).strip()
 
 
-class EmojiStrippingFilter(logging.Filter):
-    """Logging filter that strips emoji/symbols from ``record.msg`` and ``record.args``.
+class LogSanitizingFilter(logging.Filter):
+    """Logging filter that sanitizes ``record.msg`` and ``record.args``.
 
-    Attach this to the root logger's handlers so every log record is sanitised
-    before it reaches a ``StreamHandler`` (console / pipe).  File-based handlers
+    Attach this to the root logger's handlers so every log record is cleaned
+    before it reaches a ``StreamHandler`` (console / pipe): emoji/symbols are
+    removed and CR/LF + control characters are neutralised. File-based handlers
     with ``propagate=False`` (e.g. the Windows supervisor's own log) are not
-    affected.
+    affected. The exception traceback added by ``logger.exception`` lives in
+    ``record.exc_info`` and is rendered by the formatter after this filter runs,
+    so it is never altered here — the full error always reaches the log.
     """
 
     def filter(self, record):
         if isinstance(record.msg, str):
-            record.msg = _strip_emoji(record.msg)
+            record.msg = _sanitize_log_text(record.msg)
         if record.args:
             if isinstance(record.args, dict):
                 record.args = {
-                    k: _strip_emoji(v) if isinstance(v, str) else v
+                    k: _sanitize_log_text(v) if isinstance(v, str) else v
                     for k, v in record.args.items()
                 }
             elif isinstance(record.args, (list, tuple)):
                 record.args = tuple(
-                    _strip_emoji(a) if isinstance(a, str) else a
+                    _sanitize_log_text(a) if isinstance(a, str) else a
                     for a in record.args
                 )
         return True
@@ -83,10 +102,11 @@ def filter(self, record):
 def configure_logging(level: int = logging.INFO) -> None:
     """Install the project-wide root logger format. Idempotent.
 
-    An ``EmojiStrippingFilter`` is attached to every handler on the root logger,
-    making all console / pipe output safe on Windows regardless of code-page.
+    A ``LogSanitizingFilter`` is attached to every handler on the root logger,
+    making all console / pipe output safe on Windows regardless of code-page and
+    neutralising log-injection attempts from untrusted message data.
     """
     logging.basicConfig(level=level, format=LOG_FORMAT, datefmt=LOG_DATEFMT)
     for handler in logging.root.handlers:
-        if not any(isinstance(f, EmojiStrippingFilter) for f in handler.filters):
-            handler.addFilter(EmojiStrippingFilter())
+        if not any(isinstance(f, LogSanitizingFilter) for f in handler.filters):
+            handler.addFilter(LogSanitizingFilter())

database.py

@@ -304,13 +304,13 @@ def save_task_status(task_id, task_type, status=TASK_STATUS_PENDING, parent_task
                            END
         """, (task_id, parent_task_id, task_type, sub_type_identifier, status, progress, details_json, current_unix_time, status, current_unix_time, current_unix_time, current_unix_time))
         db.commit()
-    except psycopg2.Error as e:
-        logger.error(f"DB Error saving task status for {task_id}: {e}")
+    except psycopg2.Error:
+        logger.exception(f"DB Error saving task status for {task_id}")
         try:
             db.rollback()
             logger.info(f"DB transaction rolled back for task status update of {task_id}.")
-        except psycopg2.Error as rb_e:
-            logger.error(f"DB Error during rollback for task status {task_id}: {rb_e}")
+        except psycopg2.Error:
+            logger.exception(f"DB Error during rollback for task status {task_id}")
     finally:
         cur.close()
 
@@ -394,8 +394,8 @@ def get_score_data_by_ids(item_ids_list):
     try:
         cur.execute(query, (tuple(item_ids_list),))
         rows = cur.fetchall()
-    except Exception as e:
-        logger.error(f"Error fetching score data by IDs: {e}")
+    except Exception:
+        logger.exception("Error fetching score data by IDs")
         rows = []
     finally:
         cur.close()
@@ -491,8 +491,8 @@ def load_map_projection(index_name, force_reload=False):
         MAP_PROJECTION_CACHE = {'index_name': index_name, 'id_map': id_map, 'projection': proj}
         logger.info(f"Map projection '{index_name}' with {len(id_map)} items loaded successfully into memory.")
         return id_map, proj
-    except Exception as e:
-        logger.error(f"Failed to load map projection: {e}", exc_info=True)
+    except Exception:
+        logger.exception("Failed to load map projection")
         return None, None
     finally:
         cur.close()
@@ -611,9 +611,9 @@ def _parse_year_from_date(year_value):
             """, (item_id, psycopg2.Binary(embedding_blob)))
 
         conn.commit()
-    except Exception as e:
+    except Exception:
         conn.rollback()
-        logger.error("Error saving track analysis and embedding for %s: %s", item_id, e)
+        logger.exception("Error saving track analysis and embedding for %s", item_id)
         raise
     finally:
         cur.close()
@@ -633,9 +633,9 @@ def save_clap_embedding(item_id, clap_embedding_vector):
             ON CONFLICT (item_id) DO UPDATE SET embedding = EXCLUDED.embedding
         """, (item_id, psycopg2.Binary(embedding_blob)))
         conn.commit()
-    except Exception as e:
+    except Exception:
         conn.rollback()
-        logger.error(f"Error saving CLAP embedding for {item_id}: {e}")
+        logger.exception(f"Error saving CLAP embedding for {item_id}")
         raise
     finally:
         cur.close()
@@ -655,8 +655,8 @@ def get_clap_embedding(item_id):
         if row and row[0]:
             return np.frombuffer(row[0], dtype=np.float32)
         return None
-    except Exception as e:
-        logger.error(f"Error loading CLAP embedding for {item_id}: {e}")
+    except Exception:
+        logger.exception(f"Error loading CLAP embedding for {item_id}")
         return None
     finally:
         cur.close()
@@ -686,9 +686,9 @@ def save_lyrics_embedding(item_id, lyrics_embedding_vector, axis_vector=None):
         """, (item_id, psycopg2.Binary(embedding_blob),
               psycopg2.Binary(axis_blob) if axis_blob is not None else None))
         conn.commit()
-    except Exception as e:
+    except Exception:
         conn.rollback()
-        logger.error(f"Error saving lyrics embedding for {item_id}: {e}")
+        logger.exception(f"Error saving lyrics embedding for {item_id}")
         raise
     finally:
         cur.close()
@@ -1121,9 +1121,9 @@ def clean_up_previous_main_tasks():
             logger.info(f"Archived {archived_count} previous main tasks and deleted {deleted_children_count} child tasks.")
         else:
             logger.info("No previous main tasks found to clean up.")
-    except Exception as e_main_clean:
+    except Exception:
         db.rollback()
-        logger.error(f"Error during the main task cleanup process: {e_main_clean}")
+        logger.exception("Error during the main task cleanup process")
     finally:
         cur.close()
 
@@ -1192,9 +1192,9 @@ def save_alchemy_anchor(name, centroid):
         row = cur.fetchone()
         conn.commit()
         return dict(row) if row else None
-    except Exception as e:
+    except Exception:
         conn.rollback()
-        logger.error(f"Failed to save alchemy anchor '{name}': {e}")
+        logger.exception(f"Failed to save alchemy anchor '{name}'")
         return None
     finally:
         cur.close()
@@ -1206,8 +1206,8 @@ def get_alchemy_anchors():
         cur.execute("SELECT id, name, created_at FROM alchemy_anchors ORDER BY created_at DESC")
         rows = cur.fetchall()
         return [dict(row) for row in rows]
-    except Exception as e:
-        logger.error(f"Failed to load alchemy anchors: {e}")
+    except Exception:
+        logger.exception("Failed to load alchemy anchors")
         return []
     finally:
         cur.close()
@@ -1219,9 +1219,9 @@ def delete_alchemy_anchor(anchor_id):
         cur.execute("DELETE FROM alchemy_anchors WHERE id = %s", (anchor_id,))
         conn.commit()
         return cur.rowcount > 0
-    except Exception as e:
+    except Exception:
         conn.rollback()
-        logger.error(f"Failed to delete alchemy anchor id={anchor_id}: {e}")
+        logger.exception(f"Failed to delete alchemy anchor id={anchor_id}")
         return False
     finally:
         cur.close()
@@ -1241,8 +1241,8 @@ def get_alchemy_anchor_by_id(anchor_id):
             except Exception:
                 anchor['centroid'] = None
         return anchor
-    except Exception as e:
-        logger.error(f"Failed to fetch alchemy anchor id={anchor_id}: {e}")
+    except Exception:
+        logger.exception(f"Failed to fetch alchemy anchor id={anchor_id}")
         return None
     finally:
         cur.close()
@@ -1262,9 +1262,9 @@ def update_alchemy_anchor_name(anchor_id, name):
         if not row:
             return None
         return dict(row)
-    except Exception as e:
+    except Exception:
         conn.rollback()
-        logger.error(f"Failed to rename alchemy anchor id={anchor_id}: {e}")
+        logger.exception(f"Failed to rename alchemy anchor id={anchor_id}")
         return None
     finally:
         cur.close()
@@ -1379,9 +1379,9 @@ def save_map_projection(index_name, id_map, projection_array):
             logger.info(f"Saved map projection '{index_name}' to DB: {len(blob)} bytes, ids={id_count}")
         except Exception:
             logger.debug("Saved map projection but failed to compute size/id_count for log.")
-    except Exception as e:
+    except Exception:
         conn.rollback()
-        logger.error(f"Failed to save map projection: {e}")
+        logger.exception("Failed to save map projection")
         raise
 
 def load_artist_projection(index_name='artist_map', force_reload=False):
@@ -1413,8 +1413,8 @@ def load_artist_projection(index_name='artist_map', force_reload=False):
         ARTIST_PROJECTION_CACHE = {'index_name': index_name, 'component_map': component_map, 'projection': proj}
         logger.info(f"Artist projection '{index_name}' with {len(component_map)} components loaded successfully into memory.")
         return component_map, proj
-    except Exception as e:
-        logger.error(f"Failed to load artist projection: {e}", exc_info=True)
+    except Exception:
+        logger.exception("Failed to load artist projection")
         return None, None
     finally:
         cur.close()
@@ -1432,9 +1432,9 @@ def save_artist_projection(index_name, component_map, projections):
         cur.execute("INSERT INTO artist_component_projection (index_name, projection_data, artist_component_map_json) VALUES (%s, %s, %s) ON CONFLICT (index_name) DO UPDATE SET projection_data = EXCLUDED.projection_data, artist_component_map_json = EXCLUDED.artist_component_map_json, created_at = CURRENT_TIMESTAMP", (index_name, proj_blob, component_map_json))
         conn.commit()
         logger.info(f"Saved artist projection '{index_name}' with {len(component_map)} components to database.")
-    except Exception as e:
+    except Exception:
         conn.rollback()
-        logger.error(f"Failed to save artist projection: {e}", exc_info=True)
+        logger.exception("Failed to save artist projection")
     finally:
         cur.close()
 
@@ -1454,8 +1454,8 @@ def update_playlist_table(playlists): # Removed db_path
             for item_id, title, author in cluster:
                 cur.execute("INSERT INTO playlist (playlist_name, item_id, title, author) VALUES (%s, %s, %s, %s) ON CONFLICT (playlist_name, item_id) DO NOTHING", (name, item_id, title, author))
         conn.commit()
-    except Exception as e:
+    except Exception:
         conn.rollback()
-        logger.error("Error updating playlist table: %s", e)
+        logger.exception("Error updating playlist table")
     finally:
         cur.close()