diff --git a/backend/Cargo.lock b/backend/Cargo.lock index 81817d60..8b5273de 100644 --- a/backend/Cargo.lock +++ b/backend/Cargo.lock @@ -2837,7 +2837,7 @@ checksum = "ae960838283323069879657ca3de837e9f7bbb4c7bf6ea7f1b290d5e9476d2e0" [[package]] name = "mediafusion-api" -version = "6.0.0-beta.12" +version = "6.0.0-beta.13" dependencies = [ "ab_glyph", "aes 0.9.0", diff --git a/backend/Cargo.toml b/backend/Cargo.toml index 14b7d1c5..73ed6e8d 100644 --- a/backend/Cargo.toml +++ b/backend/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "mediafusion-api" -version = "6.0.0-beta.13" +version = "6.0.0-beta.14" edition = "2021" [lib] diff --git a/backend/src/config.rs b/backend/src/config.rs index e46aa7d1..9d28b478 100644 --- a/backend/src/config.rs +++ b/backend/src/config.rs @@ -35,8 +35,22 @@ pub struct AppConfig { /// Format: "U-{uuid}" or "D-{encrypted}" pub mediafusion_secret_str: Option, /// Mirrors Python's prowlarr_live_title_search (default: true). - /// When false, prowlarr is excluded from live title searches. + /// When false, title queries are omitted from live Prowlarr/Jackett searches. pub prowlarr_live_title_search: bool, + /// Mirrors Python's jackett_live_title_search (default: true). + pub jackett_live_title_search: bool, + + // ── Background search ───────────────────────────────────────────────────── + /// When false, stream requests do not enqueue items for background re-scraping. + pub background_search_enabled: bool, + /// Max results to process per indexer during background search (default: 50). + pub background_max_process: usize, + /// Overall time budget (seconds) for background Prowlarr/Jackett scrapes. + pub background_max_process_time: u64, + /// Per-request HTTP timeout (seconds) for background Prowlarr/Jackett calls. + pub background_query_timeout: u64, + /// Minimum hours between background re-scrapes for the same queued item. + pub background_search_interval_hours: i64, // ── Instance mode ──────────────────────────────────────────────────────── /// When true the instance is fully public: no api_password or X-API-Key @@ -47,6 +61,15 @@ pub struct AppConfig { // ── Torznab / auth ─────────────────────────────────────────────────────── /// Optional API password for Torznab and private-instance validation. pub api_password: Option, + /// Expose the Prometheus /api/v1/metrics endpoint (default: false). + pub enable_prometheus_metrics: bool, + /// Bearer token required to scrape /api/v1/metrics. + /// When set, the header `Authorization: Bearer ` is required on every + /// scrape request — even on public instances. Configure the matching + /// `bearer_token` in your Prometheus scrape_configs. + /// When unset and the endpoint is enabled, it is open to anyone who can + /// reach the endpoint (rely on network-level controls in that case). + pub metrics_api_key: Option, /// Enable the Torznab feed endpoint (default: true). pub enable_torznab_api: bool, @@ -427,9 +450,24 @@ impl AppConfig { mediafusion_secret_str: env("MEDIAFUSION_SECRET_STR").ok().filter(|s| !s.is_empty()), prowlarr_live_title_search: env("PROWLARR_LIVE_TITLE_SEARCH") .ok().and_then(|v| v.parse().ok()).unwrap_or(true), + jackett_live_title_search: env("JACKETT_LIVE_TITLE_SEARCH") + .ok().and_then(|v| v.parse().ok()).unwrap_or(true), + background_search_enabled: env("BACKGROUND_SEARCH_ENABLED") + .ok().and_then(|v| v.parse().ok()).unwrap_or(true), + background_max_process: env("BACKGROUND_MAX_PROCESS") + .ok().and_then(|v| v.parse().ok()).unwrap_or(50), + background_max_process_time: env("BACKGROUND_MAX_PROCESS_TIME") + .ok().and_then(|v| v.parse().ok()).unwrap_or(120), + background_query_timeout: env("BACKGROUND_QUERY_TIMEOUT") + .ok().and_then(|v| v.parse().ok()).unwrap_or(30), + background_search_interval_hours: env("BACKGROUND_SEARCH_INTERVAL_HOURS") + .ok().and_then(|v| v.parse().ok()).unwrap_or(72), is_public_instance: env("IS_PUBLIC_INSTANCE") .ok().and_then(|v| v.parse().ok()).unwrap_or(false), api_password: env("API_PASSWORD").ok().filter(|s| !s.is_empty()), + enable_prometheus_metrics: env("ENABLE_PROMETHEUS_METRICS") + .ok().and_then(|v| v.parse().ok()).unwrap_or(false), + metrics_api_key: env("PROMETHEUS_METRICS_TOKEN").ok().filter(|s| !s.is_empty()), enable_torznab_api: env("ENABLE_TORZNAB_API") .ok().and_then(|v| v.parse().ok()).unwrap_or(true), is_scrap_from_prowlarr: env("IS_SCRAP_FROM_PROWLARR") diff --git a/backend/src/db/catalog.rs b/backend/src/db/catalog.rs index b0d5e826..80ace2af 100644 --- a/backend/src/db/catalog.rs +++ b/backend/src/db/catalog.rs @@ -4,6 +4,7 @@ use tracing::warn; use super::types::{MediaId, MediaType, UserId}; const LIMIT: i64 = 100; +const WATCHLIST_LIMIT: i64 = 25; #[derive(sqlx::FromRow, Debug)] pub struct CatalogRow { @@ -175,6 +176,76 @@ async fn get_library_items( }) } +/// Watchlist catalog rows: media linked to any of the given torrent info_hashes. +pub async fn get_watchlist_items( + pool: &PgPool, + media_type: &str, + info_hashes: &[String], + skip: i64, + nudity_excludes: &[String], + cert_excludes: &[String], + sort: &str, + sort_dir: &str, +) -> Vec { + if info_hashes.is_empty() { + return vec![]; + } + + let Some(mt) = MediaType::from_wire(media_type) else { + return vec![]; + }; + + let ord = order_clause(sort, sort_dir); + let sql = format!( + r#" + SELECT + m.id AS media_id, + m.type AS media_type, + m.title, + m.year, + m.description, + mei.external_id AS imdb_id, + mi.url AS poster_url + FROM media m + JOIN stream_media_link sml ON sml.media_id = m.id + JOIN stream s ON s.id = sml.stream_id + JOIN torrent_stream ts ON ts.stream_id = s.id + LEFT JOIN media_external_id mei ON mei.media_id = m.id AND mei.provider = 'imdb' + LEFT JOIN LATERAL ( + SELECT url FROM media_image + WHERE media_id = m.id AND image_type = 'poster' AND is_primary = true + LIMIT 1 + ) mi ON true + WHERE m.type = $1 + AND m.total_streams > 0 + AND NOT m.is_blocked + AND lower(ts.info_hash) = ANY($2) + AND (cardinality($3::text[]) IS NULL OR m.nudity_status::text <> ALL($3)) + AND (cardinality($4::text[]) IS NULL OR NOT EXISTS ( + SELECT 1 FROM media_parental_certificate_link mpcl + JOIN parental_certificate pc ON pc.id = mpcl.certificate_id + WHERE mpcl.media_id = m.id AND pc.name = ANY($4) + )) + GROUP BY m.id, m.type, m.title, m.year, m.description, mei.external_id, mi.url + ORDER BY {ord} + LIMIT {WATCHLIST_LIMIT} OFFSET $5 + "# + ); + + sqlx::query_as::<_, CatalogRow>(&sql) + .bind(mt) // $1 + .bind(info_hashes) // $2 + .bind(nudity_excludes) // $3 + .bind(cert_excludes) // $4 + .bind(skip) // $5 + .fetch_all(pool) + .await + .unwrap_or_else(|e| { + warn!("watchlist catalog query [type={media_type}]: {e}"); + vec![] + }) +} + pub async fn search_metadata( pool: &PgPool, media_type: &str, diff --git a/backend/src/db/genres.rs b/backend/src/db/genres.rs index a4c498ab..d419df31 100644 --- a/backend/src/db/genres.rs +++ b/backend/src/db/genres.rs @@ -1,52 +1,87 @@ -use std::collections::HashMap; +use std::collections::{HashMap, HashSet}; use sqlx::PgPool; +use tokio::try_join; use tracing::warn; use super::types::MediaType; -const ADULT_GENRES: &[&str] = &[ - "adult", - "18+", - "xxx", - "erotic", - "erotica", - "pornography", - "porn", -]; - #[derive(sqlx::FromRow)] -struct GenreRow { +struct MediaTypeRow { + id: i32, media_type: MediaType, - genre_name: String, } -pub async fn get_all_genres_by_type(pool: &PgPool) -> HashMap> { - let rows: Vec = sqlx::query_as( - r#" - SELECT DISTINCT m.type AS media_type, g.name AS genre_name - FROM genre g - JOIN media_genre_link mgl ON mgl.genre_id = g.id - JOIN media m ON m.id = mgl.media_id - WHERE lower(g.name) <> ALL($1) - AND m.total_streams > 0 - AND NOT m.is_blocked - ORDER BY m.type, g.name - "#, +#[derive(sqlx::FromRow)] +struct GenreNameRow { + id: i32, + name: String, +} + +#[derive(sqlx::FromRow)] +struct GenreLinkRow { + genre_id: i32, + media_id: i32, +} + +fn is_manifest_media_type(media_type: MediaType) -> bool { + matches!( + media_type, + MediaType::Movie | MediaType::Series | MediaType::Tv ) - .bind(ADULT_GENRES) - .fetch_all(pool) - .await - .unwrap_or_else(|e| { - warn!("genres query: {e}"); - vec![] - }); - - let mut map: HashMap> = HashMap::new(); - for row in rows { - map.entry(row.media_type.as_wire().to_string()) +} + +pub async fn get_all_genres_by_type(pool: &PgPool) -> HashMap> { + let media_future = + sqlx::query_as::<_, MediaTypeRow>("SELECT id, type AS media_type FROM media") + .fetch_all(pool); + let genre_future = + sqlx::query_as::<_, GenreNameRow>("SELECT id, name FROM genre").fetch_all(pool); + let links_future = + sqlx::query_as::<_, GenreLinkRow>("SELECT genre_id, media_id FROM media_genre_link") + .fetch_all(pool); + + let (media_rows, genre_rows, link_rows) = + match try_join!(media_future, genre_future, links_future) { + Ok(rows) => rows, + Err(e) => { + warn!("genres query: {e}"); + return HashMap::new(); + } + }; + + let media_types: HashMap = media_rows + .into_iter() + .map(|row| (row.id, row.media_type)) + .collect(); + let genre_names: HashMap = genre_rows + .into_iter() + .map(|row| (row.id, row.name)) + .collect(); + + let mut by_type: HashMap> = HashMap::new(); + for link in link_rows { + let Some(media_type) = media_types.get(&link.media_id) else { + continue; + }; + if !is_manifest_media_type(*media_type) { + continue; + } + let Some(genre_name) = genre_names.get(&link.genre_id) else { + continue; + }; + by_type + .entry(media_type.as_wire().to_string()) .or_default() - .push(row.genre_name); + .insert(genre_name.clone()); } - map + + by_type + .into_iter() + .map(|(media_type, genres)| { + let mut list: Vec = genres.into_iter().collect(); + list.sort_unstable(); + (media_type, list) + }) + .collect() } diff --git a/backend/src/db/meta.rs b/backend/src/db/meta.rs index ef441735..6bf9266b 100644 --- a/backend/src/db/meta.rs +++ b/backend/src/db/meta.rs @@ -67,12 +67,14 @@ pub async fn get_media_meta( let result = match parse_meta_id(meta_id) { MetaIdKind::Internal(id) => { - sqlx::query_as!( - MediaMetaRow, + // Use non-macro query_as so LEFT JOIN LATERAL columns (poster_url, background_url, + // imdb_rating) are decoded as nullable via the struct's Option field types. + // The query_as! offline cache incorrectly infers those columns as non-nullable. + sqlx::query_as::<_, MediaMetaRow>( r#" SELECT - m.id AS "media_id: MediaId", - m.type AS "media_type: MediaType", + m.id AS media_id, + m.type AS media_type, m.title, m.year, EXTRACT(YEAR FROM m.end_date)::int AS end_year, @@ -108,19 +110,18 @@ pub async fn get_media_meta( WHERE m.id = $1 AND m.type = $2 LIMIT 1 "#, - id as MediaId, - media_type as MediaType, ) + .bind(id) + .bind(media_type) .fetch_optional(pool) .await } MetaIdKind::External(ext_id) => { - sqlx::query_as!( - MediaMetaRow, + sqlx::query_as::<_, MediaMetaRow>( r#" SELECT - m.id AS "media_id: MediaId", - m.type AS "media_type: MediaType", + m.id AS media_id, + m.type AS media_type, m.title, m.year, EXTRACT(YEAR FROM m.end_date)::int AS end_year, @@ -158,9 +159,9 @@ pub async fn get_media_meta( WHERE m.type = $2 LIMIT 1 "#, - ext_id, - media_type as MediaType, ) + .bind(ext_id) + .bind(media_type) .fetch_optional(pool) .await } @@ -214,8 +215,9 @@ pub async fn get_cast(pool: &PgPool, media_id: MediaId) -> Vec { } pub async fn get_episodes(pool: &PgPool, media_id: MediaId) -> Vec { - sqlx::query_as!( - EpisodeRow, + // Use non-macro query_as so LEFT JOIN LATERAL (thumbnail_url) and LEFT JOIN + // (media_id) columns decode as nullable via the struct's Option field types. + sqlx::query_as::<_, EpisodeRow>( r#" SELECT s.season_number, @@ -224,7 +226,7 @@ pub async fn get_episodes(pool: &PgPool, media_id: MediaId) -> Vec { e.overview, e.air_date, ei.url AS thumbnail_url, - fml.media_id AS "media_id: Option" + fml.media_id AS media_id FROM series_metadata sm JOIN season s ON s.series_id = sm.id JOIN episode e ON e.season_id = s.id @@ -240,8 +242,8 @@ pub async fn get_episodes(pool: &PgPool, media_id: MediaId) -> Vec { WHERE sm.media_id = $1 ORDER BY s.season_number, e.episode_number "#, - media_id as MediaId, ) + .bind(media_id) .fetch_all(pool) .await .unwrap_or_else(|e| { diff --git a/backend/src/db/streams.rs b/backend/src/db/streams.rs index 677dcb46..9114160a 100644 --- a/backend/src/db/streams.rs +++ b/backend/src/db/streams.rs @@ -620,7 +620,7 @@ pub async fn fetch_stream_playback_info( LEFT JOIN torrent_tracker_link ttl ON ttl.torrent_id = ts.id LEFT JOIN tracker t ON t.id = ttl.tracker_id WHERE ts.info_hash = $1 - GROUP BY st.id, st.name, ts.total_size + GROUP BY st.id, st.name, ts.total_size, ts.torrent_file "#, ) .bind(info_hash) diff --git a/backend/src/exception_tracker.rs b/backend/src/exception_tracker.rs index c294a74e..078697fc 100644 --- a/backend/src/exception_tracker.rs +++ b/backend/src/exception_tracker.rs @@ -109,10 +109,38 @@ impl tracing::field::Visit for MessageVisitor { // ─── Fingerprint ────────────────────────────────────────────────────────────── +/// Normalize a message so that variable parts (numbers, hex hashes) collapse to +/// a placeholder, allowing similar errors from the same site to share a bucket. +/// +/// Examples: +/// "realdebrid torrents page 3: HTTP 401" → "realdebrid torrents page N: HTTP NNN" +/// "meta query [tt12345]: decode error" → "meta query [ttNNNNN]: decode error" +fn normalize_message(msg: &str) -> String { + let mut out = String::with_capacity(msg.len().min(80)); + let mut chars = msg.chars().peekable(); + while out.len() < 80 { + match chars.next() { + None => break, + Some(c) if c.is_ascii_digit() => { + out.push('N'); + // consume the rest of the digit run + while chars.peek().map(|c| c.is_ascii_digit()).unwrap_or(false) { + chars.next(); + } + } + Some(c) => out.push(c), + } + } + out +} + fn fingerprint(file: Option<&str>, line: Option, msg: &str) -> String { + // Include a normalized message prefix so errors at the same source location + // but with different messages (different error classes) get distinct fingerprints. + let normalized = normalize_message(msg); let raw = match (file, line) { - (Some(f), Some(l)) => format!("{f}:{l}"), - _ => msg.chars().take(100).collect::(), + (Some(f), Some(l)) => format!("{f}:{l}:{normalized}"), + _ => normalized, }; let hash = Sha256::digest(raw.as_bytes()); hash.iter().take(8).map(|b| format!("{b:02x}")).collect() @@ -173,11 +201,15 @@ async fn store_event(redis: &fred::clients::Client, ev: ExcEvent, ttl: i64, max_ .and_then(|c| c.parse().ok()) .unwrap_or(1) + 1; + // Keep `traceback` in sync with the stored message so the detail view is + // consistent. The message shown in the list is always the latest occurrence. + let traceback = format!("{} at {source}\n{}", ev.level, ev.message); let mut fields: HashMap = HashMap::new(); fields.insert("count".into(), count.to_string()); fields.insert("last_seen".into(), now); fields.insert("message".into(), ev.message); fields.insert("source".into(), source); + fields.insert("traceback".into(), traceback); let _ = redis.hset::<(), _, _>(&key, fields).await; } diff --git a/backend/src/jobs/handlers/background_search.rs b/backend/src/jobs/handlers/background_search.rs index 4af70244..d93297e1 100644 --- a/backend/src/jobs/handlers/background_search.rs +++ b/backend/src/jobs/handlers/background_search.rs @@ -1,5 +1,6 @@ use std::collections::HashMap; -use std::time::{Duration, SystemTime, UNIX_EPOCH}; +use std::sync::Arc; +use std::time::{SystemTime, UNIX_EPOCH}; use async_trait::async_trait; use fred::prelude::{HashesInterface, SetsInterface}; @@ -12,19 +13,14 @@ use crate::{ error::JobError, handler::{JobCtx, JobHandler}, }, - scrapers::{persist, SearchMeta}, + models::user_data::UserData, + scrapers::{background_queue, orchestrator, SearchMeta}, + state::AppState, }; pub struct BackgroundSearch; -// ─── Constants ──────────────────────────────────────────────────────────────── - -const MOVIES_KEY: &str = "background_search:movies"; -const SERIES_KEY: &str = "background_search:series"; -const PROCESSING_KEY: &str = "background_search:processing"; const BATCH_SIZE: usize = 10; -/// Default re-scrape interval: 24 hours in seconds. -const DEFAULT_INTERVAL_SECS: f64 = 86_400.0; // ─── Queue item value shape ─────────────────────────────────────────────────── @@ -105,36 +101,20 @@ impl JobHandler for BackgroundSearch { async fn run(&self, _args: Self::Args, ctx: JobCtx) -> Result<(), JobError> { let redis = &ctx.state.redis; - let pool = &ctx.state.pool; let config = &ctx.state.config; - let client = &ctx.state.http; - let now = now_f64(); + let interval_secs = (config.background_search_interval_hours * 3600) as f64; - // Re-scrape interval: use the configured prowlarr/jackett TTL as proxy, - // falling back to 24 h. - let interval_secs = if config.prowlarr_url.is_some() { - config.prowlarr_search_ttl as f64 - } else if config.jackett_url.is_some() { - config.jackett_search_ttl as f64 - } else { - DEFAULT_INTERVAL_SECS - }; - - // Collect due items from both queues let movies_raw: HashMap = redis - .hgetall::, _>(MOVIES_KEY) + .hgetall::, _>(background_queue::MOVIES_KEY) .await .unwrap_or_default(); let series_raw: HashMap = redis - .hgetall::, _>(SERIES_KEY) + .hgetall::, _>(background_queue::SERIES_KEY) .await .unwrap_or_default(); - // Each item in the work queue: (key, media_type_queue_key, item_key) - // item_key for movies = media_id string - // item_key for series = "{media_id}:{season}:{episode}" let mut due: Vec<(String, &'static str)> = Vec::new(); for (item_key, json_val) in &movies_raw { @@ -148,13 +128,13 @@ impl JobHandler for BackgroundSearch { continue; } let in_processing: bool = redis - .sismember::(PROCESSING_KEY, item_key.as_str()) + .sismember::(background_queue::PROCESSING_KEY, item_key.as_str()) .await .unwrap_or(false); if in_processing { continue; } - due.push((item_key.clone(), MOVIES_KEY)); + due.push((item_key.clone(), background_queue::MOVIES_KEY)); } for (item_key, json_val) in &series_raw { @@ -168,13 +148,13 @@ impl JobHandler for BackgroundSearch { continue; } let in_processing: bool = redis - .sismember::(PROCESSING_KEY, item_key.as_str()) + .sismember::(background_queue::PROCESSING_KEY, item_key.as_str()) .await .unwrap_or(false); if in_processing { continue; } - due.push((item_key.clone(), SERIES_KEY)); + due.push((item_key.clone(), background_queue::SERIES_KEY)); } debug!("background_search: {} items due for re-scrape", due.len()); @@ -185,20 +165,18 @@ impl JobHandler for BackgroundSearch { return Err(JobError::Cancelled); } - // Mark as processing let _ = redis - .sadd::<(), _, _>(PROCESSING_KEY, item_key.as_str()) + .sadd::<(), _, _>(background_queue::PROCESSING_KEY, item_key.as_str()) .await; - let result = process_item(&item_key, queue_key, pool, client, redis, config, now).await; + let result = process_item(&item_key, queue_key, &ctx.state, now).await; if let Err(e) = result { warn!("background_search: item {item_key} failed: {e}"); } - // Always unmark processing let _ = redis - .srem::<(), _, _>(PROCESSING_KEY, item_key.as_str()) + .srem::<(), _, _>(background_queue::PROCESSING_KEY, item_key.as_str()) .await; } @@ -209,32 +187,27 @@ impl JobHandler for BackgroundSearch { async fn process_item( item_key: &str, queue_key: &'static str, - pool: &sqlx::PgPool, - client: &reqwest::Client, - redis: &fred::clients::Client, - config: &crate::config::AppConfig, + state: &Arc, now: f64, ) -> Result<(), Box> { - let is_movie = queue_key == MOVIES_KEY; + let redis = &state.redis; + let pool = &state.pool; + let is_movie = queue_key == background_queue::MOVIES_KEY; - // Parse the item_key let (media_id_str, season, episode): (&str, Option, Option) = if is_movie { (item_key, None, None) } else { - // Format: "{media_id}:{season}:{episode}" let parts: Vec<&str> = item_key.splitn(3, ':').collect(); if parts.len() == 3 { let s: Option = parts[1].parse().ok(); let e: Option = parts[2].parse().ok(); (parts[0], s, e) } else { - // Malformed key — remove from hash let _ = redis.hdel::<(), _, _>(queue_key, item_key).await; return Ok(()); } }; - // Look up media in DB let media = if is_movie { match lookup_movie(pool, media_id_str).await { Some(m) => m, @@ -273,62 +246,17 @@ async fn process_item( let media_type = if is_movie { "movie" } else { "series" }; - // Scrape from configured providers - let max_process = 50usize; - let max_time = Duration::from_secs(120); - let query_timeout = Duration::from_secs(30); - - let mut all_streams = Vec::new(); - - if let (Some(url), Some(key)) = (&config.prowlarr_url, &config.prowlarr_api_key) { - let streams = crate::scrapers::prowlarr::scrape( - client, - url, - key, - &meta, - media_type, - season, - episode, - max_process, - max_time, - query_timeout, - ) - .await; - debug!( - "background_search: prowlarr returned {} streams for {}", - streams.len(), - item_key - ); - all_streams.extend(streams); - } - - if let (Some(url), Some(key)) = (&config.jackett_url, &config.jackett_api_key) { - let streams = crate::scrapers::jackett::scrape( - client, - url, - key, - &meta, - media_type, - season, - episode, - max_process, - max_time, - query_timeout, - ) - .await; - debug!( - "background_search: jackett returned {} streams for {}", - streams.len(), - item_key - ); - all_streams.extend(streams); - } - - if !all_streams.is_empty() { - persist::write_back(&all_streams, pool, &meta, media_type, season, episode).await; - } + orchestrator::run_background( + state, + &UserData::default(), + &meta, + media_type, + season, + episode, + "background", + ) + .await; - // Update last_scrape timestamp in Redis let updated = serde_json::json!({ "last_scrape": now, "added_at": now, diff --git a/backend/src/jobs/handlers/discover_prewarm.rs b/backend/src/jobs/handlers/discover_prewarm.rs index cd9c226f..4ebedd1d 100644 --- a/backend/src/jobs/handlers/discover_prewarm.rs +++ b/backend/src/jobs/handlers/discover_prewarm.rs @@ -132,12 +132,20 @@ async fn upsert_media( Ok(media_id) } -/// Link a media row to a catalog by name (does nothing if already linked). +/// Link a media row to a catalog by name, creating the catalog row if it doesn't exist. async fn link_to_catalog( pool: &sqlx::PgPool, media_id: i32, catalog_name: &str, ) -> Result<(), sqlx::Error> { + sqlx::query( + "INSERT INTO catalog (name, display_name, is_system, display_order) \ + VALUES ($1, $1, true, 0) ON CONFLICT (name) DO NOTHING", + ) + .bind(catalog_name) + .execute(pool) + .await?; + let catalog_id: Option<(i32,)> = sqlx::query_as("SELECT id FROM catalog WHERE name = $1") .bind(catalog_name) .fetch_optional(pool) @@ -145,7 +153,7 @@ async fn link_to_catalog( let Some((catalog_id,)) = catalog_id else { warn!( - "discover_prewarm: catalog '{}' not found — skipping", + "discover_prewarm: catalog '{}' not found after upsert — skipping", catalog_name ); return Ok(()); diff --git a/backend/src/jobs/handlers/dmm_hashlist.rs b/backend/src/jobs/handlers/dmm_hashlist.rs index 722f2259..8cd6fbe5 100644 --- a/backend/src/jobs/handlers/dmm_hashlist.rs +++ b/backend/src/jobs/handlers/dmm_hashlist.rs @@ -356,7 +356,6 @@ async fn store_torrent_stream( }; let category = sports.catalog.as_deref().unwrap_or("other_sports"); - let genre = parser::sports_category_to_genre(category); let stub_type = if sports.media_type == "series" { "SERIES" } else { @@ -367,7 +366,6 @@ async fn store_torrent_stream( pool, &sports.meta.title, sports.meta.year, - genre, None, stub_type, ) diff --git a/backend/src/jobs/handlers/spiders/ext_to.rs b/backend/src/jobs/handlers/spiders/ext_to.rs index 8d2c5ce5..1d4a9882 100644 --- a/backend/src/jobs/handlers/spiders/ext_to.rs +++ b/backend/src/jobs/handlers/spiders/ext_to.rs @@ -100,8 +100,6 @@ pub(crate) struct CatalogSpec { media_type: &'static str, /// Sports category key (matches `catalog.name` in the DB, e.g. "formula_racing"). category: &'static str, - /// Display genre name matching keys in sports_artifacts.json (e.g. "Formula Racing"). - genre: &'static str, } const FORMULA_SPEC: CatalogSpec = CatalogSpec { @@ -111,7 +109,6 @@ const FORMULA_SPEC: CatalogSpec = CatalogSpec { keyword: "formula", media_type: "movie", category: "formula_racing", - genre: "Formula Racing", }; const MOTOGP_SPEC: CatalogSpec = CatalogSpec { @@ -121,7 +118,6 @@ const MOTOGP_SPEC: CatalogSpec = CatalogSpec { keyword: "motogp", media_type: "movie", category: "motogp_racing", - genre: "MotoGP Racing", }; const WWE_SPEC: CatalogSpec = CatalogSpec { @@ -131,7 +127,6 @@ const WWE_SPEC: CatalogSpec = CatalogSpec { keyword: "wwe", media_type: "movie", category: "fighting", - genre: "Fighting (WWE, UFC)", }; const UFC_SPEC: CatalogSpec = CatalogSpec { @@ -141,7 +136,6 @@ const UFC_SPEC: CatalogSpec = CatalogSpec { keyword: "ufc", media_type: "movie", category: "fighting", - genre: "Fighting (WWE, UFC)", }; const MOVIES_TV_SPEC: CatalogSpec = CatalogSpec { @@ -151,7 +145,6 @@ const MOVIES_TV_SPEC: CatalogSpec = CatalogSpec { keyword: "", media_type: "movie", category: "ext_to_movie", - genre: "", }; // ─── HMAC helpers ───────────────────────────────────────────────────────────── @@ -632,7 +625,6 @@ pub(crate) async fn scrape_ext_catalog(spec: &CatalogSpec, ctx: &JobCtx) -> Resu pool, &clean_title, year, - spec.genre, None, &stub_media_type, ) diff --git a/backend/src/jobs/handlers/spiders/sport_video.rs b/backend/src/jobs/handlers/spiders/sport_video.rs index 86c38e0f..db311b86 100644 --- a/backend/src/jobs/handlers/spiders/sport_video.rs +++ b/backend/src/jobs/handlers/spiders/sport_video.rs @@ -404,13 +404,11 @@ impl JobHandler for SportVideoCrawl { // with its true catalog key (e.g. "formula_racing"). let detected_category: &str = parser::detect_sports_category(&block.title).unwrap_or(category.as_str()); - let display_genre = parser::sports_category_to_genre(detected_category); let media_id = media_resolve::find_or_create_sports_stub( pool, &block.title, parsed_title.year, - display_genre, block.poster_url.as_deref(), "MOVIE", ) diff --git a/backend/src/metrics_middleware.rs b/backend/src/metrics_middleware.rs index 1f887994..6ba4d40c 100644 --- a/backend/src/metrics_middleware.rs +++ b/backend/src/metrics_middleware.rs @@ -3,10 +3,26 @@ use std::sync::Arc; use std::sync::OnceLock; use axum::{extract::State, middleware::Next, response::Response}; +use fred::prelude::{HashesInterface, KeysInterface, ListInterface, SortedSetsInterface}; use regex::Regex; use crate::state::AppState; +pub const AGG_PREFIX: &str = "req_metrics:agg:"; +pub const ENDPOINTS_KEY: &str = "req_metrics:endpoints"; +pub const RECENT_KEY: &str = "req_metrics:recent"; +const METRICS_TTL: i64 = 86400; +const RECENT_TTL: i64 = 3600; +const MAX_RECENT: i64 = 10000; + +const SKIP_PREFIXES: &[&str] = &[ + "/health", + "/ready", + "/static", + "/app/assets", + "/favicon.ico", +]; + static RE_UUID: OnceLock = OnceLock::new(); static RE_NUM: OnceLock = OnceLock::new(); @@ -39,6 +55,87 @@ impl Drop for InFlightGuard<'_> { } } +async fn record_to_redis( + redis: fred::clients::Client, + method: String, + route: String, + status: u16, + duration_s: f64, + now_iso: String, + now_ts: f64, +) { + let endpoint_key = format!("{method}:{route}"); + let agg_key = format!("{AGG_PREFIX}{endpoint_key}"); + + let status_class = format!("status_{}xx", status / 100); + + // Aggregate stats (atomic increments — no read-modify-write race) + let _: Result = redis.hincrby(&agg_key, "total_requests", 1).await; + let _: Result = redis.hincrbyfloat(&agg_key, "total_time", duration_s).await; + let _: Result = redis.hincrby(&agg_key, &status_class, 1).await; + if status >= 400 { + let _: Result = redis.hincrby(&agg_key, "error_count", 1).await; + } + // last_seen and identity fields (non-atomic, small race acceptable for metrics) + let mut fields = std::collections::HashMap::new(); + fields.insert("last_seen", now_iso.clone()); + let _: Result<(), _> = redis.hset(&agg_key, fields).await; + let _: Result = redis.hsetnx(&agg_key, "method", &method).await; + let _: Result = redis.hsetnx(&agg_key, "route", &route).await; + + // min/max time (read-then-update; approximate under concurrent writes is fine for metrics) + let existing_min: Option = redis.hget(&agg_key, "min_time").await.unwrap_or(None); + let existing_max: Option = redis.hget(&agg_key, "max_time").await.unwrap_or(None); + let cur_min = existing_min + .as_deref() + .and_then(|v| v.parse::().ok()) + .unwrap_or(f64::INFINITY); + let cur_max = existing_max + .as_deref() + .and_then(|v| v.parse::().ok()) + .unwrap_or(0.0_f64); + if duration_s < cur_min { + let _: Result = redis + .hset(&agg_key, [("min_time", format!("{duration_s:.6}"))]) + .await; + } + if duration_s > cur_max { + let _: Result = redis + .hset(&agg_key, [("max_time", format!("{duration_s:.6}"))]) + .await; + } + + let _: Result<(), _> = redis.expire(&agg_key, METRICS_TTL, None).await; + + // Endpoints sorted set — score = Unix timestamp for recency ordering + let _: Result = redis + .zadd( + ENDPOINTS_KEY, + None, + None, + false, + false, + (now_ts, endpoint_key.as_str()), + ) + .await; + let _: Result<(), _> = redis.expire(ENDPOINTS_KEY, METRICS_TTL, None).await; + + // Recent requests list + let entry = serde_json::json!({ + "method": method, + "path": route, + "route_template": route, + "status_code": status, + "process_time": duration_s, + "timestamp": now_iso, + }); + if let Ok(json_str) = serde_json::to_string(&entry) { + let _: Result = redis.lpush(RECENT_KEY, json_str).await; + let _: Result<(), _> = redis.ltrim(RECENT_KEY, 0, MAX_RECENT - 1).await; + let _: Result<(), _> = redis.expire(RECENT_KEY, RECENT_TTL, None).await; + } +} + pub async fn metrics_middleware( State(state): State>, req: axum::http::Request, @@ -58,11 +155,23 @@ pub async fn metrics_middleware( let resp = next.run(req).await; let duration_ms = start.elapsed().as_secs_f64() * 1000.0; let status = resp.status().as_u16(); - // Guard drops here (or earlier on cancellation). drop(_guard); state .metrics .record_request(&method, &route, status, duration_ms); + + // Skip noisy health/asset paths for Redis metrics + if !SKIP_PREFIXES.iter().any(|p| path.starts_with(p)) { + let redis = state.redis.clone(); + let now = chrono::Utc::now(); + let now_iso = now.to_rfc3339(); + let now_ts = now.timestamp() as f64; + let duration_s = duration_ms / 1000.0; + tokio::spawn(record_to_redis( + redis, method, route, status, duration_s, now_iso, now_ts, + )); + } + resp } diff --git a/backend/src/models/stremio.rs b/backend/src/models/stremio.rs index 0494baa0..672ed4ea 100644 --- a/backend/src/models/stremio.rs +++ b/backend/src/models/stremio.rs @@ -63,6 +63,8 @@ pub struct MetaPreview { #[serde(skip_serializing_if = "Option::is_none")] pub poster: Option, #[serde(skip_serializing_if = "Option::is_none")] + pub background: Option, + #[serde(skip_serializing_if = "Option::is_none")] pub description: Option, } diff --git a/backend/src/models/user_data.rs b/backend/src/models/user_data.rs index f61418f5..69520f45 100644 --- a/backend/src/models/user_data.rs +++ b/backend/src/models/user_data.rs @@ -507,7 +507,7 @@ const PROVIDER_SHORT_NAMES: &[(&str, &str)] = &[ ("offcloud", "OC"), ("pikpak", "PKP"), ("torbox", "TRB"), - ("seedr", "SEEDR"), + ("seedr", "SDR"), ("stremthru", "ST"), ("qbittorrent", "QB"), ("easydebrid", "ED"), @@ -521,6 +521,11 @@ fn short_name(service: &str) -> Option<&'static str> { .map(|(_, n)| *n) } +/// Short label for a streaming provider (e.g. `seedr` → `SDR`). +pub fn provider_short_name(service: &str) -> &str { + short_name(service).unwrap_or(service) +} + // ─── UserData methods ───────────────────────────────────────────────────────── impl UserData { diff --git a/backend/src/parser/mod.rs b/backend/src/parser/mod.rs index 77fd1ef3..a532cc1a 100644 --- a/backend/src/parser/mod.rs +++ b/backend/src/parser/mod.rs @@ -4,8 +4,7 @@ pub mod sports_parser; pub use sports_parser::{ classify_wwe_title, clean_sports_title, detect_sports_category, is_sports_title, - parse_racing_title, parse_sports_title, racing_session_episode, sports_category_to_genre, - RacingParsed, WweEpisodeInfo, + parse_racing_title, parse_sports_title, racing_session_episode, RacingParsed, WweEpisodeInfo, }; use std::sync::OnceLock; diff --git a/backend/src/providers/mod.rs b/backend/src/providers/mod.rs index 99fecdc2..518dc4d8 100644 --- a/backend/src/providers/mod.rs +++ b/backend/src/providers/mod.rs @@ -7,6 +7,10 @@ use thiserror::Error; /// On any failure (HTTP error, read error, parse error) logs a WARN with /// the raw response body/status so callers can diagnose what the provider /// actually returned, then returns the original error. +/// +/// For 401/403 responses whose body is not valid JSON (e.g. an HTML error page +/// from a reverse proxy), returns a typed `invalid_token.mp4` error rather than +/// the generic `api_error.mp4` that a bare parse failure would produce. pub async fn response_json( resp: reqwest::Response, context: &str, @@ -17,6 +21,14 @@ pub async fn response_json( ProviderError::Http(e) })?; serde_json::from_str(&text).map_err(|e| { + // Auth failures with non-JSON bodies (HTML proxy error pages) should show + // the credentials error video, not the generic api_error. + if status == reqwest::StatusCode::UNAUTHORIZED || status == reqwest::StatusCode::FORBIDDEN { + return ProviderError::api( + format!("Authentication failed (HTTP {status})"), + "invalid_token.mp4", + ); + } // Truncate very long bodies (e.g. full HTML error pages) in the log. let preview: &str = if text.len() > 500 { &text[..500] diff --git a/backend/src/providers/torrents/easydebrid.rs b/backend/src/providers/torrents/easydebrid.rs index 93931d78..f01437d7 100644 --- a/backend/src/providers/torrents/easydebrid.rs +++ b/backend/src/providers/torrents/easydebrid.rs @@ -207,6 +207,12 @@ pub async fn get_video_url( )) } +// ─── Downloaded torrent list (not supported) ────────────────────────────────── + +// EasyDebrid is cache-only instant debrid. The API exposes /link/generate, +// /link/request, and /link/lookup but no account-level "list my downloads" +// endpoint, so watchlist catalogs cannot be populated from EasyDebrid. + // ─── Delete all torrents (no-op for EasyDebrid) ─────────────────────────────── /// EasyDebrid has no account-level deletion API — this is a no-op. diff --git a/backend/src/providers/torrents/mod.rs b/backend/src/providers/torrents/mod.rs index e1ecaa83..e65d3dce 100644 --- a/backend/src/providers/torrents/mod.rs +++ b/backend/src/providers/torrents/mod.rs @@ -18,3 +18,61 @@ pub mod seedr; pub mod stremthru; pub mod torbox; pub mod transport; + +pub use realdebrid::DownloadedTorrent; + +use crate::providers::ProviderError; + +/// Providers that expose a downloaded-torrent listing API. +pub fn supports_download_list(service: &str) -> bool { + matches!( + service, + "realdebrid" + | "torbox" + | "alldebrid" + | "debridlink" + | "premiumize" + | "offcloud" + | "pikpak" + | "seedr" + | "stremthru" + ) +} + +/// List all downloaded torrents for a debrid provider (single dispatch point). +pub async fn list_downloaded_torrents( + http: &reqwest::Client, + service: &str, + token: &str, +) -> Result, ProviderError> { + match service { + "realdebrid" => realdebrid::list_downloaded_torrents(http, token).await, + "torbox" => torbox::list_downloaded_torrents(http, token).await, + "alldebrid" => alldebrid::list_downloaded_torrents(http, token).await, + "debridlink" => debridlink::list_downloaded_torrents(http, token).await, + "premiumize" => premiumize::list_downloaded_torrents(http, token).await, + "offcloud" => offcloud::list_downloaded_torrents(http, token).await, + "pikpak" => pikpak::list_downloaded_torrents(http, token).await, + "seedr" => seedr::list_downloaded_torrents(http, token).await, + "stremthru" => stremthru::list_downloaded_torrents(http, token).await, + // EasyDebrid is cache-only — no account-level torrent list endpoint exists. + "easydebrid" => Ok(vec![]), + other => { + tracing::debug!("list_downloaded_torrents: no list API for '{other}'"); + Ok(vec![]) + } + } +} + +/// Return lowercased info_hashes for all downloaded torrents in the provider account. +pub async fn list_downloaded_hashes( + http: &reqwest::Client, + service: &str, + token: &str, +) -> Result, ProviderError> { + Ok(list_downloaded_torrents(http, service, token) + .await? + .into_iter() + .map(|t| t.info_hash) + .collect()) +} diff --git a/backend/src/providers/torrents/premiumize.rs b/backend/src/providers/torrents/premiumize.rs index 73c7f9ff..7c7123ab 100644 --- a/backend/src/providers/torrents/premiumize.rs +++ b/backend/src/providers/torrents/premiumize.rs @@ -230,9 +230,21 @@ fn check_pm_error(body: &Value) -> Result<(), ProviderError> { .get("message") .and_then(|v| v.as_str()) .unwrap_or("Unknown error"); + // Authentication failures use a dedicated video. + let msg_lower = msg.to_lowercase(); + let video = if msg_lower.contains("not logged in") + || msg_lower.contains("invalid api key") + || msg_lower.contains("invalid apikey") + || msg_lower.contains("unauthorized") + || msg_lower.contains("authentication") + { + "invalid_token.mp4" + } else { + "transfer_error.mp4" + }; return Err(ProviderError::api( format!("Premiumize API error: {msg}"), - "transfer_error.mp4", + video, )); } } diff --git a/backend/src/providers/torrents/realdebrid.rs b/backend/src/providers/torrents/realdebrid.rs index 6b15ec7b..fa03e9ca 100644 --- a/backend/src/providers/torrents/realdebrid.rs +++ b/backend/src/providers/torrents/realdebrid.rs @@ -1125,7 +1125,10 @@ pub async fn check_cached(http: &reqwest::Client, token: &str, hashes: &[String] code, } => match get_access_token(http, &client_id, &client_secret, &code, None).await { Ok(t) => t, - Err(_) => return vec![], + Err(e) => { + tracing::warn!("realdebrid check_cached: token exchange failed: {e}"); + return vec![]; + } }, }; @@ -1133,42 +1136,17 @@ pub async fn check_cached(http: &reqwest::Client, token: &str, hashes: &[String] let mut found = Vec::new(); for page in 1..=MAX_PAGES { - let url = format!("{BASE_URL}/torrents?page={page}&limit={PAGE_SIZE}"); - let resp = match http.get(&url).bearer_auth(&bearer).send().await { - Ok(r) => r, + // Use get_torrent_list so the request goes through rd_get → response_json → + // check_rd_error, giving consistent 401/API-error handling. + let arr = match get_torrent_list(http, &bearer, page, PAGE_SIZE).await { + Ok(a) if a.is_empty() => break, + Ok(a) => a, Err(e) => { - tracing::warn!("realdebrid torrents page {page}: {e}"); + tracing::warn!("realdebrid check_cached page {page}: {e}"); break; } }; - if resp.status() == 204 { - break; - } - if !resp.status().is_success() { - tracing::warn!("realdebrid torrents page {page}: HTTP {}", resp.status()); - break; - } - let text = match resp.text().await { - Ok(t) => t, - Err(e) => { - tracing::warn!("realdebrid torrents read page {page}: {e}"); - break; - } - }; - let body: serde_json::Value = match serde_json::from_str(&text) { - Ok(v) => v, - Err(e) => { - tracing::warn!( - "realdebrid torrents json page {page}: {e} — body: {}", - &text[..text.len().min(200)] - ); - break; - } - }; - let arr = match body.as_array() { - Some(a) if !a.is_empty() => a.clone(), - _ => break, - }; + for t in &arr { if t.get("status").and_then(|v| v.as_str()) == Some("downloaded") { if let Some(h) = t.get("hash").and_then(|v| v.as_str()) { diff --git a/backend/src/providers/torrents/seedr.rs b/backend/src/providers/torrents/seedr.rs index fbc52bd8..1b29d925 100644 --- a/backend/src/providers/torrents/seedr.rs +++ b/backend/src/providers/torrents/seedr.rs @@ -230,6 +230,36 @@ fn task_is_downloading(task: &Value) -> bool { ) } +/// Root-level Seedr folders are named with the torrent info_hash (40-char SHA-1 or 32-char MD5). +fn is_info_hash_folder_name(name: &str) -> bool { + let len = name.len(); + len == 40 || len == 32 +} + +/// List root folders whose names are info hashes — mirrors Python `list_contents().folders`. +async fn list_root_hash_folders( + http: &Client, + token: &str, + forward: Option<&MediaFlowForward>, +) -> Result, ProviderError> { + let root = api_get(http, token, "/fs/root/contents", forward).await?; + let mut out = Vec::new(); + if let Some(folders) = root["folders"].as_array() { + for folder in folders { + let path = folder["path"].as_str().unwrap_or("").to_lowercase(); + if !is_info_hash_folder_name(&path) { + continue; + } + let Some(folder_id) = folder["id"].as_i64() else { + continue; + }; + let size = folder["size"].as_i64().unwrap_or(0); + out.push((path, folder_id, size)); + } + } + Ok(out) +} + // ─── File collection ────────────────────────────────────────────────────────── // Returns Vec<(name, size, file_id)> collecting all files recursively @@ -978,7 +1008,7 @@ pub async fn delete_all_torrents(http: &Client, token: &str) -> Result<(), Provi if let Some(folders) = root["folders"].as_array() { for folder in folders { let path = folder["path"].as_str().unwrap_or("").to_lowercase(); - if path.len() == 40 || path.len() == 32 { + if is_info_hash_folder_name(&path) { if let Some(id) = folder["id"].as_i64() { api_delete(http, &bearer, &format!("/fs/folder/{id}"), None) .await @@ -1023,35 +1053,19 @@ pub async fn delete_torrent_by_hash( // ─── Torrent list ──────────────────────────────────────────────────────────── -/// Return all completed tasks with their video files, ready for the missing-import flow. +/// Return all downloaded torrents by scanning root hash-named folders. +/// +/// Seedr stores completed downloads as root folders named with the info_hash +/// (see Python `fetch_downloaded_info_hashes_from_seedr`), not via the tasks API. pub async fn list_downloaded_torrents( http: &Client, token: &str, ) -> Result, ProviderError> { let bearer = resolve_token(token); - let tasks = list_tasks(http, &bearer, None).await.unwrap_or_default(); - - let mut results = Vec::new(); - for task in &tasks { - if !task_is_complete(task) { - continue; - } - let info_hash = match task_info_hash(task) { - Some(h) => h, - None => continue, - }; - let id = task["id"] - .as_i64() - .map(|v| v.to_string()) - .unwrap_or_default(); - let name = task["name"].as_str().unwrap_or(&info_hash).to_string(); - let size = task["size"].as_i64().unwrap_or(0); - - let folder_id = match task["folder_created_id"].as_i64() { - Some(id) => id, - None => continue, - }; + let hash_folders = list_root_hash_folders(http, &bearer, None).await?; + let mut results = Vec::with_capacity(hash_folders.len()); + for (info_hash, folder_id, size) in hash_folders { let all_files = folder_files_recursive(http, &bearer, folder_id, None) .await .unwrap_or_default(); @@ -1063,12 +1077,25 @@ pub async fn list_downloaded_torrents( .collect(); results.push(crate::providers::torrents::realdebrid::DownloadedTorrent { - id, - info_hash, - name, + id: folder_id.to_string(), + info_hash: info_hash.clone(), + name: info_hash, size, raw: serde_json::json!({ "files": files }), }); } Ok(results) } + +#[cfg(test)] +mod tests { + use super::is_info_hash_folder_name; + + #[test] + fn info_hash_folder_name_matches_sha1_and_md5_lengths() { + assert!(is_info_hash_folder_name(&"a".repeat(40))); + assert!(is_info_hash_folder_name(&"b".repeat(32))); + assert!(!is_info_hash_folder_name("not-a-hash")); + assert!(!is_info_hash_folder_name(&"c".repeat(39))); + } +} diff --git a/backend/src/providers/torrents/stremthru.rs b/backend/src/providers/torrents/stremthru.rs index 9d157dc9..73cb7dda 100644 --- a/backend/src/providers/torrents/stremthru.rs +++ b/backend/src/providers/torrents/stremthru.rs @@ -326,6 +326,41 @@ async fn wait_for_downloaded( // ─── Public entry points ────────────────────────────────────────────────────── +/// Return all magnets stored in the user's StremThru account. +pub async fn list_downloaded_torrents( + http: &reqwest::Client, + token: &str, +) -> Result, ProviderError> { + let cfg = parse_config(token); + let items = list_magnets(http, &cfg).await?; + let mut results = Vec::with_capacity(items.len()); + for item in items { + let info_hash = match item.get("hash").and_then(|v| v.as_str()) { + Some(h) => h.to_lowercase(), + None => continue, + }; + let id = item + .get("id") + .and_then(|v| v.as_str()) + .unwrap_or("") + .to_string(); + let name = item + .get("name") + .and_then(|v| v.as_str()) + .unwrap_or(&info_hash) + .to_string(); + let size = item.get("size").and_then(|v| v.as_i64()).unwrap_or(0); + results.push(crate::providers::torrents::realdebrid::DownloadedTorrent { + id, + info_hash, + name, + size, + raw: item, + }); + } + Ok(results) +} + /// Resolve a direct video URL from StremThru for the given torrent. #[allow(clippy::too_many_arguments)] pub async fn get_video_url( diff --git a/backend/src/routes/admin_extended.rs b/backend/src/routes/admin_extended.rs index b7381e21..2906db0c 100644 --- a/backend/src/routes/admin_extended.rs +++ b/backend/src/routes/admin_extended.rs @@ -753,9 +753,7 @@ pub async fn clear_single_exception( // ─── request_metrics.py endpoints ──────────────────────────────────────────── -const METRICS_KEY_PREFIX: &str = "req_metric:"; -const METRICS_INDEX_KEY: &str = "req_metrics:index"; -const RECENT_REQUESTS_KEY: &str = "req_metrics:recent"; +use crate::metrics_middleware::{AGG_PREFIX, ENDPOINTS_KEY, RECENT_KEY}; #[derive(Deserialize)] pub struct MetricsEndpointListQuery { @@ -774,26 +772,60 @@ pub struct RecentRequestsQuery { pub route: Option, } +fn agg_hash_to_item(ep_key: &str, data: std::collections::HashMap) -> Value { + let total_req: i64 = data + .get("total_requests") + .and_then(|v| v.parse().ok()) + .unwrap_or(0); + let total_time: f64 = data + .get("total_time") + .and_then(|v| v.parse().ok()) + .unwrap_or(0.0); + let avg_time = if total_req > 0 { + (total_time / total_req as f64 * 1_000_000.0).round() / 1_000_000.0 + } else { + 0.0 + }; + let method = data + .get("method") + .cloned() + .unwrap_or_else(|| ep_key.split(':').next().unwrap_or("").to_string()); + let route = data.get("route").cloned().unwrap_or_else(|| { + ep_key + .split_once(':') + .map(|(_, rest)| rest.to_string()) + .unwrap_or_else(|| ep_key.to_string()) + }); + + json!({ + "endpoint_key": ep_key, + "method": method, + "route": route, + "total_requests": total_req, + "avg_time": avg_time, + "min_time": data.get("min_time").and_then(|v| v.parse::().ok()).unwrap_or(0.0), + "max_time": data.get("max_time").and_then(|v| v.parse::().ok()).unwrap_or(0.0), + "error_count": data.get("error_count").and_then(|v| v.parse::().ok()).unwrap_or(0), + "status_2xx": data.get("status_2xx").and_then(|v| v.parse::().ok()).unwrap_or(0), + "status_3xx": data.get("status_3xx").and_then(|v| v.parse::().ok()).unwrap_or(0), + "status_4xx": data.get("status_4xx").and_then(|v| v.parse::().ok()).unwrap_or(0), + "status_5xx": data.get("status_5xx").and_then(|v| v.parse::().ok()).unwrap_or(0), + "last_seen": data.get("last_seen").cloned().unwrap_or_default(), + }) +} + /// GET /api/v1/admin/request-metrics/status pub async fn get_request_metrics_status( headers: HeaderMap, State(state): State>, ) -> impl IntoResponse { + use fred::prelude::SortedSetsInterface; if validate_admin(&headers, &state.config.secret_key_raw).is_none() { return forbidden(); } - let total_endpoints: i64 = state - .redis - .llen::(METRICS_INDEX_KEY) - .await - .unwrap_or(0); - - let total_recent: i64 = state - .redis - .llen::(RECENT_REQUESTS_KEY) - .await - .unwrap_or(0); + let total_endpoints: i64 = state.redis.zcard(ENDPOINTS_KEY).await.unwrap_or(0); + let total_recent: i64 = state.redis.llen::(RECENT_KEY).await.unwrap_or(0); Json(json!({ "enabled": true, @@ -814,33 +846,54 @@ pub async fn list_endpoint_stats( State(state): State>, Query(params): Query, ) -> impl IntoResponse { + use fred::prelude::SortedSetsInterface; if validate_admin(&headers, &state.config.secret_key_raw).is_none() { return forbidden(); } let page = params.page.unwrap_or(1).max(1); let per_page = params.per_page.unwrap_or(20).clamp(1, 100); + let sort_by = params.sort_by.as_deref().unwrap_or("total_requests"); + let sort_desc = params.sort_order.as_deref().unwrap_or("desc") != "asc"; - let all_keys: Vec = state + // Get all endpoints (most-recently-seen first) + let all_ep_keys: Vec = state .redis - .lrange::, _>(METRICS_INDEX_KEY, 0, -1) + .zrange(ENDPOINTS_KEY, 0i64, -1i64, None, true, None, false) .await .unwrap_or_default(); let mut items: Vec = Vec::new(); - for key in &all_keys { - let data: Option = state - .redis - .get::, _>(format!("{METRICS_KEY_PREFIX}{key}")) - .await - .unwrap_or(None); - if let Some(raw) = data { - if let Ok(v) = serde_json::from_str::(&raw) { - items.push(v); - } + for ep_key in &all_ep_keys { + let agg_key = format!("{AGG_PREFIX}{ep_key}"); + let data: std::collections::HashMap = + state.redis.hgetall(&agg_key).await.unwrap_or_default(); + if !data.is_empty() { + items.push(agg_hash_to_item(ep_key, data)); } } + // Sort + items.sort_by(|a, b| { + let va = match sort_by { + "avg_time" | "min_time" | "max_time" => { + a.get(sort_by).and_then(|v| v.as_f64()).unwrap_or(0.0) + } + _ => a.get(sort_by).and_then(|v| v.as_i64()).unwrap_or(0) as f64, + }; + let vb = match sort_by { + "avg_time" | "min_time" | "max_time" => { + b.get(sort_by).and_then(|v| v.as_f64()).unwrap_or(0.0) + } + _ => b.get(sort_by).and_then(|v| v.as_i64()).unwrap_or(0) as f64, + }; + if sort_desc { + vb.partial_cmp(&va).unwrap_or(std::cmp::Ordering::Equal) + } else { + va.partial_cmp(&vb).unwrap_or(std::cmp::Ordering::Equal) + } + }); + let total = items.len() as i64; let pages = (total + per_page - 1) / per_page; let offset = ((page - 1) * per_page) as usize; @@ -876,24 +929,20 @@ pub async fn get_endpoint_detail( format!("/{route}") }; - let key = format!("{}{}-{}", METRICS_KEY_PREFIX, method.to_uppercase(), route); - let data: Option = state - .redis - .get::, _>(&key) - .await - .unwrap_or(None); + let ep_key = format!("{}:{route}", method.to_uppercase()); + let agg_key = format!("{AGG_PREFIX}{ep_key}"); + let data: std::collections::HashMap = + state.redis.hgetall(&agg_key).await.unwrap_or_default(); - match data { - Some(raw) => match serde_json::from_str::(&raw) { - Ok(v) => Json(v).into_response(), - Err(_) => StatusCode::INTERNAL_SERVER_ERROR.into_response(), - }, - None => ( + if data.is_empty() { + return ( StatusCode::NOT_FOUND, Json(json!({"detail": "Endpoint metrics not found. It may have expired."})), ) - .into_response(), + .into_response(); } + + Json(agg_hash_to_item(&ep_key, data)).into_response() } /// GET /api/v1/admin/request-metrics/recent @@ -911,14 +960,13 @@ pub async fn list_recent_requests( let raw_entries: Vec = state .redis - .lrange::, _>(RECENT_REQUESTS_KEY, 0, -1) + .lrange::, _>(RECENT_KEY, 0, -1) .await .unwrap_or_default(); let mut items: Vec = Vec::new(); for raw in &raw_entries { if let Ok(v) = serde_json::from_str::(raw) { - // Apply filters if let Some(ref method) = params.method { if v.get("method").and_then(|m| m.as_str()) != Some(method.as_str()) { continue; @@ -930,7 +978,11 @@ pub async fn list_recent_requests( } } if let Some(ref route) = params.route { - let path_val = v.get("path").and_then(|p| p.as_str()).unwrap_or(""); + let path_val = v + .get("route_template") + .or_else(|| v.get("path")) + .and_then(|p| p.as_str()) + .unwrap_or(""); if !path_val.contains(route.as_str()) { continue; } @@ -963,32 +1015,29 @@ pub async fn clear_request_metrics( headers: HeaderMap, State(state): State>, ) -> impl IntoResponse { + use fred::prelude::SortedSetsInterface; if validate_admin(&headers, &state.config.secret_key_raw).is_none() { return forbidden(); } - let all_keys: Vec = state + let all_ep_keys: Vec = state .redis - .lrange::, _>(METRICS_INDEX_KEY, 0, -1) + .zrange(ENDPOINTS_KEY, 0i64, -1i64, None, false, None, false) .await .unwrap_or_default(); let mut cleared: i64 = 0; - for key in &all_keys { + for ep_key in &all_ep_keys { let n: i64 = state .redis - .del::(format!("{METRICS_KEY_PREFIX}{key}")) + .del::(format!("{AGG_PREFIX}{ep_key}")) .await .unwrap_or(0); cleared += n; } - let _ = state.redis.del::(METRICS_INDEX_KEY).await; - let n: i64 = state - .redis - .del::(RECENT_REQUESTS_KEY) - .await - .unwrap_or(0); - cleared += n; + let _: Result = state.redis.del(ENDPOINTS_KEY).await; + let _: Result = state.redis.del(RECENT_KEY).await; + cleared += 2; Json(json!({ "cleared": cleared, diff --git a/backend/src/routes/catalog.rs b/backend/src/routes/catalog.rs index 62af6a3b..2dae28c6 100644 --- a/backend/src/routes/catalog.rs +++ b/backend/src/routes/catalog.rs @@ -14,6 +14,7 @@ use crate::{ stremio::{MetaPreview, Metas}, user_data::UserData, }, + routes::delete_all_watchlist, state::AppState, }; @@ -79,6 +80,7 @@ fn rows_to_metas(rows: Vec, host_url: &str) -> Metas { name: r.title, release_info: r.year.map(|y| y.to_string()), poster, + background: None, description: r.description, } }) @@ -86,6 +88,87 @@ fn rows_to_metas(rows: Vec, host_url: &str) -> Metas { Metas { metas } } +fn parse_watchlist_service<'a>(catalog_id: &'a str, media_type: &str) -> Option<&'a str> { + if let Some(service) = catalog_id.strip_suffix("_watchlist_movies") { + if media_type == "movie" { + return Some(service); + } + } else if let Some(service) = catalog_id.strip_suffix("_watchlist_series") { + if media_type == "series" { + return Some(service); + } + } + None +} + +async fn handle_watchlist_catalog( + state: &AppState, + user_data: &UserData, + media_type: &str, + catalog_id: &str, + service: &str, + extra: &ExtraParams, +) -> axum::response::Response { + let provider = match user_data.get_provider_by_name(service) { + Some(p) if p.enable_watchlist_catalogs => p, + _ => return Json(Metas { metas: vec![] }).into_response(), + }; + + let token = match provider.token.as_deref().filter(|t| !t.is_empty()) { + Some(t) => t, + None => return Json(Metas { metas: vec![] }).into_response(), + }; + + let hashes = + match crate::providers::torrents::list_downloaded_hashes(&state.http, service, token).await + { + Ok(h) => h, + Err(e) => { + tracing::warn!("watchlist catalog {service}: {e}"); + return Json(Metas { metas: vec![] }).into_response(); + } + }; + + if hashes.is_empty() { + return Json(Metas { metas: vec![] }).into_response(); + } + + let (sort, sort_dir) = user_data.catalog_sort(catalog_id); + let nudity_excludes = user_data.nudity_filter.clone(); + let cert_excludes: Vec = user_data + .certification_filter + .iter() + .filter(|s| s.as_str() != "Disable") + .cloned() + .collect(); + + let rows = db_catalog::get_watchlist_items( + &state.pool_ro, + media_type, + &hashes, + extra.skip, + &nudity_excludes, + &cert_excludes, + &sort, + &sort_dir, + ) + .await; + + let mut metas = rows_to_metas(rows, &state.config.host_url).metas; + + if media_type == "movie" + && delete_all_watchlist::supports_delete_all(service) + && !metas.is_empty() + { + metas.insert( + 0, + delete_all_watchlist::delete_all_meta_preview(&state.config.host_url, service), + ); + } + + Json(Metas { metas }).into_response() +} + // ─── Shared dispatch ────────────────────────────────────────────────────────── async fn handle_catalog( @@ -108,9 +191,17 @@ async fn handle_catalog( .into_response(); }; - // Watchlist catalogs stay in Python → return 404 so nginx falls back - if catalog_id.ends_with("_watchlist_movies") || catalog_id.ends_with("_watchlist_series") { - return StatusCode::NOT_FOUND.into_response(); + // Watchlist catalogs: fetch debrid downloads and map to media rows (never cached). + if let Some(service) = parse_watchlist_service(&catalog_id, media_type) { + return handle_watchlist_catalog( + &state, + &user_data, + media_type, + &catalog_id, + service, + &extra, + ) + .await; } let (sort, sort_dir) = user_data.catalog_sort(&catalog_id); diff --git a/backend/src/routes/content/contributions.rs b/backend/src/routes/content/contributions.rs index 2324c13b..f00fbaf9 100644 --- a/backend/src/routes/content/contributions.rs +++ b/backend/src/routes/content/contributions.rs @@ -996,9 +996,9 @@ pub async fn create_contribution( || (!is_anonymous && is_active && auto_types.contains(&body.contribution_type.as_str())); let initial_status = if should_auto_approve { - "APPROVED" + crate::db::ContributionStatus::Approved } else { - "PENDING" + crate::db::ContributionStatus::Pending }; let reviewer_id = if should_auto_approve { Some("auto".to_string()) @@ -1610,18 +1610,7 @@ fn is_adult_contribution( data: &serde_json::Value, cache: &crate::state::KeywordFilterCache, ) -> bool { - let check_text = |text: &str| -> bool { - if text.is_empty() { - return false; - } - let lower = text.to_lowercase(); - // whitelist check first - if cache.whitelist.iter().any(|p| lower.contains(p.as_str())) { - return false; - } - // keyword check - cache.keywords.iter().any(|kw| lower.contains(kw.as_str())) - }; + let check_text = |text: &str| -> bool { cache.matches_blocked_keyword(text) }; // Check top-level name and title fields (torrent_name, display name, resolved title) for key in &["name", "title"] { diff --git a/backend/src/routes/content/import_helpers.rs b/backend/src/routes/content/import_helpers.rs index 83dfc113..aee8f50e 100644 --- a/backend/src/routes/content/import_helpers.rs +++ b/backend/src/routes/content/import_helpers.rs @@ -200,7 +200,11 @@ pub async fn create_contribution_record( is_privileged: bool, ) -> Result { let id = Uuid::new_v4().to_string(); - let status = if auto_approve { "APPROVED" } else { "PENDING" }; + let status = if auto_approve { + crate::db::ContributionStatus::Approved + } else { + crate::db::ContributionStatus::Pending + }; let reviewed_by: Option<&str> = if auto_approve { Some("auto") } else { None }; let review_notes: Option = if is_privileged { Some("Auto-approved: Privileged reviewer".to_string()) diff --git a/backend/src/routes/delete_all_watchlist.rs b/backend/src/routes/delete_all_watchlist.rs new file mode 100644 index 00000000..67979447 --- /dev/null +++ b/backend/src/routes/delete_all_watchlist.rs @@ -0,0 +1,170 @@ +use std::sync::Arc; + +use axum::{ + http::StatusCode, + response::{IntoResponse, Json}, +}; +use serde_json::{json, Value}; + +use crate::{ + models::{ + stremio::{Meta, MetaItem, MetaPreview}, + user_data::{provider_short_name, UserData}, + }, + providers, + state::AppState, +}; + +const DELETE_ALL_NAME: &str = "🗑️💩 Delete all files"; +const DELETE_ALL_DESCRIPTION: &str = "🚨💀⚠ Delete all files in streaming provider"; + +/// Parse `dl{service}` watchlist pseudo-IDs (e.g. `dlseedr` → `seedr`). +pub fn parse_service(id: &str) -> Option<&str> { + id.strip_prefix("dl").filter(|s| !s.is_empty()) +} + +/// Providers that expose a delete-all watchlist action (Python `DELETE_ALL_WATCHLIST_FUNCTIONS`). +pub fn supports_delete_all(service: &str) -> bool { + service != "easydebrid" +} + +pub fn delete_all_meta_preview(host_url: &str, service: &str) -> MetaPreview { + MetaPreview { + id: format!("dl{service}"), + media_type: "movie".into(), + name: DELETE_ALL_NAME.into(), + release_info: None, + poster: Some(format!("{host_url}/static/images/delete_all_poster.jpg")), + background: Some(format!( + "{host_url}/static/images/delete_all_background.png" + )), + description: Some(DELETE_ALL_DESCRIPTION.into()), + } +} + +pub fn delete_all_meta(host_url: &str, service: &str) -> Meta { + Meta { + id: format!("dl{service}"), + media_type: "movie".into(), + name: DELETE_ALL_NAME.into(), + release_info: None, + description: Some(DELETE_ALL_DESCRIPTION.into()), + poster: Some(format!("{host_url}/static/images/delete_all_poster.jpg")), + background: Some(format!( + "{host_url}/static/images/delete_all_background.png" + )), + runtime: None, + website: None, + language: None, + country: None, + genres: vec![], + cast: vec![], + imdb_rating: None, + videos: vec![], + links: None, + } +} + +pub fn delete_all_stream_json( + host_url: &str, + secret_str: &str, + service: &str, + addon_name: &str, +) -> Value { + let short_name = provider_short_name(service); + json!({ + "name": format!("{addon_name} {short_name} 🗑️💩🚨"), + "description": format!("🚨💀⚠ Delete all files in {short_name} watchlist."), + "url": format!( + "{host_url}/streaming_provider/{secret_str}/delete_all_watchlist?provider={service}" + ), + }) +} + +/// Whether this user may access the delete-all item for `service`. +pub fn user_has_delete_all_provider(user_data: &UserData, service: &str) -> bool { + if !supports_delete_all(service) { + return false; + } + user_data + .get_provider_by_name(service) + .is_some_and(|p| p.enable_watchlist_catalogs) +} + +pub async fn delete_all_for_service( + state: &AppState, + user_data: &UserData, + service: &str, +) -> Result<(), providers::ProviderError> { + let provider = user_data.get_provider_by_name(service).ok_or_else(|| { + providers::ProviderError::api( + format!("Provider '{service}' not configured"), + "api_error.mp4", + ) + })?; + + let token = provider.token.as_deref().ok_or_else(|| { + providers::ProviderError::api("Provider token is missing", "invalid_token.mp4") + })?; + + match service { + "realdebrid" => { + providers::torrents::realdebrid::delete_all_torrents(&state.http, token).await + } + "alldebrid" => { + providers::torrents::alldebrid::delete_all_torrents(&state.http, token).await + } + "premiumize" => { + providers::torrents::premiumize::delete_all_torrents(&state.http, token).await + } + "debridlink" => { + providers::torrents::debridlink::delete_all_torrents(&state.http, token).await + } + "torbox" => providers::torrents::torbox::delete_all_torrents(&state.http, token).await, + "stremthru" => { + providers::torrents::stremthru::delete_all_torrents(&state.http, token).await + } + "offcloud" => providers::torrents::offcloud::delete_all_torrents(&state.http, token).await, + "easydebrid" => { + providers::torrents::easydebrid::delete_all_torrents(&state.http, token).await + } + "seedr" => providers::torrents::seedr::delete_all_torrents(&state.http, token).await, + "pikpak" => providers::torrents::pikpak::delete_all_torrents(&state.http, token).await, + other => Err(providers::ProviderError::api( + format!("Provider '{other}' does not support delete-all-watchlist"), + "provider_error.mp4", + )), + } +} + +pub fn delete_all_meta_response( + state: &AppState, + user_data: &UserData, + service: &str, +) -> axum::response::Response { + if !user_has_delete_all_provider(user_data, service) { + return StatusCode::NOT_FOUND.into_response(); + } + let item = MetaItem { + meta: delete_all_meta(&state.config.host_url, service), + }; + Json(item).into_response() +} + +pub fn delete_all_streams_response( + state: Arc, + user_data: &UserData, + secret_str: &str, + service: &str, +) -> axum::response::Response { + if !user_has_delete_all_provider(user_data, service) { + return StatusCode::NOT_FOUND.into_response(); + } + let stream = delete_all_stream_json( + &state.config.host_url, + secret_str, + service, + &state.config.addon_name, + ); + Json(json!({"streams": [stream]})).into_response() +} diff --git a/backend/src/routes/encrypt.rs b/backend/src/routes/encrypt.rs index f7bff85b..23146e27 100644 --- a/backend/src/routes/encrypt.rs +++ b/backend/src/routes/encrypt.rs @@ -54,7 +54,7 @@ pub async fn decrypt_handler( match raw { Ok(val) => Json(json!({"status": "success", "data": val})).into_response(), Err(e) => { - tracing::warn!("decrypt_user_data failed: {e}"); + tracing::debug!("decrypt_user_data failed: {e}"); ( StatusCode::BAD_REQUEST, Json(json!({"error": "decryption failed"})), diff --git a/backend/src/routes/manifest.rs b/backend/src/routes/manifest.rs index 2abfbc3d..cbbf3e35 100644 --- a/backend/src/routes/manifest.rs +++ b/backend/src/routes/manifest.rs @@ -311,6 +311,13 @@ async fn serve_manifest(state: Arc, user_data: UserData) -> impl IntoR cache::set_json(&state.redis, GENRES_KEY, &gv, 3600).await; g }; + let genres = { + let keyword_filters = state + .keyword_filters + .read() + .unwrap_or_else(|e| e.into_inner()); + keyword_filters.filter_genres_by_type(genres) + }; let manifest = build_manifest(&state.config, &user_data, &genres); cache::set_json(&state.redis, &cache_key, &manifest, ttl).await; diff --git a/backend/src/routes/meta.rs b/backend/src/routes/meta.rs index d28425e4..51a5e82f 100644 --- a/backend/src/routes/meta.rs +++ b/backend/src/routes/meta.rs @@ -14,6 +14,7 @@ use crate::{ stremio::{Meta, MetaItem, Video}, user_data::UserData, }, + routes::delete_all_watchlist, state::AppState, }; @@ -106,11 +107,18 @@ async fn build_meta(state: &AppState, media_type: &str, meta_id: &str) -> Option async fn serve_meta( state: Arc, - _user_data: UserData, + user_data: UserData, media_type: &str, raw_id: &str, ) -> axum::response::Response { let meta_id = raw_id.trim_end_matches(".json"); + + if media_type == "movie" { + if let Some(service) = delete_all_watchlist::parse_service(meta_id) { + return delete_all_watchlist::delete_all_meta_response(&state, &user_data, service); + } + } + let cache_key = format!("meta:{media_type}:{meta_id}"); let ttl = state.config.meta_cache_ttl; @@ -118,7 +126,32 @@ async fn serve_meta( return Json(cached).into_response(); } - let Some(meta) = build_meta(&state, media_type, meta_id).await else { + let mut meta = build_meta(&state, media_type, meta_id).await; + if meta.is_none() + && crate::scrapers::metadata::parse_import_meta_id(meta_id).is_some() + && crate::scrapers::media_resolve::ensure_media_for_import( + &state.pool, + &state.http, + meta_id, + media_type, + state.config.tmdb_api_key.as_deref(), + state.config.tvdb_api_key.as_deref(), + crate::scrapers::media_resolve::ImportMediaOverrides { + title: None, + poster: None, + background: None, + release_date: None, + year: None, + }, + None, + ) + .await + .is_some() + { + meta = build_meta(&state, media_type, meta_id).await; + } + + let Some(meta) = meta else { return StatusCode::NOT_FOUND.into_response(); }; diff --git a/backend/src/routes/metrics.rs b/backend/src/routes/metrics.rs index a5729732..461182ee 100644 --- a/backend/src/routes/metrics.rs +++ b/backend/src/routes/metrics.rs @@ -2,11 +2,13 @@ /// /// Route: GET /api/v1/metrics /// -/// Exposes live DB counts (torrents, metadata) and Redis stats +/// Exposes live DB counts (torrents, metadata) and HTTP request metrics /// in Prometheus text format. /// -/// No authentication on this endpoint — protect it at nginx/firewall level -/// or prefix it with a shared secret path if needed. +/// Enabled only when ENABLE_PROMETHEUS_METRICS=true. +/// When PROMETHEUS_METRICS_TOKEN is set the request must carry +/// `Authorization: Bearer ` — this applies even on public instances, +/// so Prometheus can be configured with `bearer_token` in scrape_configs. use std::sync::Arc; use axum::{ @@ -20,7 +22,18 @@ use std::sync::atomic::AtomicU64; use crate::state::AppState; -pub async fn handler(State(state): State>) -> Response { +pub async fn handler(State(state): State>, req: axum::extract::Request) -> Response { + if let Some(ref required) = state.config.metrics_api_key { + let provided = req + .headers() + .get(header::AUTHORIZATION) + .and_then(|v| v.to_str().ok()) + .and_then(|v| v.strip_prefix("Bearer ")) + .unwrap_or(""); + if provided != required.as_str() { + return StatusCode::UNAUTHORIZED.into_response(); + } + } let mut registry = Registry::default(); // ── DB gauges ──────────────────────────────────────────────────────────── diff --git a/backend/src/routes/mod.rs b/backend/src/routes/mod.rs index 368c55ec..86507546 100644 --- a/backend/src/routes/mod.rs +++ b/backend/src/routes/mod.rs @@ -8,6 +8,7 @@ pub mod auth; pub mod catalog; pub mod configure; pub mod content; +pub mod delete_all_watchlist; pub mod downloads; pub mod encrypt; pub mod health; @@ -456,8 +457,7 @@ pub fn router(state: Arc) -> Router { "/api/v1/content/{media_id}/likes", get(content::voting::get_content_likes), ) - // ── Prometheus metrics (protected by api_key_middleware on private instances) ─ - .route("/api/v1/metrics", get(metrics::handler)) + // ── Prometheus metrics (opt-in via ENABLE_PROMETHEUS_METRICS=true) ───── // ── Admin ───────────────────────────────────────────────────────────── .route("/api/v1/admin/cache/stats", get(admin::cache_stats)) .route("/api/v1/admin/cache/keys", get(admin::cache_keys)) @@ -714,7 +714,16 @@ pub fn router(state: Arc) -> Router { .route("/api/v1/telegram/channels/{channel_id}", delete(integrations::remove_telegram_channel).patch(integrations::update_telegram_channel)) .route("/api/v1/telegram/validate", post(integrations::validate_telegram_channel)) .route("/api/v1/telegram/login", get(integrations::telegram_login)) - .route("/api/v1/telegram/unlink", delete(integrations::telegram_unlink)) + .route("/api/v1/telegram/unlink", delete(integrations::telegram_unlink)); + + // ── Prometheus metrics (opt-in via ENABLE_PROMETHEUS_METRICS=true) ─────── + let api_router = if state.config.enable_prometheus_metrics { + api_router.route("/api/v1/metrics", get(metrics::handler)) + } else { + api_router + }; + + let api_router = api_router // ── Middleware ─────────────────────────────────────────────────────── .layer(axum::middleware::from_fn_with_state( Arc::clone(&state), diff --git a/backend/src/routes/playback_dedup.rs b/backend/src/routes/playback_dedup.rs index 3dc77e26..9b06099d 100644 --- a/backend/src/routes/playback_dedup.rs +++ b/backend/src/routes/playback_dedup.rs @@ -131,7 +131,7 @@ pub async fn reclaim_stale_lock(redis: &fred::clients::Client, cache_key: &str) if stale { let age = lock_age_secs(redis, cache_key).await; - tracing::warn!( + tracing::debug!( cache_key = %cache_key, lock_value = %raw, lock_age_secs = ?age, diff --git a/backend/src/routes/stream.rs b/backend/src/routes/stream.rs index aefc6bb2..1e1f8851 100644 --- a/backend/src/routes/stream.rs +++ b/backend/src/routes/stream.rs @@ -15,7 +15,7 @@ use crate::{ cache::{self, codec, stream_cache}, crypto, db, db::TorrentType, - models::user_data::SortingOption, + models::user_data::{provider_short_name, SortingOption}, scrapers::{orchestrator, torrent_metadata}, state::AppState, template, @@ -141,6 +141,31 @@ pub async fn movie( headers: HeaderMap, ) -> impl IntoResponse { let imdb_id = video_id.trim_end_matches(".json").to_string(); + if let Some(service) = crate::routes::delete_all_watchlist::parse_service(&imdb_id) { + let raw_user_data = if let Some(hv) = headers + .get("encoded_user_data") + .and_then(|v| v.to_str().ok()) + { + crypto::decode_encoded_user_data(hv) + .unwrap_or_else(|| serde_json::Value::Object(Default::default())) + } else { + crypto::resolve_user_data( + &secret_str, + &state.config.secret_key, + &state.pool, + &state.redis, + ) + .await + }; + let user_data: crate::models::user_data::UserData = + serde_json::from_value(raw_user_data).unwrap_or_default(); + return crate::routes::delete_all_watchlist::delete_all_streams_response( + state, + &user_data, + &secret_str, + service, + ); + } dispatch(state, secret_str, imdb_id, "movie", None, None, headers).await } @@ -817,6 +842,24 @@ async fn build_pipeline( } } + if state.config.background_search_enabled && media_id != db::MediaId(0) { + let item_key = match (media_type, season, episode) { + ("series", Some(s), Some(e)) => { + crate::scrapers::background_queue::series_item_key(media_id.0, s, e) + } + ("movie", _, _) => crate::scrapers::background_queue::movie_item_key(media_id.0), + _ => String::new(), + }; + if !item_key.is_empty() { + let queue_key = if media_type == "series" { + crate::scrapers::background_queue::SERIES_KEY + } else { + crate::scrapers::background_queue::MOVIES_KEY + }; + crate::scrapers::background_queue::enqueue(&state.redis, queue_key, &item_key).await; + } + } + let disabled = &state.config.disabled_providers; if media_id == db::MediaId(0) && related_ids.is_empty() { @@ -2040,20 +2083,6 @@ fn build_stream_context( }) } -fn provider_short_name(svc: &str) -> &str { - match svc { - "realdebrid" => "RD", - "alldebrid" => "AD", - "premiumize" => "PM", - "debridlink" => "DL", - "torbox" => "TB", - "stremthru" => "ST", - "offcloud" => "OC", - "easydebrid" => "ED", - other => other, - } -} - #[allow(clippy::too_many_arguments)] fn format_streams( torrents: &[Value], diff --git a/backend/src/routes/user_library.rs b/backend/src/routes/user_library.rs index 0dee74ff..484bb845 100644 --- a/backend/src/routes/user_library.rs +++ b/backend/src/routes/user_library.rs @@ -1243,52 +1243,20 @@ pub async fn get_missing_torrents( // Fetch all torrents for the provider. Providers that embed files in their list // (TorBox, AllDebrid, Debrid-Link) populate `raw`; RD leaves it Null and we // fetch file details separately. - macro_rules! fetch_list { - ($call:expr) => { - match $call.await { - Ok(t) => t, - Err(e) => { - tracing::warn!("get_missing_torrents {provider} list: {e}"); - return ( - StatusCode::BAD_GATEWAY, - Json(json!({"detail": format!("Provider error: {e}")})), - ) - .into_response(); - } + let all_torrents = + match crate::providers::torrents::list_downloaded_torrents(&state.http, &provider, &token) + .await + { + Ok(t) => t, + Err(e) => { + tracing::warn!("get_missing_torrents {provider} list: {e}"); + return ( + StatusCode::BAD_GATEWAY, + Json(json!({"detail": format!("Provider error: {e}")})), + ) + .into_response(); } }; - } - - let all_torrents = match provider.as_str() { - "realdebrid" => fetch_list!( - crate::providers::torrents::realdebrid::list_downloaded_torrents(&state.http, &token) - ), - "torbox" => fetch_list!( - crate::providers::torrents::torbox::list_downloaded_torrents(&state.http, &token) - ), - "alldebrid" => fetch_list!( - crate::providers::torrents::alldebrid::list_downloaded_torrents(&state.http, &token) - ), - "debridlink" => fetch_list!( - crate::providers::torrents::debridlink::list_downloaded_torrents(&state.http, &token) - ), - "premiumize" => fetch_list!( - crate::providers::torrents::premiumize::list_downloaded_torrents(&state.http, &token) - ), - "offcloud" => fetch_list!( - crate::providers::torrents::offcloud::list_downloaded_torrents(&state.http, &token) - ), - "pikpak" => fetch_list!( - crate::providers::torrents::pikpak::list_downloaded_torrents(&state.http, &token) - ), - "seedr" => fetch_list!(crate::providers::torrents::seedr::list_downloaded_torrents( - &state.http, - &token - )), - _ => { - return Json(json!({"items": [], "total": 0, "provider": provider})).into_response(); - } - }; let all_hashes: Vec = all_torrents.iter().map(|t| t.info_hash.clone()).collect(); let existing: std::collections::HashSet = @@ -1769,20 +1737,13 @@ async fn fetch_downloaded_torrents( state: &AppState, provider: &str, token: &str, -) -> Result, String> { - use crate::providers::torrents as t; - let res = match provider { - "realdebrid" => t::realdebrid::list_downloaded_torrents(&state.http, token).await, - "torbox" => t::torbox::list_downloaded_torrents(&state.http, token).await, - "alldebrid" => t::alldebrid::list_downloaded_torrents(&state.http, token).await, - "debridlink" => t::debridlink::list_downloaded_torrents(&state.http, token).await, - "premiumize" => t::premiumize::list_downloaded_torrents(&state.http, token).await, - "offcloud" => t::offcloud::list_downloaded_torrents(&state.http, token).await, - "pikpak" => t::pikpak::list_downloaded_torrents(&state.http, token).await, - "seedr" => t::seedr::list_downloaded_torrents(&state.http, token).await, - _ => return Err(format!("Unsupported provider: {provider}")), - }; - res.map_err(|e| e.to_string()) +) -> Result, String> { + if !crate::providers::torrents::supports_download_list(provider) { + return Err(format!("Unsupported provider: {provider}")); + } + crate::providers::torrents::list_downloaded_torrents(&state.http, provider, token) + .await + .map_err(|e| e.to_string()) } // ─── Remove / clear-all body shapes ───────────────────────────────────────── @@ -1988,12 +1949,10 @@ async fn process_advanced_import( } else { "MOVIE" }; - let genre = crate::parser::sports_category_to_genre(cat); crate::scrapers::media_resolve::find_or_create_sports_stub( &state.pool, title, parsed.year, - genre, item.poster.as_deref(), db_type, ) diff --git a/backend/src/routes/watchlist.rs b/backend/src/routes/watchlist.rs index 4f9ca8e5..defaef0a 100644 --- a/backend/src/routes/watchlist.rs +++ b/backend/src/routes/watchlist.rs @@ -21,7 +21,9 @@ use chrono::Utc; use hmac::{Hmac, KeyInit, Mac}; use sha2::Sha256; -use crate::{crypto, models::user_data::UserData, providers, state::AppState}; +use crate::{ + crypto, models::user_data::UserData, providers, routes::delete_all_watchlist, state::AppState, +}; // ─── Providers that support watchlist (cached hash lookup) ─────────────────── @@ -255,11 +257,17 @@ pub async fn get_watchlist( .into_response() } +#[derive(Deserialize)] +pub struct DeleteAllQuery { + provider: Option, +} + pub async fn delete_all_handler( Path(secret_str): Path, + Query(params): Query, State(state): State>, ) -> Response { - match dispatch(&state, &secret_str).await { + match dispatch(&state, &secret_str, params.provider.as_deref()).await { Ok(()) => redirect(format!( "{}/static/exceptions/watchlist_deleted.mp4", state.config.host_url @@ -272,7 +280,11 @@ pub async fn delete_all_handler( } } -async fn dispatch(state: &AppState, secret_str: &str) -> Result<(), providers::ProviderError> { +async fn dispatch( + state: &AppState, + secret_str: &str, + provider_name: Option<&str>, +) -> Result<(), providers::ProviderError> { let raw = crypto::resolve_user_data( secret_str, &state.config.secret_key, @@ -282,40 +294,18 @@ async fn dispatch(state: &AppState, secret_str: &str) -> Result<(), providers::P .await; let user_data: UserData = serde_json::from_value(raw).unwrap_or_default(); - let provider = user_data.get_primary_provider().ok_or_else(|| { - providers::ProviderError::api("No streaming provider configured", "api_error.mp4") - })?; - - let token = provider.token.as_deref().ok_or_else(|| { - providers::ProviderError::api("Provider token is missing", "invalid_token.mp4") - })?; + let service = if let Some(name) = provider_name.filter(|s| !s.is_empty()) { + name.to_string() + } else { + user_data + .get_primary_provider() + .map(|p| p.service.clone()) + .ok_or_else(|| { + providers::ProviderError::api("No streaming provider configured", "api_error.mp4") + })? + }; - match provider.service.as_str() { - "realdebrid" => { - providers::torrents::realdebrid::delete_all_torrents(&state.http, token).await - } - "alldebrid" => { - providers::torrents::alldebrid::delete_all_torrents(&state.http, token).await - } - "premiumize" => { - providers::torrents::premiumize::delete_all_torrents(&state.http, token).await - } - "debridlink" => { - providers::torrents::debridlink::delete_all_torrents(&state.http, token).await - } - "torbox" => providers::torrents::torbox::delete_all_torrents(&state.http, token).await, - "stremthru" => { - providers::torrents::stremthru::delete_all_torrents(&state.http, token).await - } - "offcloud" => providers::torrents::offcloud::delete_all_torrents(&state.http, token).await, - "easydebrid" => { - providers::torrents::easydebrid::delete_all_torrents(&state.http, token).await - } - other => Err(providers::ProviderError::api( - format!("Provider '{other}' does not support delete-all-watchlist"), - "provider_error.mp4", - )), - } + delete_all_watchlist::delete_all_for_service(state, &user_data, &service).await } fn redirect(url: String) -> Response { diff --git a/backend/src/scrapers/background_queue.rs b/backend/src/scrapers/background_queue.rs new file mode 100644 index 00000000..9f41ad5b --- /dev/null +++ b/backend/src/scrapers/background_queue.rs @@ -0,0 +1,30 @@ +use std::time::{SystemTime, UNIX_EPOCH}; + +use fred::prelude::HashesInterface; + +pub const MOVIES_KEY: &str = "background_search:movies"; +pub const SERIES_KEY: &str = "background_search:series"; +pub const PROCESSING_KEY: &str = "background_search:processing"; + +pub fn movie_item_key(media_id: i32) -> String { + media_id.to_string() +} + +pub fn series_item_key(media_id: i32, season: i32, episode: i32) -> String { + format!("{media_id}:{season}:{episode}") +} + +/// Idempotently enqueue a media item for background re-scraping. +pub async fn enqueue(redis: &fred::clients::Client, key: &str, item_key: &str) { + let now = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap_or_default() + .as_secs_f64(); + let entry = serde_json::json!({ + "last_scrape": 0.0, + "added_at": now, + }); + let _ = redis + .hset::<(), _, _>(key, (item_key, entry.to_string())) + .await; +} diff --git a/backend/src/scrapers/easynews.rs b/backend/src/scrapers/easynews.rs index d336c96c..5ee2708f 100644 --- a/backend/src/scrapers/easynews.rs +++ b/backend/src/scrapers/easynews.rs @@ -62,7 +62,7 @@ pub async fn scrape( match search(client, username, password, &query, max_results).await { Err(SearchError::AuthFailed) => { // 401 — credentials are invalid; no point trying the remaining queries. - tracing::warn!( + tracing::debug!( "easynews: credentials rejected (401) for user '{}' — skipping all queries", username ); @@ -406,7 +406,7 @@ pub async fn scrape_with_credentials( for (query, max_results) in queries { let raw_items = match search(client, username, password, &query, max_results).await { Err(SearchError::AuthFailed) => { - tracing::warn!( + tracing::debug!( "easynews: credentials rejected (401) for user '{}' — skipping all queries", username ); diff --git a/backend/src/scrapers/indexer_credentials.rs b/backend/src/scrapers/indexer_credentials.rs new file mode 100644 index 00000000..88266363 --- /dev/null +++ b/backend/src/scrapers/indexer_credentials.rs @@ -0,0 +1,137 @@ +use crate::config::AppConfig; +use crate::models::user_data::{IndexerConfig, IndexerInstanceConfig}; + +fn non_empty(value: &Option) -> Option<&str> { + value.as_deref().map(str::trim).filter(|s| !s.is_empty()) +} + +/// Resolve URL + API key for a Prowlarr/Jackett profile block. +/// +/// Mirrors Python `scraper_tasks._create_prowlarr_task` / `_create_jackett_task`: +/// - `enabled=false` → no scrape (no global fallback) +/// - `use_global=true` → global env credentials (requires global scrape flag) +/// - `use_global=false` → user URL + user API key only (both required, no mixing) +fn resolve_instance( + instance: &IndexerInstanceConfig, + global_url: &Option, + global_key: &Option, + global_scrape_enabled: bool, +) -> Option<(String, String)> { + if !instance.enabled { + return None; + } + + if instance.use_global { + if !global_scrape_enabled { + return None; + } + let url = non_empty(global_url)?.to_string(); + let key = non_empty(global_key)?.to_string(); + return Some((url, key)); + } + + let url = non_empty(&instance.url)?.to_string(); + let key = non_empty(&instance.api_key)?.to_string(); + Some((url, key)) +} + +/// Resolve Prowlarr credentials from the user profile (`ic.pr`) or global env. +pub fn resolve_prowlarr_credentials( + ic: &IndexerConfig, + cfg: &AppConfig, +) -> Option<(String, String)> { + if let Some(instance) = &ic.prowlarr { + return resolve_instance( + instance, + &cfg.prowlarr_url, + &cfg.prowlarr_api_key, + cfg.is_scrap_from_prowlarr, + ); + } + + if !cfg.is_scrap_from_prowlarr { + return None; + } + let url = non_empty(&cfg.prowlarr_url)?.to_string(); + let key = non_empty(&cfg.prowlarr_api_key)?.to_string(); + Some((url, key)) +} + +/// Resolve Jackett credentials from the user profile (`ic.jk`) or global env. +pub fn resolve_jackett_credentials( + ic: &IndexerConfig, + cfg: &AppConfig, +) -> Option<(String, String)> { + if let Some(instance) = &ic.jackett { + return resolve_instance( + instance, + &cfg.jackett_url, + &cfg.jackett_api_key, + cfg.is_scrap_from_jackett, + ); + } + + if !cfg.is_scrap_from_jackett { + return None; + } + let url = non_empty(&cfg.jackett_url)?.to_string(); + let key = non_empty(&cfg.jackett_api_key)?.to_string(); + Some((url, key)) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn custom_instance_requires_api_key() { + let instance = IndexerInstanceConfig { + enabled: true, + use_global: false, + url: Some("http://127.0.0.1:9696".into()), + api_key: None, + }; + assert!(resolve_instance( + &instance, + &Some("http://global:9696".into()), + &Some("global-key".into()), + true, + ) + .is_none()); + } + + #[test] + fn use_global_ignores_profile_url_and_key() { + let instance = IndexerInstanceConfig { + enabled: true, + use_global: true, + url: Some("http://ignored:9696".into()), + api_key: Some("ignored".into()), + }; + assert_eq!( + resolve_instance( + &instance, + &Some("http://prowlarr:9696".into()), + &Some("global-key".into()), + true, + ), + Some(("http://prowlarr:9696".into(), "global-key".into())) + ); + } + + #[test] + fn disabled_instance_does_not_fall_back_to_global() { + let instance = IndexerInstanceConfig { + enabled: false, + use_global: true, + ..IndexerInstanceConfig::default() + }; + assert!(resolve_instance( + &instance, + &Some("http://prowlarr:9696".into()), + &Some("global-key".into()), + true, + ) + .is_none()); + } +} diff --git a/backend/src/scrapers/jackett.rs b/backend/src/scrapers/jackett.rs index cd6bcda9..697b3b20 100644 --- a/backend/src/scrapers/jackett.rs +++ b/backend/src/scrapers/jackett.rs @@ -1,6 +1,10 @@ +use std::collections::{HashMap, HashSet}; +use std::time::Duration; + +use quick_xml::events::Event; +use quick_xml::Reader; use reqwest::Client; use serde::Deserialize; -use std::time::Duration; use crate::{ parser, @@ -16,7 +20,8 @@ use crate::{ pub(crate) const RESULT_PROCESS_CONCURRENCY: usize = 5; -// ─── Jackett JSON response shapes ───────────────────────────────────────────── +const MOVIE_CATEGORY_IDS: &[i64] = &[2000, 2010, 2020, 2030, 2040, 2045, 2050, 2060, 2070]; +const SERIES_CATEGORY_IDS: &[i64] = &[5000, 5010, 5020, 5030, 5040, 5045, 5050, 5060, 5070]; #[derive(Debug, Deserialize)] struct JackettResponse { @@ -48,48 +53,103 @@ pub(crate) struct JackettResult { category_desc: Option, } -// ─── Public entry point ─────────────────────────────────────────────────────── +#[derive(Debug, Clone)] +pub struct JackettIndexer { + pub id: String, + pub name: String, + pub categories: Vec, + pub supports_imdb_movie: bool, + pub supports_imdb_tv: bool, + pub supports_search: bool, +} + +pub async fn list_healthy_indexers( + client: &Client, + base_url: &str, + api_key: &str, +) -> Vec { + let url = format!( + "{}/api/v2.0/indexers/!status:failing/results/torznab/api", + base_url.trim_end_matches('/') + ); + let resp = match client + .get(url) + .query(&[ + ("apikey", api_key), + ("t", "indexers"), + ("configured", "true"), + ]) + .timeout(Duration::from_secs(15)) + .send() + .await + { + Ok(r) if r.status().is_success() => r, + Ok(r) => { + tracing::debug!("jackett: indexer list HTTP {}", r.status()); + return vec![]; + } + Err(e) => { + tracing::debug!("jackett: indexer list failed: {e}"); + return vec![]; + } + }; + + let xml = resp.text().await.unwrap_or_default(); + parse_jackett_indexers(&xml) +} #[allow(clippy::too_many_arguments)] -pub async fn scrape( +pub async fn scrape_indexer( client: &Client, base_url: &str, api_key: &str, + idx: &JackettIndexer, meta: &SearchMeta, media_type: &str, season: Option, episode: Option, max_process: usize, - max_process_time: std::time::Duration, - query_timeout: std::time::Duration, + query_timeout: Duration, + title_queries: &[String], + deadline: tokio::time::Instant, ) -> Vec { - let imdb_id = meta.imdb_id.as_deref().unwrap_or(""); - let is_series = media_type == "series"; + if tokio::time::Instant::now() >= deadline { + return vec![]; + } - let query = if !imdb_id.is_empty() { - format!("{{IMDbId:{imdb_id}}}") - } else { - build_title_query(&meta.title, meta.year, media_type, season, episode) - }; + let queries = build_queries(meta, media_type, season, episode, title_queries); + if queries.is_empty() { + return vec![]; + } - let categories: &[&str] = if is_series { - &[ - "5000", "5010", "5020", "5030", "5040", "5045", "5050", "5060", "5070", - ] + let categories: Vec = if media_type == "series" { + SERIES_CATEGORY_IDS.to_vec() } else { - &[ - "2000", "2010", "2020", "2030", "2040", "2045", "2050", "2060", "2070", - ] + MOVIE_CATEGORY_IDS.to_vec() }; - let mut params: Vec<(&str, String)> = vec![("apikey", api_key.to_string()), ("Query", query)]; - for cat in categories { - params.push(("Category[]", cat.to_string())); - } + let mut all_results: Vec = Vec::new(); + let mut seen: HashSet = HashSet::new(); + + for query in queries { + if tokio::time::Instant::now() >= deadline { + break; + } + + let mut params: Vec<(&str, String)> = vec![ + ("apikey", api_key.to_string()), + ("Query", query), + ("Tracker[]", idx.id.clone()), + ]; + for cat in &categories { + params.push(("Category[]", cat.to_string())); + } - match tokio::time::timeout(max_process_time, async { let resp = match client - .get(format!("{base_url}/api/v2.0/indexers/all/results")) + .get(format!( + "{}/api/v2.0/indexers/all/results", + base_url.trim_end_matches('/') + )) .query(¶ms) .timeout(query_timeout) .send() @@ -97,39 +157,253 @@ pub async fn scrape( { Ok(r) => r, Err(e) => { - tracing::debug!("jackett request failed: {e}"); - return vec![]; + tracing::debug!("jackett indexer {} request failed: {e}", idx.name); + continue; } }; let body: JackettResponse = match resp.json().await { Ok(v) => v, Err(e) => { - tracing::debug!("jackett response parse failed: {e}"); - return vec![]; + tracing::debug!("jackett indexer {} parse failed: {e}", idx.name); + continue; } }; - let items: Vec = body.results.into_iter().take(max_process).collect(); - use futures::stream::{self, StreamExt}; - stream::iter(items) - .map(|r| process_result(client, r, media_type, season, episode, query_timeout)) - .buffer_unordered(RESULT_PROCESS_CONCURRENCY) - .filter_map(|result| async move { result }) - .collect() - .await - }) - .await - { - Ok(r) => r, - Err(_) => { - tracing::debug!("jackett: max_process_time exceeded"); - vec![] + for result in body.results { + let dedupe_key = result + .info_hash + .clone() + .or_else(|| result.guid.clone()) + .unwrap_or_default() + .to_lowercase(); + if dedupe_key.is_empty() || seen.insert(dedupe_key) { + all_results.push(result); + if all_results.len() >= max_process { + break; + } + } + } + if all_results.len() >= max_process { + break; + } + } + + let items: Vec = all_results.into_iter().take(max_process).collect(); + use futures::stream::{self, StreamExt}; + stream::iter(items) + .map(|r| process_result(client, r, media_type, season, episode, query_timeout)) + .buffer_unordered(RESULT_PROCESS_CONCURRENCY) + .filter_map(|result| async move { result }) + .collect() + .await +} + +#[allow(clippy::too_many_arguments)] +pub async fn scrape( + client: &Client, + base_url: &str, + api_key: &str, + meta: &SearchMeta, + media_type: &str, + season: Option, + episode: Option, + max_process: usize, + max_process_time: std::time::Duration, + query_timeout: std::time::Duration, + title_queries: &[String], +) -> Vec { + let indexers = list_healthy_indexers(client, base_url, api_key).await; + if indexers.is_empty() { + return vec![]; + } + + let deadline = tokio::time::Instant::now() + max_process_time; + let mut all = Vec::new(); + for idx in &indexers { + if tokio::time::Instant::now() >= deadline { + break; + } + let mut batch = scrape_indexer( + client, + base_url, + api_key, + idx, + meta, + media_type, + season, + episode, + max_process, + query_timeout, + title_queries, + deadline, + ) + .await; + all.append(&mut batch); + } + all +} + +fn build_queries( + meta: &SearchMeta, + media_type: &str, + season: Option, + episode: Option, + title_queries: &[String], +) -> Vec { + let imdb_id = meta.imdb_id.as_deref().unwrap_or(""); + let mut queries = Vec::new(); + if !imdb_id.is_empty() { + queries.push(format!("{{IMDbId:{imdb_id}}}")); + } + if !title_queries.is_empty() { + queries.extend(title_queries.iter().cloned()); + } else if imdb_id.is_empty() { + queries.push(build_title_query( + &meta.title, + meta.year, + media_type, + season, + episode, + )); + } + queries +} + +fn parse_jackett_indexers(xml: &str) -> Vec { + let mut reader = Reader::from_str(xml); + reader.config_mut().trim_text(true); + let mut buf = Vec::new(); + let mut indexers = Vec::new(); + let mut current_id: Option = None; + let mut current_name = String::new(); + let mut current_categories: Vec = Vec::new(); + let mut search_caps: HashSet = HashSet::new(); + let mut movie_params: HashSet = HashSet::new(); + let mut tv_params: HashSet = HashSet::new(); + + let flush = |indexers: &mut Vec, + id: Option, + name: String, + categories: Vec, + search_caps: &HashSet, + movie_params: &HashSet, + tv_params: &HashSet| { + let Some(id) = id else { return }; + if id.is_empty() { + return; } + indexers.push(JackettIndexer { + id, + name: if name.is_empty() { + "unknown".into() + } else { + name + }, + categories, + supports_imdb_movie: search_caps.contains("movie-search") + && movie_params + .iter() + .any(|p| p.eq_ignore_ascii_case("imdbid")), + supports_imdb_tv: search_caps.contains("tv-search") + && tv_params.iter().any(|p| p.eq_ignore_ascii_case("imdbid")), + supports_search: search_caps.contains("search"), + }); + }; + + loop { + match reader.read_event_into(&mut buf) { + Ok(Event::Start(ref e)) | Ok(Event::Empty(ref e)) => { + let tag = xml_tag_name(e.name().as_ref()); + let attrs = xml_attrs(e); + match tag.as_str() { + "indexer" => { + flush( + &mut indexers, + current_id.take(), + std::mem::take(&mut current_name), + std::mem::take(&mut current_categories), + &search_caps, + &movie_params, + &tv_params, + ); + search_caps.clear(); + movie_params.clear(); + tv_params.clear(); + current_id = attrs.get("id").cloned(); + } + "category" => { + if let Some(id) = attrs.get("id").and_then(|s| s.parse().ok()) { + current_categories.push(id); + } + } + "subcat" => { + if let Some(id) = attrs.get("id").and_then(|s| s.parse().ok()) { + current_categories.push(id); + } + } + "search" | "tv-search" | "movie-search" + if attrs.get("available").map(String::as_str) == Some("yes") => + { + search_caps.insert(tag.clone()); + if let Some(params) = attrs.get("supportedParams") { + let set = match tag.as_str() { + "movie-search" => &mut movie_params, + "tv-search" => &mut tv_params, + _ => &mut movie_params, + }; + for p in params.split(',') { + set.insert(p.trim().to_string()); + } + } + } + _ => {} + } + } + Ok(Event::Text(ref e)) if current_id.is_some() && current_name.is_empty() => { + if let Ok(text) = e.decode() { + let s = text.trim(); + if !s.is_empty() { + current_name = s.to_string(); + } + } + } + Ok(Event::Eof) => break, + Err(_) => break, + _ => {} + } + buf.clear(); } + + flush( + &mut indexers, + current_id.take(), + current_name, + current_categories, + &search_caps, + &movie_params, + &tv_params, + ); + + indexers.sort_by(|a, b| a.name.cmp(&b.name)); + indexers } -// ─── Helpers ────────────────────────────────────────────────────────────────── +fn xml_tag_name(name: &[u8]) -> String { + std::str::from_utf8(name).unwrap_or("").to_lowercase() +} + +fn xml_attrs(e: &quick_xml::events::BytesStart) -> HashMap { + e.attributes() + .filter_map(|a| a.ok()) + .map(|a| { + ( + String::from_utf8_lossy(a.key.as_ref()).to_string(), + String::from_utf8_lossy(&a.value).to_string(), + ) + }) + .collect() +} fn build_title_query( title: &str, diff --git a/backend/src/scrapers/media_resolve.rs b/backend/src/scrapers/media_resolve.rs index d46a57c3..9699fd19 100644 --- a/backend/src/scrapers/media_resolve.rs +++ b/backend/src/scrapers/media_resolve.rs @@ -1121,9 +1121,10 @@ pub async fn search_meta_for_scraped( } pub async fn store_external_id(pool: &PgPool, media_id: i32, provider: &str, external_id: &str) { + // media_external_id.created_at is NOT NULL with no column default — must supply NOW(). let _ = sqlx::query( - "INSERT INTO media_external_id (media_id, provider, external_id) \ - VALUES ($1, $2, $3) ON CONFLICT DO NOTHING", + "INSERT INTO media_external_id (media_id, provider, external_id, created_at) \ + VALUES ($1, $2, $3, NOW()) ON CONFLICT DO NOTHING", ) .bind(media_id) .bind(provider) @@ -1134,6 +1135,16 @@ pub async fn store_external_id(pool: &PgPool, media_id: i32, provider: &str, ext pub async fn link_to_catalogs(pool: &PgPool, media_id: i32, catalog_ids: &[&str]) { for catalog_name in catalog_ids { + // Ensure the catalog row exists before linking — the catalog table has no seed data + // on fresh installs, so a pure SELECT would always miss. + let _ = sqlx::query( + "INSERT INTO catalog (name, display_name, is_system, display_order) \ + VALUES ($1, $1, true, 0) ON CONFLICT (name) DO NOTHING", + ) + .bind(catalog_name) + .execute(pool) + .await; + let _ = sqlx::query( "INSERT INTO media_catalog_link (media_id, catalog_id) \ SELECT $1, c.id FROM catalog c WHERE c.name = $2 \ @@ -1156,7 +1167,6 @@ pub async fn find_or_create_sports_stub( pool: &PgPool, title: &str, year: Option, - genre_name: &str, poster_url: Option<&str>, media_type: &str, ) -> Option { @@ -1188,7 +1198,6 @@ pub async fn find_or_create_sports_stub( }; if let Some((id,)) = row { - link_genre(pool, id, genre_name).await; return Some(id); } @@ -1206,7 +1215,6 @@ pub async fn find_or_create_sports_stub( if let Some((id, existing_title)) = fuzzy { if crate::parser::similarity_ratio(title, &existing_title) >= 70 { - link_genre(pool, id, genre_name).await; return Some(id); } } @@ -1261,8 +1269,6 @@ pub async fn find_or_create_sports_stub( .await; } - link_genre(pool, media_id, genre_name).await; - debug!("media_resolve: created sports stub {media_id} ({media_type}) for '{title}'"); Some(media_id) } diff --git a/backend/src/scrapers/mod.rs b/backend/src/scrapers/mod.rs index 643b1a60..086d5faf 100644 --- a/backend/src/scrapers/mod.rs +++ b/backend/src/scrapers/mod.rs @@ -1,7 +1,9 @@ pub mod anilist; +pub mod background_queue; pub mod browser; pub mod easynews; pub mod fetcher; +pub mod indexer_credentials; pub mod jackett; pub mod kitsu; pub mod media_resolve; @@ -17,6 +19,7 @@ pub mod public_usenet; pub mod rss; pub mod source_health; pub mod telegram; +pub mod title_queries; pub mod torbox_search; pub mod torrent_metadata; pub mod torrentio; diff --git a/backend/src/scrapers/orchestrator.rs b/backend/src/scrapers/orchestrator.rs index 9e7e910f..87039550 100644 --- a/backend/src/scrapers/orchestrator.rs +++ b/backend/src/scrapers/orchestrator.rs @@ -1,4 +1,5 @@ use std::sync::Arc; +use std::time::Duration; use chrono::Utc; use fred::prelude::*; @@ -9,12 +10,56 @@ use crate::{ scrapers::{ easynews, jackett, mediafusion, newznab, persist, prowlarr, public_indexers, public_usenet, source_health::{self, HealthGateConfig}, - telegram, torbox_search, torrentio, torznab, zilean, ScrapedStream, ScrapedUsenetStream, - SearchMeta, + telegram, title_queries, torbox_search, torrentio, torznab, zilean, ScrapedStream, + ScrapedUsenetStream, SearchMeta, }, state::AppState, }; +pub(crate) struct FanOutOpts { + pub byparr_url: Option, + pub bypass_ttl: bool, + pub max_process_override: Option<(usize, Duration)>, + pub query_timeout_override: Option, + pub title_search: bool, +} + +fn live_fan_out_opts(cfg: &crate::config::AppConfig) -> FanOutOpts { + FanOutOpts { + byparr_url: None, + bypass_ttl: false, + max_process_override: None, + query_timeout_override: None, + title_search: cfg.prowlarr_live_title_search || cfg.jackett_live_title_search, + } +} + +fn background_fan_out_opts(cfg: &crate::config::AppConfig) -> FanOutOpts { + FanOutOpts { + byparr_url: cfg.byparr_url.clone(), + bypass_ttl: true, + max_process_override: Some(( + cfg.background_max_process, + Duration::from_secs(cfg.background_max_process_time), + )), + query_timeout_override: Some(Duration::from_secs(cfg.background_query_timeout)), + title_search: true, + } +} + +fn build_title_queries( + meta: &SearchMeta, + media_type: &str, + season: Option, + episode: Option, +) -> Vec { + match (media_type, season, episode) { + ("series", Some(s), Some(e)) => title_queries::series_title_queries(&meta.title, s, e), + ("movie", _, _) => title_queries::movie_title_queries(&meta.title, meta.year), + _ => Vec::new(), + } +} + /// Run all live scrapers for the given media item. /// /// Returns immediately with an empty vec if: @@ -41,7 +86,7 @@ pub async fn run( } } - let results = fan_out( + let results = fan_out_with_opts( state, user_data, meta, @@ -49,6 +94,7 @@ pub async fn run( season, episode, &state.redis, + &live_fan_out_opts(&state.config), ) .await; @@ -129,7 +175,7 @@ pub async fn run_forced( } } - let results = fan_out( + let results = fan_out_with_opts( state, user_data, meta, @@ -137,6 +183,7 @@ pub async fn run_forced( season, episode, &state.redis, + &live_fan_out_opts(&state.config), ) .await; @@ -285,6 +332,43 @@ pub async fn run_usenet( results } +/// Full torrent + usenet fan-out for background worker re-scrapes. +/// +/// Bypasses live-search TTL gates and Redis lock; uses deeper processing limits +/// and Byparr for public indexers. Invalidates stream cache when done. +pub async fn run_background( + state: &AppState, + user_data: &UserData, + meta: &SearchMeta, + media_type: &str, + season: Option, + episode: Option, + scope: &str, +) { + let opts = background_fan_out_opts(&state.config); + let results = fan_out_with_opts( + state, + user_data, + meta, + media_type, + season, + episode, + &state.redis, + &opts, + ) + .await; + + let mut seen = std::collections::HashSet::new(); + let deduped: Vec = results + .into_iter() + .filter(|s| seen.insert(s.info_hash.clone())) + .collect(); + + persist::write_back(&deduped, &state.pool, meta, media_type, season, episode).await; + run_usenet(state, user_data, meta, media_type, season, episode, scope).await; + invalidate_stream_cache(&state.redis, meta, media_type, season, episode, scope).await; +} + // ─── Internal helpers ───────────────────────────────────────────────────────── fn lock_key(meta: &SearchMeta, season: Option, episode: Option) -> String { @@ -334,7 +418,7 @@ async fn try_acquire_lock( Ok(result.is_some()) } -async fn fan_out( +async fn fan_out_with_opts( state: &AppState, user_data: &UserData, meta: &SearchMeta, @@ -342,6 +426,7 @@ async fn fan_out( season: Option, episode: Option, redis: &fred::clients::Client, + opts: &FanOutOpts, ) -> Vec { use fred::prelude::SortedSetsInterface; @@ -395,12 +480,54 @@ async fn fan_out( } let is_stale = |scraper_id: &str, ttl: i64| -> bool { + if opts.bypass_ttl { + return true; + } match last_scraped.get(scraper_id) { Some(&ts) => (now - ts) as i64 >= ttl, None => true, // never scraped → stale → should scrape } }; + let title_queries = if opts.title_search { + build_title_queries(&meta, media_type, season, episode) + } else { + Vec::new() + }; + let title_queries = Arc::new(title_queries); + + let (prowlarr_max_process, prowlarr_max_time, prowlarr_query_timeout) = + if let Some((max_process, max_time)) = opts.max_process_override { + ( + max_process, + max_time, + opts.query_timeout_override + .unwrap_or_else(|| Duration::from_secs(cfg.prowlarr_search_query_timeout)), + ) + } else { + ( + cfg.prowlarr_immediate_max_process, + Duration::from_secs(cfg.prowlarr_immediate_max_process_time), + Duration::from_secs(cfg.prowlarr_search_query_timeout), + ) + }; + + let (jackett_max_process, jackett_max_time, jackett_query_timeout) = + if let Some((max_process, max_time)) = opts.max_process_override { + ( + max_process, + max_time, + opts.query_timeout_override + .unwrap_or_else(|| Duration::from_secs(cfg.jackett_search_query_timeout)), + ) + } else { + ( + cfg.jackett_immediate_max_process, + Duration::from_secs(cfg.jackett_immediate_max_process_time), + Duration::from_secs(cfg.jackett_search_query_timeout), + ) + }; + // Health gate config for public indexers let health_gate = HealthGateConfig { redis: redis.clone(), @@ -437,116 +564,133 @@ async fn fan_out( } // ── Prowlarr (global config, or user-overridden URL/key) ────────────────── - if cfg.is_scrap_from_prowlarr - && cfg.prowlarr_live_title_search - && is_stale("prowlarr", cfg.prowlarr_search_ttl) - { - let prowlarr_url = ic - .prowlarr - .as_ref() - .and_then(|p| if p.use_global { None } else { p.url.clone() }) - .or_else(|| cfg.prowlarr_url.clone()); - let prowlarr_key = ic - .prowlarr - .as_ref() - .and_then(|p| { - if p.use_global { - None - } else { - p.api_key.clone() + if is_stale("prowlarr", cfg.prowlarr_search_ttl) { + if let Some((url, key)) = + crate::scrapers::indexer_credentials::resolve_prowlarr_credentials(&ic, &cfg) + { + let indexers = prowlarr::list_healthy_indexers(&http, &url, &key).await; + if !indexers.is_empty() { + let privacy_by_id = prowlarr::fetch_indexer_privacy_map(&http, &url, &key).await; + let privacy_by_id = Arc::new(privacy_by_id); + let deadline = Arc::new(tokio::time::Instant::now() + prowlarr_max_time); + for idx in indexers { + let http = http.clone(); + let url = url.clone(); + let key = key.clone(); + let meta = meta.clone(); + let mt = media_type.to_string(); + let title_queries = title_queries.clone(); + let privacy_by_id = privacy_by_id.clone(); + let deadline = deadline.clone(); + let indexer_name = idx.name.clone(); + set.spawn(async move { + let start = Utc::now(); + let t = std::time::Instant::now(); + let streams = prowlarr::scrape_indexer( + &http, + &url, + &key, + &idx, + &meta, + &mt, + season, + episode, + prowlarr_max_process, + prowlarr_query_timeout, + title_queries.as_slice(), + &privacy_by_id, + *deadline, + ) + .await; + tracing::debug!( + "prowlarr indexer {indexer_name}: {} streams", + streams.len() + ); + ("prowlarr", streams, start, t.elapsed().as_secs_f64()) + }); } - }) - .or_else(|| cfg.prowlarr_api_key.clone()); - if let (Some(url), Some(key)) = (prowlarr_url, prowlarr_key) { + spawned_scrapers.push("prowlarr"); + } + } + } + + // ── Jackett (global config, or user-overridden) ─────────────────────────── + if cfg.is_scrap_from_jackett && is_stale("jackett", cfg.jackett_search_ttl) { + if let Some((url, key)) = + crate::scrapers::indexer_credentials::resolve_jackett_credentials(&ic, &cfg) + { + let indexers = jackett::list_healthy_indexers(&http, &url, &key).await; + if !indexers.is_empty() { + let deadline = Arc::new(tokio::time::Instant::now() + jackett_max_time); + for idx in indexers { + let http = http.clone(); + let url = url.clone(); + let key = key.clone(); + let meta = meta.clone(); + let mt = media_type.to_string(); + let title_queries = title_queries.clone(); + let deadline = deadline.clone(); + let indexer_name = idx.name.clone(); + set.spawn(async move { + let start = Utc::now(); + let t = std::time::Instant::now(); + let streams = jackett::scrape_indexer( + &http, + &url, + &key, + &idx, + &meta, + &mt, + season, + episode, + jackett_max_process, + jackett_query_timeout, + title_queries.as_slice(), + *deadline, + ) + .await; + tracing::debug!( + "jackett indexer {indexer_name}: {} streams", + streams.len() + ); + ("jackett", streams, start, t.elapsed().as_secs_f64()) + }); + } + spawned_scrapers.push("jackett"); + } + } + } + + // ── Zilean (search + filtered endpoints in parallel) ────────────────────── + if cfg.is_scrap_from_zilean && is_stale("zilean", cfg.zilean_search_ttl) { + let url = cfg.zilean_url.clone(); + { let http = http.clone(); let meta = meta.clone(); let mt = media_type.to_string(); - let max_process = cfg.prowlarr_immediate_max_process; - let max_process_time = - std::time::Duration::from_secs(cfg.prowlarr_immediate_max_process_time); - let query_timeout = std::time::Duration::from_secs(cfg.prowlarr_search_query_timeout); + let url = url.clone(); set.spawn(async move { let start = Utc::now(); let t = std::time::Instant::now(); - let streams = prowlarr::scrape( - &http, - &url, - &key, - &meta, - &mt, - season, - episode, - max_process, - max_process_time, - query_timeout, - ) - .await; - ("prowlarr", streams, start, t.elapsed().as_secs_f64()) + let streams = zilean::scrape_search(&http, &url, &meta, &mt, season, episode).await; + tracing::debug!("zilean search: {} streams", streams.len()); + ("zilean", streams, start, t.elapsed().as_secs_f64()) }); - spawned_scrapers.push("prowlarr"); } - } - - // ── Jackett (global config, or user-overridden) ─────────────────────────── - if cfg.is_scrap_from_jackett && is_stale("jackett", cfg.jackett_search_ttl) { - let jackett_url = ic - .jackett - .as_ref() - .and_then(|j| if j.use_global { None } else { j.url.clone() }) - .or_else(|| cfg.jackett_url.clone()); - let jackett_key = ic - .jackett - .as_ref() - .and_then(|j| { - if j.use_global { - None - } else { - j.api_key.clone() - } - }) - .or_else(|| cfg.jackett_api_key.clone()); - if let (Some(url), Some(key)) = (jackett_url, jackett_key) { + { let http = http.clone(); let meta = meta.clone(); let mt = media_type.to_string(); - let max_process = cfg.jackett_immediate_max_process; - let max_process_time = - std::time::Duration::from_secs(cfg.jackett_immediate_max_process_time); - let query_timeout = std::time::Duration::from_secs(cfg.jackett_search_query_timeout); + let url = url.clone(); set.spawn(async move { let start = Utc::now(); let t = std::time::Instant::now(); - let streams = jackett::scrape( - &http, - &url, - &key, - &meta, - &mt, - season, - episode, - max_process, - max_process_time, - query_timeout, - ) - .await; - ("jackett", streams, start, t.elapsed().as_secs_f64()) + let streams = + zilean::scrape_filtered(&http, &url, &meta, &mt, season, episode).await; + tracing::debug!("zilean filtered: {} streams", streams.len()); + ("zilean", streams, start, t.elapsed().as_secs_f64()) }); - spawned_scrapers.push("jackett"); } - } - - // ── Zilean ───────────────────────────────────────────────────────────────── - if cfg.is_scrap_from_zilean && is_stale("zilean", cfg.zilean_search_ttl) { - let url = cfg.zilean_url.clone(); - let http = http.clone(); - let meta = meta.clone(); - let mt = media_type.to_string(); - set.spawn(async move { - let start = Utc::now(); - let t = std::time::Instant::now(); - let streams = zilean::scrape(&http, &url, &meta, &mt, season, episode).await; - ("zilean", streams, start, t.elapsed().as_secs_f64()) - }); spawned_scrapers.push("zilean"); } @@ -644,8 +788,8 @@ async fn fan_out( let mt = media_type.to_string(); // CF bypass (byparr) must NOT run during live API requests — it launches // Chromium sessions and causes severe CPU/latency spikes. Background workers - // call public_indexers::scrape directly with the full byparr URL. - let byparr: Option = None; + // pass the configured byparr URL via FanOutOpts. + let byparr = opts.byparr_url.clone(); let sites = cfg.public_indexers_live_search_sites.clone(); let hg = health_gate.clone(); set.spawn(async move { diff --git a/backend/src/scrapers/prowlarr.rs b/backend/src/scrapers/prowlarr.rs index e3eab75f..b8a493ba 100644 --- a/backend/src/scrapers/prowlarr.rs +++ b/backend/src/scrapers/prowlarr.rs @@ -17,6 +17,17 @@ use crate::{ pub(crate) const RESULT_PROCESS_CONCURRENCY: usize = 5; +fn format_request_error(e: &(dyn std::error::Error + Send + Sync)) -> String { + let msg = e.to_string(); + if msg.contains("401 Unauthorized") { + return "HTTP 401 Unauthorized — invalid or missing X-Api-Key (check PROWLARR_API_KEY or profile indexer API key)".into(); + } + if msg.contains("timed out") { + return "request timed out".into(); + } + msg +} + // ─── Prowlarr response shapes ───────────────────────────────────────────────── #[derive(Debug, Deserialize)] @@ -99,6 +110,19 @@ pub(crate) struct SearchResult { // ─── Healthy indexer (local representation) ────────────────────────────────── +#[derive(Debug, Clone)] +pub struct ProwlarrIndexer { + pub id: i64, + pub name: String, + pub priority: i64, + pub is_public: bool, + pub privacy: String, + pub categories: Vec, + pub supports_imdb_movie: bool, + pub supports_imdb_tv: bool, + pub supports_basic_search: bool, +} + #[derive(Debug, Clone)] struct Indexer { id: i64, @@ -112,7 +136,85 @@ struct Indexer { supports_basic_search: bool, } -// ─── Public entry point ─────────────────────────────────────────────────────── +impl From for ProwlarrIndexer { + fn from(value: Indexer) -> Self { + Self { + id: value.id, + name: value.name, + priority: value.priority, + is_public: value.is_public, + privacy: value.privacy, + categories: value.categories, + supports_imdb_movie: value.supports_imdb_movie, + supports_imdb_tv: value.supports_imdb_tv, + supports_basic_search: value.supports_basic_search, + } + } +} + +// ─── Public entry points ────────────────────────────────────────────────────── + +pub async fn list_healthy_indexers( + client: &Client, + base_url: &str, + api_key: &str, +) -> Vec { + match fetch_indexers(client, base_url, api_key).await { + Ok(indexers) => indexers.into_iter().map(ProwlarrIndexer::from).collect(), + Err(e) => { + tracing::debug!( + "prowlarr: failed to fetch indexers: {}", + format_request_error(&*e) + ); + vec![] + } + } +} + +#[allow(clippy::too_many_arguments)] +pub async fn scrape_indexer( + client: &Client, + base_url: &str, + api_key: &str, + idx: &ProwlarrIndexer, + meta: &SearchMeta, + media_type: &str, + season: Option, + episode: Option, + max_process: usize, + query_timeout: Duration, + title_queries: &[String], + privacy_by_id: &HashMap, + deadline: tokio::time::Instant, +) -> Vec { + let idx = Indexer { + id: idx.id, + name: idx.name.clone(), + priority: idx.priority, + is_public: idx.is_public, + privacy: idx.privacy.clone(), + categories: idx.categories.clone(), + supports_imdb_movie: idx.supports_imdb_movie, + supports_imdb_tv: idx.supports_imdb_tv, + supports_basic_search: idx.supports_basic_search, + }; + scrape_indexer_inner( + client, + base_url, + api_key, + &idx, + meta, + media_type, + season, + episode, + max_process, + query_timeout, + title_queries, + privacy_by_id, + deadline, + ) + .await +} #[allow(clippy::too_many_arguments)] pub async fn scrape( @@ -126,6 +228,7 @@ pub async fn scrape( max_process: usize, max_process_time: std::time::Duration, query_timeout: std::time::Duration, + title_queries: &[String], ) -> Vec { let indexers = match fetch_indexers(client, base_url, api_key).await { Ok(v) if !v.is_empty() => v, @@ -134,49 +237,135 @@ pub async fn scrape( return vec![]; } Err(e) => { - tracing::warn!("prowlarr: failed to fetch indexers: {e}"); + tracing::debug!( + "prowlarr: failed to fetch indexers: {}", + format_request_error(&*e) + ); return vec![]; } }; let privacy_by_id: HashMap = indexers.iter().map(|i| (i.id, i.privacy.clone())).collect(); + let deadline = tokio::time::Instant::now() + max_process_time; + let mut results = Vec::new(); + + for idx in &indexers { + if tokio::time::Instant::now() >= deadline { + break; + } + let mut batch = scrape_indexer_inner( + client, + base_url, + api_key, + idx, + meta, + media_type, + season, + episode, + max_process, + query_timeout, + title_queries, + &privacy_by_id, + deadline, + ) + .await; + results.append(&mut batch); + } + results +} + +#[allow(clippy::too_many_arguments)] +async fn scrape_indexer_inner( + client: &Client, + base_url: &str, + api_key: &str, + idx: &Indexer, + meta: &SearchMeta, + media_type: &str, + season: Option, + episode: Option, + max_process: usize, + query_timeout: Duration, + title_queries: &[String], + privacy_by_id: &HashMap, + deadline: tokio::time::Instant, +) -> Vec { let imdb_id = meta.imdb_id.as_deref().unwrap_or(""); let is_series = media_type == "series"; - let mut results: Vec = Vec::new(); let mut consecutive_failures: u32 = 0; - let deadline = tokio::time::Instant::now() + max_process_time; - for idx in &indexers { + if tokio::time::Instant::now() >= deadline { + return results; + } + + let (imdb_search_type, imdb_categories) = if is_series && idx.supports_imdb_tv { + ("tvsearch", movie_tv_categories(idx, true)) + } else if !is_series && idx.supports_imdb_movie { + ("movie", movie_tv_categories(idx, false)) + } else { + ("", Vec::new()) + }; + + let fallback_search_type = if is_series && idx.supports_imdb_tv { + "tvsearch" + } else if !is_series && idx.supports_imdb_movie { + "movie" + } else if idx.supports_imdb_movie || idx.supports_imdb_tv || idx.supports_basic_search { + "search" + } else { + return results; + }; + let fallback_categories = if imdb_categories.is_empty() { + if fallback_search_type == "search" { + idx.categories.clone() + } else { + movie_tv_categories(idx, is_series) + } + } else { + imdb_categories.clone() + }; + + let mut queries: Vec<(String, &str, Vec)> = Vec::new(); + + if !imdb_id.is_empty() && !imdb_search_type.is_empty() { + queries.push(( + format!("{{IMDbId:{imdb_id}}}"), + imdb_search_type, + imdb_categories, + )); + } + + if !title_queries.is_empty() + && (idx.supports_basic_search || idx.supports_imdb_movie || idx.supports_imdb_tv) + { + for title_query in title_queries { + queries.push((title_query.clone(), "search", idx.categories.clone())); + } + } else if imdb_id.is_empty() && title_queries.is_empty() { + queries.push(( + meta.title.clone(), + fallback_search_type, + fallback_categories, + )); + } + + for (query, search_type, categories) in queries { if tokio::time::Instant::now() >= deadline { - tracing::debug!("prowlarr: max_process_time exceeded after processing some indexers"); + tracing::debug!( + "prowlarr: max_process_time exceeded for indexer {}", + idx.name + ); break; } if consecutive_failures >= 3 { - tracing::debug!("prowlarr: stopping after 3 consecutive failures"); + tracing::debug!("prowlarr: stopping indexer {} after 3 failures", idx.name); break; } - let (search_type, categories) = if is_series && idx.supports_imdb_tv { - ("tvsearch", movie_tv_categories(idx, true)) - } else if !is_series && idx.supports_imdb_movie { - ("movie", movie_tv_categories(idx, false)) - } else if idx.supports_imdb_movie || idx.supports_imdb_tv || idx.supports_basic_search { - ("search", idx.categories.clone()) - } else { - continue; - }; - - let query = if !imdb_id.is_empty() && (search_type == "movie" || search_type == "tvsearch") - { - format!("{{IMDbId:{imdb_id}}}") - } else { - meta.title.clone() - }; - let mut params = vec![ ("query".to_string(), query), ("type".to_string(), search_type.to_string()), @@ -197,7 +386,7 @@ pub async fn scrape( client, item, &idx.name, - &privacy_by_id, + privacy_by_id, media_type, season, episode, @@ -212,7 +401,11 @@ pub async fn scrape( } Err(e) => { consecutive_failures += 1; - tracing::debug!("prowlarr: indexer {} failed: {e}", idx.name); + tracing::debug!( + "prowlarr: indexer {} failed: {}", + idx.name, + format_request_error(&*e) + ); } } } diff --git a/backend/src/scrapers/title_queries.rs b/backend/src/scrapers/title_queries.rs new file mode 100644 index 00000000..42a7ee34 --- /dev/null +++ b/backend/src/scrapers/title_queries.rs @@ -0,0 +1,41 @@ +//! Title-based search query templates for indexer scrapers (Prowlarr / Jackett). +//! Ported from Python `IndexerBaseScraper.MOVIE/SERIES_SEARCH_QUERY_TEMPLATES`. + +pub fn movie_title_queries(title: &str, year: Option) -> Vec { + let year = year.unwrap_or(0); + vec![ + format!("{title} ({year})"), + format!("{title} {year}"), + title.to_string(), + ] +} + +pub fn series_title_queries(title: &str, season: i32, episode: i32) -> Vec { + vec![ + format!("{title} S{season:02}E{episode:02}"), + format!("{title} Season {season} Episode {episode}"), + format!("{title} {season}x{episode}"), + format!("{title} S{season:02}"), + title.to_string(), + ] +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn movie_queries_include_year_variants() { + let queries = movie_title_queries("Inception", Some(2010)); + assert_eq!(queries[0], "Inception (2010)"); + assert_eq!(queries[1], "Inception 2010"); + assert_eq!(queries[2], "Inception"); + } + + #[test] + fn series_queries_cover_common_formats() { + let queries = series_title_queries("Breaking Bad", 1, 1); + assert!(queries.contains(&"Breaking Bad S01E01".to_string())); + assert!(queries.contains(&"Breaking Bad 1x1".to_string())); + } +} diff --git a/backend/src/scrapers/torbox_search.rs b/backend/src/scrapers/torbox_search.rs index eb606c4d..f6331e5d 100644 --- a/backend/src/scrapers/torbox_search.rs +++ b/backend/src/scrapers/torbox_search.rs @@ -325,7 +325,7 @@ pub async fn scrape_usenet( } } Ok(r) if r.status().as_u16() == 429 => { - tracing::warn!("torbox_usenet query '{query}': rate-limited (429)"); + tracing::debug!("torbox_usenet query '{query}': rate-limited (429)"); } Ok(r) => tracing::debug!("torbox_usenet query {query}: HTTP {}", r.status()), Err(e) => tracing::debug!("torbox_usenet query {query}: {e}"), @@ -374,7 +374,7 @@ async fn search_usenet_by_imdb( Some(parse_usenet_json(&json, media_type, season, episode, None)) } Ok(r) if r.status().as_u16() == 429 => { - tracing::warn!( + tracing::debug!( "torbox_usenet imdb {imdb_id}: rate-limited (429) — skipping title query" ); None diff --git a/backend/src/scrapers/zilean.rs b/backend/src/scrapers/zilean.rs index 1f1fc56e..1b56fd95 100644 --- a/backend/src/scrapers/zilean.rs +++ b/backend/src/scrapers/zilean.rs @@ -9,7 +9,8 @@ use crate::{ scrapers::{ScrapedStream, SearchMeta, StreamFile}, }; -pub async fn scrape( +/// POST /dmm/search — broad title lookup. +pub async fn scrape_search( client: &Client, base_url: &str, meta: &SearchMeta, @@ -17,13 +18,33 @@ pub async fn scrape( season: Option, episode: Option, ) -> Vec { - // Fire POST /dmm/search and GET /dmm/filtered concurrently. - let search_fut = client + let resp = client .post(format!("{base_url}/dmm/search")) .timeout(std::time::Duration::from_secs(10)) .json(&json!({"queryText": meta.title})) - .send(); + .send() + .await; + + let raw_items = match resp { + Ok(r) => parse_json_array(r).await, + Err(e) => { + tracing::debug!("zilean /dmm/search request error: {e}"); + vec![] + } + }; + + process_items(raw_items, media_type, season, episode).await +} +/// GET /dmm/filtered — structured movie/series lookup. +pub async fn scrape_filtered( + client: &Client, + base_url: &str, + meta: &SearchMeta, + media_type: &str, + season: Option, + episode: Option, +) -> Vec { let mut filter_params: Vec<(&str, String)> = vec![("Query", meta.title.clone())]; if media_type == "movie" { if let Some(y) = meta.year { @@ -38,43 +59,69 @@ pub async fn scrape( } } - let filtered_fut = client + let resp = client .get(format!("{base_url}/dmm/filtered")) .timeout(std::time::Duration::from_secs(10)) .query(&filter_params) - .send(); + .send() + .await; + + let raw_items = match resp { + Ok(r) => parse_json_array(r).await, + Err(e) => { + tracing::debug!("zilean /dmm/filtered request error: {e}"); + vec![] + } + }; - let (search_res, filtered_res) = tokio::join!(search_fut, filtered_fut); + process_items(raw_items, media_type, season, episode).await +} - let mut raw_items: Vec = Vec::new(); +pub async fn scrape( + client: &Client, + base_url: &str, + meta: &SearchMeta, + media_type: &str, + season: Option, + episode: Option, +) -> Vec { + let (search, filtered) = tokio::join!( + scrape_search(client, base_url, meta, media_type, season, episode), + scrape_filtered(client, base_url, meta, media_type, season, episode), + ); + let mut seen = std::collections::HashSet::new(); + search + .into_iter() + .chain(filtered) + .filter(|s| seen.insert(s.info_hash.clone())) + .collect() +} - match search_res { - Ok(resp) => match resp.text().await { - Ok(body) => match serde_json::from_str::>(&body) { - Ok(items) => raw_items.extend(items), - Err(e) => tracing::debug!( - "zilean /dmm/search parse error: {e} — body: {}", +async fn parse_json_array(resp: reqwest::Response) -> Vec { + match resp.text().await { + Ok(body) => match serde_json::from_str::>(&body) { + Ok(items) => items, + Err(e) => { + tracing::debug!( + "zilean response parse error: {e} — body: {}", body.chars().take(500).collect::() - ), - }, - Err(e) => tracing::debug!("zilean /dmm/search body error: {e}"), - }, - Err(e) => tracing::debug!("zilean /dmm/search request error: {e}"), - } - match filtered_res { - Ok(resp) => match resp.text().await { - Ok(body) => match serde_json::from_str::>(&body) { - Ok(items) => raw_items.extend(items), - Err(e) => tracing::debug!( - "zilean /dmm/filtered parse error: {e} — body: {}", - body.chars().take(500).collect::() - ), - }, - Err(e) => tracing::debug!("zilean /dmm/filtered body error: {e}"), + ); + vec![] + } }, - Err(e) => tracing::debug!("zilean /dmm/filtered request error: {e}"), + Err(e) => { + tracing::debug!("zilean response body error: {e}"); + vec![] + } } +} +async fn process_items( + raw_items: Vec, + media_type: &str, + season: Option, + episode: Option, +) -> Vec { if raw_items.is_empty() { return vec![]; } @@ -155,7 +202,6 @@ fn process_item( } f } else { - // Movie: reject if it has season/episode markers if !parsed.seasons.is_empty() || !parsed.episodes.is_empty() { return None; } diff --git a/backend/src/state.rs b/backend/src/state.rs index 036958d4..d5a6be81 100644 --- a/backend/src/state.rs +++ b/backend/src/state.rs @@ -17,6 +17,37 @@ pub struct KeywordFilterCache { pub whitelist: Vec, // whitelist phrases, lowercased } +impl KeywordFilterCache { + /// Returns true when `text` matches an active blacklist keyword and is not whitelisted. + pub fn matches_blocked_keyword(&self, text: &str) -> bool { + if text.is_empty() { + return false; + } + let lower = text.to_lowercase(); + if self + .whitelist + .iter() + .any(|phrase| lower.contains(phrase.as_str())) + { + return false; + } + self.keywords + .iter() + .any(|keyword| lower.contains(keyword.as_str())) + } + + /// Remove genres whose names match blacklist keywords. + pub fn filter_genres_by_type( + &self, + mut genres: std::collections::HashMap>, + ) -> std::collections::HashMap> { + for list in genres.values_mut() { + list.retain(|name| !self.matches_blocked_keyword(name)); + } + genres + } +} + #[derive(Clone)] pub struct AppState { pub config: AppConfig, diff --git a/backend/tests/db_queries.rs b/backend/tests/db_queries.rs new file mode 100644 index 00000000..b8c1e79b --- /dev/null +++ b/backend/tests/db_queries.rs @@ -0,0 +1,371 @@ +/// Integration tests for database query correctness. +/// +/// These tests protect against two families of runtime bugs that static analysis +/// and `cargo sqlx prepare --check` cannot catch for non-macro `query_as` calls: +/// +/// 1. LEFT JOIN LATERAL NULL-decode — columns that are NOT NULL in their base +/// table are inferred as non-nullable, but a LEFT JOIN can return NULL when +/// there is no matching row. These tests insert media rows with deliberately +/// missing images/ratings and verify the query returns `None` fields rather +/// than a decode error. +/// +/// 2. GROUP BY completeness — a non-aggregate column missing from GROUP BY +/// causes a Postgres error at runtime. The playback-info test exercises +/// the exact query path that had this bug with `ts.torrent_file`. +/// +/// Each test inserts data prefixed with "test_db_queries::" and deletes it in a +/// finally-block, so the test database is left clean even on assertion failure. +mod common; + +// db::meta::get_media_meta returns the full Stremio MetaItem row (poster, rating, etc.) +// db::media::get_media_meta (re-exported as db::get_media_meta) returns a SearchMeta stub. +// Tests here cover the former. +use mediafusion_api::db::{ + catalog::get_watchlist_items, + fetch_stream_playback_info, + meta::{get_episodes, get_media_meta as get_full_meta}, + types::{MediaId, MediaType}, +}; + +// ─── helpers ───────────────────────────────────────────────────────────────── + +struct Cleanup { + pool: sqlx::PgPool, + media_ids: Vec, + stream_ids: Vec, +} + +impl Drop for Cleanup { + fn drop(&mut self) { + let pool = self.pool.clone(); + let media_ids = self.media_ids.clone(); + let stream_ids = self.stream_ids.clone(); + // Run cleanup synchronously via a blocking task; ignore errors. + let _ = std::thread::spawn(move || { + tokio::runtime::Runtime::new().unwrap().block_on(async { + if !stream_ids.is_empty() { + let _ = sqlx::query("DELETE FROM stream WHERE id = ANY($1)") + .bind(&stream_ids) + .execute(&pool) + .await; + } + if !media_ids.is_empty() { + let _ = sqlx::query("DELETE FROM media WHERE id = ANY($1)") + .bind(&media_ids) + .execute(&pool) + .await; + } + }); + }) + .join(); + } +} + +async fn insert_media(pool: &sqlx::PgPool, media_type: MediaType, title: &str) -> i32 { + sqlx::query_scalar::<_, i32>( + r#"INSERT INTO media (type, title, adult, is_blocked, is_public, is_user_created, + total_streams, nudity_status, created_at) + VALUES ($1, $2, false, false, true, false, 0, 'UNKNOWN', NOW()) + RETURNING id"#, + ) + .bind(media_type) + .bind(title) + .fetch_one(pool) + .await + .expect("insert media") +} + +async fn link_imdb(pool: &sqlx::PgPool, media_id: i32, imdb_id: &str) { + sqlx::query( + "INSERT INTO media_external_id (media_id, provider, external_id, created_at) + VALUES ($1, 'imdb', $2, NOW())", + ) + .bind(media_id) + .bind(imdb_id) + .execute(pool) + .await + .expect("link imdb"); +} + +// ─── get_media_meta: NULL poster / background / rating ─────────────────────── + +/// A movie with no poster, no background image, and no IMDb rating must decode +/// without error and return `None` for those three nullable fields. +/// +/// Protects against: LEFT JOIN LATERAL NULL-decode on `mi_poster.url`, +/// `mi_bg.url`, `mr.rating` — src/db/meta.rs `get_media_meta` (external-id path). +/// Regression for the error logged as: +/// "meta query [tt37532356]: error occurred while decoding column 13: unexpected null" +#[tokio::test] +async fn get_media_meta_null_poster_and_rating_external_id() { + let pool = common::test_pool().await; + let media_id = insert_media(&pool, MediaType::Movie, "test_db_queries::no_poster_movie").await; + let imdb_id = format!("tt_test_{media_id}"); + link_imdb(&pool, media_id, &imdb_id).await; + + let mut cleanup = Cleanup { + pool: pool.clone(), + media_ids: vec![media_id], + stream_ids: vec![], + }; + + // No media_image or media_rating rows — forces LEFT JOIN LATERAL to return NULL. + let result = get_full_meta(&pool, &imdb_id, "movie").await; + + cleanup.media_ids.clear(); // handled below so we can assert first + sqlx::query("DELETE FROM media WHERE id = $1") + .bind(media_id) + .execute(&pool) + .await + .ok(); + + let row = result.expect("query must succeed (not panic/warn on NULL lateral)"); + assert_eq!(row.title, "test_db_queries::no_poster_movie"); + assert!( + row.poster_url.is_none(), + "poster_url must be None — no media_image row" + ); + assert!( + row.background_url.is_none(), + "background_url must be None — no media_image row" + ); + assert!( + row.imdb_rating.is_none(), + "imdb_rating must be None — no media_rating row" + ); +} + +/// Same test via internal `mf{id}` lookup path. +#[tokio::test] +async fn get_media_meta_null_poster_and_rating_internal_id() { + let pool = common::test_pool().await; + let media_id = insert_media( + &pool, + MediaType::Movie, + "test_db_queries::no_poster_internal", + ) + .await; + + let result = get_full_meta(&pool, &format!("mf{media_id}"), "movie").await; + + sqlx::query("DELETE FROM media WHERE id = $1") + .bind(media_id) + .execute(&pool) + .await + .ok(); + + let row = result.expect("query must succeed via internal-id path"); + assert!(row.poster_url.is_none()); + assert!(row.background_url.is_none()); + assert!(row.imdb_rating.is_none()); +} + +// ─── get_episodes: NULL thumbnail / NULL file-link ─────────────────────────── + +/// Episodes with no `episode_image` and no `file_media_link` must decode with +/// `thumbnail_url = None` and `media_id = None`. +/// +/// Protects against: LEFT JOIN LATERAL NULL-decode on `ei.url` (column 5) and +/// LEFT JOIN NULL on `fml.media_id` — src/db/meta.rs `get_episodes`. +/// Regression for: +/// "episodes for media 200645: error occurred while decoding column 5: unexpected null" +#[tokio::test] +async fn get_episodes_null_thumbnail_and_null_file_link() { + let pool = common::test_pool().await; + let media_id = insert_media(&pool, MediaType::Series, "test_db_queries::no_thumb_series").await; + + sqlx::query( + "INSERT INTO series_metadata (media_id, total_seasons, total_episodes, created_at, updated_at) + VALUES ($1, 1, 2, NOW(), NOW())", + ) + .bind(media_id) + .execute(&pool) + .await + .expect("series_metadata"); + + let season_id: i32 = sqlx::query_scalar::<_, i32>( + "INSERT INTO season (series_id, season_number, episode_count) + VALUES ((SELECT id FROM series_metadata WHERE media_id = $1), 1, 2) RETURNING id", + ) + .bind(media_id) + .fetch_one(&pool) + .await + .expect("season"); + + for ep in [1i32, 2] { + sqlx::query( + "INSERT INTO episode (season_id, episode_number, title, is_user_created, + is_user_addition, created_at, updated_at) + VALUES ($1, $2, $3, false, false, NOW(), NOW())", + ) + .bind(season_id) + .bind(ep) + .bind(format!("Episode {ep}")) + .execute(&pool) + .await + .expect("episode"); + } + + // No episode_image rows, no file_media_link rows. + let rows = get_episodes(&pool, MediaId(media_id)).await; + + sqlx::query("DELETE FROM media WHERE id = $1") + .bind(media_id) + .execute(&pool) + .await + .ok(); + + assert_eq!( + rows.len(), + 2, + "must return both episodes (not an empty vec from a decode error)" + ); + for row in &rows { + assert!( + row.thumbnail_url.is_none(), + "thumbnail_url must be None when no episode_image row exists (column 5 NULL)" + ); + assert!( + row.media_id.is_none(), + "media_id must be None when no file_media_link row exists" + ); + } +} + +// ─── fetch_stream_playback_info: GROUP BY completeness ─────────────────────── + +/// Fetch playback info for a movie torrent (no season/episode). +/// +/// Protects against: GROUP BY missing `ts.torrent_file` in the non-series +/// query path — src/db/streams.rs `fetch_stream_playback_info`. +/// Regression for: +/// "fetch_stream_playback_info error: column 'ts.torrent_file' must appear in GROUP BY" +#[tokio::test] +async fn fetch_stream_playback_info_movie_group_by_is_complete() { + let pool = common::test_pool().await; + + let stream_id: i32 = sqlx::query_scalar::<_, i32>( + r#"INSERT INTO stream (stream_type, name, source, is_active, is_blocked, is_public, + playback_count, is_remastered, is_upscaled, is_proper, is_repack, + is_extended, is_complete, is_dubbed, is_subbed, created_at) + VALUES ('TORRENT', 'test_db_queries::playback_movie.mkv', 'test', + true, false, true, 0, + false, false, false, false, false, false, false, false, NOW()) + RETURNING id"#, + ) + .fetch_one(&pool) + .await + .expect("insert stream"); + + // Use a unique hash to avoid conflicts with other test data. + let info_hash = format!("test{stream_id:0>36}"); + + // Include a non-NULL torrent_file to exercise the GROUP BY column that was missing. + sqlx::query( + r#"INSERT INTO torrent_stream (stream_id, info_hash, total_size, torrent_type, + file_count, torrent_file, created_at) + VALUES ($1, $2, 2147483648, 'PUBLIC', 1, '\xdeadbeef'::bytea, NOW())"#, + ) + .bind(stream_id) + .bind(&info_hash) + .execute(&pool) + .await + .expect("insert torrent_stream"); + + let result = fetch_stream_playback_info(&pool, &info_hash, None, None).await; + + sqlx::query("DELETE FROM stream WHERE id = $1") + .bind(stream_id) + .execute(&pool) + .await + .ok(); + + let info = result.expect( + "query must succeed — GROUP BY was missing ts.torrent_file causing a Postgres error", + ); + assert_eq!(info.name, "test_db_queries::playback_movie.mkv"); + assert_eq!(info.size_bytes, Some(2147483648)); + assert!( + info.torrent_file.is_some(), + "stored torrent_file bytes must be returned" + ); +} + +// ─── get_watchlist_items: info_hash → media join ───────────────────────────── + +/// A downloaded info_hash linked to media via torrent_stream must appear in +/// watchlist catalog results. +#[tokio::test] +async fn get_watchlist_items_resolves_info_hash_to_media() { + let pool = common::test_pool().await; + let media_id = insert_media(&pool, MediaType::Movie, "test_db_queries::watchlist_movie").await; + + sqlx::query("UPDATE media SET total_streams = 1 WHERE id = $1") + .bind(media_id) + .execute(&pool) + .await + .expect("bump total_streams"); + + let stream_id: i32 = sqlx::query_scalar::<_, i32>( + r#"INSERT INTO stream (stream_type, name, source, is_active, is_blocked, is_public, + playback_count, is_remastered, is_upscaled, is_proper, is_repack, + is_extended, is_complete, is_dubbed, is_subbed, created_at) + VALUES ('TORRENT', 'test_db_queries::watchlist.mkv', 'test', + true, false, true, 0, + false, false, false, false, false, false, false, false, NOW()) + RETURNING id"#, + ) + .fetch_one(&pool) + .await + .expect("insert stream"); + + let info_hash = format!("watchlist{stream_id:0>32}"); + + sqlx::query( + r#"INSERT INTO torrent_stream (stream_id, info_hash, total_size, torrent_type, + file_count, created_at) + VALUES ($1, $2, 1000, 'PUBLIC', 1, NOW())"#, + ) + .bind(stream_id) + .bind(&info_hash) + .execute(&pool) + .await + .expect("insert torrent_stream"); + + sqlx::query( + "INSERT INTO stream_media_link (stream_id, media_id, is_primary, is_verified, created_at) + VALUES ($1, $2, true, false, NOW())", + ) + .bind(stream_id) + .bind(media_id) + .execute(&pool) + .await + .expect("insert stream_media_link"); + + let rows = get_watchlist_items( + &pool, + "movie", + std::slice::from_ref(&info_hash), + 0, + &[], + &[], + "latest", + "desc", + ) + .await; + + sqlx::query("DELETE FROM stream WHERE id = $1") + .bind(stream_id) + .execute(&pool) + .await + .ok(); + sqlx::query("DELETE FROM media WHERE id = $1") + .bind(media_id) + .execute(&pool) + .await + .ok(); + + assert_eq!(rows.len(), 1, "expected one watchlist row for linked hash"); + assert_eq!(rows[0].media_id, MediaId(media_id)); + assert_eq!(rows[0].title, "test_db_queries::watchlist_movie"); +} diff --git a/clients/kodi/plugin.video.mediafusion/addon.xml b/clients/kodi/plugin.video.mediafusion/addon.xml index e42f92e2..02f76cfe 100644 --- a/clients/kodi/plugin.video.mediafusion/addon.xml +++ b/clients/kodi/plugin.video.mediafusion/addon.xml @@ -1,5 +1,5 @@ - + diff --git a/clients/kodi/repository.mediafusion/addon.xml b/clients/kodi/repository.mediafusion/addon.xml index c48e9614..36a4429f 100644 --- a/clients/kodi/repository.mediafusion/addon.xml +++ b/clients/kodi/repository.mediafusion/addon.xml @@ -1,5 +1,5 @@ - + https://mhdzumair.github.io/MediaFusion/addons.xml diff --git a/deployment/docker-compose/docker-compose-minimal.yml b/deployment/docker-compose/docker-compose-minimal.yml index e93b80c4..e076703e 100644 --- a/deployment/docker-compose/docker-compose-minimal.yml +++ b/deployment/docker-compose/docker-compose-minimal.yml @@ -1,7 +1,7 @@ services: # ── Primary API (Rust) ──────────────────────────────────────────────────── mediafusion-api: - image: mhdzumair/mediafusion:6.0.0-beta.13 + image: mhdzumair/mediafusion:6.0.0-beta.14 ports: - "8000:8000" env_file: .env diff --git a/deployment/docker-compose/docker-compose-perf.yml b/deployment/docker-compose/docker-compose-perf.yml index 0aabab77..3f9426f2 100644 --- a/deployment/docker-compose/docker-compose-perf.yml +++ b/deployment/docker-compose/docker-compose-perf.yml @@ -117,11 +117,11 @@ services: max-file: "2" # --------------------------------------------------------------------------- - # Override mediafusion-api to expose /metrics without auth (perf only!) + # Override mediafusion-api to enable the metrics endpoint (no token for perf). # --------------------------------------------------------------------------- mediafusion-api: environment: - - PROMETHEUS_DISABLE_AUTH=true + - ENABLE_PROMETHEUS_METRICS=true volumes: prometheus-data: diff --git a/deployment/docker-compose/docker-compose-postgres-ha.yml b/deployment/docker-compose/docker-compose-postgres-ha.yml index 36b7e3fa..94506d24 100644 --- a/deployment/docker-compose/docker-compose-postgres-ha.yml +++ b/deployment/docker-compose/docker-compose-postgres-ha.yml @@ -236,7 +236,7 @@ services: condition: service_started taskiq-worker-scrapy: - image: mhdzumair/mediafusion:6.0.0-beta.13 + image: mhdzumair/mediafusion:6.0.0-beta.14 command: ["taskiq", "worker", "workers.taskiq_worker:broker_scrapy", "--workers", "1", "--max-async-tasks", "1", "--ack-type", "when_executed"] env_file: .env environment: @@ -263,7 +263,7 @@ services: - mediafusion-network taskiq-worker-import: - image: mhdzumair/mediafusion:6.0.0-beta.13 + image: mhdzumair/mediafusion:6.0.0-beta.14 command: ["taskiq", "worker", "workers.taskiq_worker:broker_import", "--workers", "1", "--max-async-tasks", "4", "--ack-type", "when_executed"] env_file: .env environment: @@ -290,7 +290,7 @@ services: - mediafusion-network taskiq-worker-priority: - image: mhdzumair/mediafusion:6.0.0-beta.13 + image: mhdzumair/mediafusion:6.0.0-beta.14 command: ["taskiq", "worker", "workers.taskiq_worker:broker_priority", "--workers", "1", "--max-async-tasks", "4", "--ack-type", "when_executed"] env_file: .env environment: diff --git a/deployment/docker-compose/docker-compose.yml b/deployment/docker-compose/docker-compose.yml index 66488c08..cb8b34f9 100644 --- a/deployment/docker-compose/docker-compose.yml +++ b/deployment/docker-compose/docker-compose.yml @@ -1,7 +1,7 @@ services: # ── Primary API (Rust) ──────────────────────────────────────────────────── mediafusion-api: - image: mhdzumair/mediafusion:6.0.0-beta.13 + image: mhdzumair/mediafusion:6.0.0-beta.14 ports: - "8000:8000" env_file: .env @@ -31,7 +31,7 @@ services: # ── Background Worker (Rust) ────────────────────────────────────────────── mediafusion-worker: - image: mhdzumair/mediafusion:6.0.0-beta.13 + image: mhdzumair/mediafusion:6.0.0-beta.14 command: ["/usr/local/bin/mediafusion-worker"] env_file: .env environment: diff --git a/deployment/docker-compose/prometheus-dev.yml b/deployment/docker-compose/prometheus-dev.yml index 81f52182..456cb297 100644 --- a/deployment/docker-compose/prometheus-dev.yml +++ b/deployment/docker-compose/prometheus-dev.yml @@ -1,8 +1,9 @@ # Prometheus config for local dev — scrapes the Rust server on the host machine. # host.docker.internal resolves to the host IP from inside Docker. # -# /api/v1/metrics requires X-API-Key on private instances; open on public instances. -# Set MEDIAFUSION_API_KEY in the environment or pass an empty string for public instances. +# The Rust server must have ENABLE_PROMETHEUS_METRICS=true. +# Set PROMETHEUS_METRICS_TOKEN to match the server's PROMETHEUS_METRICS_TOKEN. +# Leave it empty/unset if the server has no token configured. global: scrape_interval: 5s @@ -13,9 +14,8 @@ scrape_configs: static_configs: - targets: ["host.docker.internal:8001"] metrics_path: /api/v1/metrics - params: {} - http_headers: - X-API-Key: ["${MEDIAFUSION_API_KEY:-}"] + authorization: + credentials: "${PROMETHEUS_METRICS_TOKEN:-}" # Optional: Redis exporter if running locally on the default port - job_name: redis diff --git a/deployment/docker-compose/prometheus.yml b/deployment/docker-compose/prometheus.yml index 19ca8a70..19e5169b 100644 --- a/deployment/docker-compose/prometheus.yml +++ b/deployment/docker-compose/prometheus.yml @@ -6,8 +6,9 @@ scrape_configs: - job_name: mediafusion static_configs: - targets: ["mediafusion-api:8000"] - metrics_path: /api/v1/admin/metrics/prometheus - # No auth header needed when PROMETHEUS_DISABLE_AUTH=true + metrics_path: /api/v1/metrics + authorization: + credentials: "${PROMETHEUS_METRICS_TOKEN}" - job_name: redis static_configs: diff --git a/deployment/k8s/local-deployment.yaml b/deployment/k8s/local-deployment.yaml index 8638c6df..c6679500 100644 --- a/deployment/k8s/local-deployment.yaml +++ b/deployment/k8s/local-deployment.yaml @@ -19,7 +19,7 @@ spec: spec: containers: - name: mediafusion-scheduler - image: mhdzumair/mediafusion:6.0.0-beta.13 + image: mhdzumair/mediafusion:6.0.0-beta.14 command: ["python", "-m", "workers.run_scheduler"] resources: requests: @@ -210,7 +210,7 @@ spec: spec: containers: - name: taskiq-worker-default - image: mhdzumair/mediafusion:6.0.0-beta.13 + image: mhdzumair/mediafusion:6.0.0-beta.14 command: ["taskiq", "worker", "workers.taskiq_worker:broker_default", "--workers", "1", "--max-async-tasks", "8", "--ack-type", "when_executed"] env: - name: HOST_URL @@ -272,7 +272,7 @@ spec: spec: containers: - name: taskiq-worker-scrapy - image: mhdzumair/mediafusion:6.0.0-beta.13 + image: mhdzumair/mediafusion:6.0.0-beta.14 command: ["taskiq", "worker", "workers.taskiq_worker:broker_scrapy", "--workers", "1", "--max-async-tasks", "1", "--ack-type", "when_executed"] env: - name: HOST_URL @@ -356,7 +356,7 @@ spec: spec: containers: - name: taskiq-worker-import - image: mhdzumair/mediafusion:6.0.0-beta.13 + image: mhdzumair/mediafusion:6.0.0-beta.14 command: ["taskiq", "worker", "workers.taskiq_worker:broker_import", "--workers", "1", "--max-async-tasks", "4", "--ack-type", "when_executed"] env: - name: HOST_URL @@ -418,7 +418,7 @@ spec: spec: containers: - name: taskiq-worker-priority - image: mhdzumair/mediafusion:6.0.0-beta.13 + image: mhdzumair/mediafusion:6.0.0-beta.14 command: ["taskiq", "worker", "workers.taskiq_worker:broker_priority", "--workers", "1", "--max-async-tasks", "4", "--ack-type", "when_executed"] env: - name: HOST_URL diff --git a/pyproject.toml b/pyproject.toml index 6c56ae75..d689065e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "mediafusion" -version = "6.0.0-beta.13" +version = "6.0.0-beta.14" description = "Media Fusion Add-On For Stremio & Kodi - A powerful streaming API with support for multiple providers and advanced scraping capabilities" readme = "README.md" requires-python = ">=3.12" diff --git a/uv.lock b/uv.lock index c6a03083..2ad28d4e 100644 --- a/uv.lock +++ b/uv.lock @@ -1765,7 +1765,7 @@ wheels = [ [[package]] name = "mediafusion" -version = "6.0.0b13" +version = "6.0.0b14" source = { virtual = "." } dependencies = [ { name = "aioboto3" },