Filter statistics to only include books in book_path

Author: paviro

README.md

@@ -215,6 +215,7 @@ KoShelf can operate in several modes:
 - `--min-pages-per-day`: Minimum pages read per book per day to be counted in statistics (optional)
 - `--min-time-per-day`: Minimum reading time per book per day to be counted in statistics (e.g., "15m", "1h") (optional)
   > **Note:** If both `--min-pages-per-day` and `--min-time-per-day` are provided, a book's data for a day is counted if **either** condition is met for that book on that day. These filters apply **per book per day**, meaning each book must individually meet the threshold for each day to be included in statistics.
+- `--include-all-stats`: By default, statistics are filtered to only include books present in your `--books-path` directory. This prevents deleted books or external files (like Wallabag articles) from skewing your recap and statistics. Use this flag to include statistics for all books in the database, regardless of whether they exist in your library.
 - `--github`: Print GitHub repository URL
 
 ### Example

src/book_scanner.rs

@@ -1,4 +1,4 @@
-use std::collections::HashMap;
+use std::collections::{HashMap, HashSet};
 use std::path::PathBuf;
 use anyhow::{Result, bail};
 use log::{debug, info, warn};
@@ -151,7 +151,7 @@ fn build_hashdocsettings_index(hashdocsettings_path: &PathBuf) -> Result<HashMap
 pub async fn scan_books(
     books_path: &PathBuf,
     metadata_location: &MetadataLocation,
-) -> Result<Vec<Book>> {
+) -> Result<(Vec<Book>, HashSet<String>)> {
     info!("Scanning books in directory: {:?}", books_path);
     let epub_parser = EpubParser::new();
     let lua_parser = LuaParser::new();
@@ -168,6 +168,7 @@ pub async fn scan_books(
     };
 
     let mut books = Vec::new();
+    let mut library_md5s = HashSet::new();
 
     // Walk through all epub files
     for entry in walkdir::WalkDir::new(books_path) {
@@ -191,6 +192,10 @@ pub async fn scan_books(
                 }
             };
 
+            // Track the MD5 for this book (for statistics filtering)
+            // We may already have it from hashdocsettings lookup, or will get it from metadata
+            let mut book_md5: Option<String> = None;
+            
             // Find metadata based on the configured location
             let metadata_path = match metadata_location {
                 MetadataLocation::InBookFolder => {
@@ -219,6 +224,8 @@ pub async fn scan_books(
                     match calculate_partial_md5(path) {
                         Ok(hash) => {
                             debug!("Calculated partial MD5 for {:?}: {}", path, hash);
+                            // Store the calculated MD5 for later use
+                            book_md5 = Some(hash.clone());
                             hashdocsettings_index
                                 .as_ref()
                                 .and_then(|idx| idx.get(&hash.to_lowercase()).cloned())
@@ -246,6 +253,24 @@ pub async fn scan_books(
                 None
             };
 
+            // Collect MD5 for statistics filtering:
+            // 1. Prefer MD5 from metadata (stable even if file is updated)
+            // 2. Use calculated MD5 from hashdocsettings lookup if available
+            // 3. Fall back to calculating MD5 for books without metadata
+            if let Some(ref metadata) = koreader_metadata {
+                if let Some(ref md5) = metadata.partial_md5_checksum {
+                    library_md5s.insert(md5.clone());
+                } else if let Some(ref md5) = book_md5 {
+                    library_md5s.insert(md5.clone());
+                } else if let Ok(md5) = calculate_partial_md5(path) {
+                    library_md5s.insert(md5);
+                }
+            } else if let Some(ref md5) = book_md5 {
+                library_md5s.insert(md5.clone());
+            } else if let Ok(md5) = calculate_partial_md5(path) {
+                library_md5s.insert(md5);
+            }
+            
             let book = Book {
                 id: generate_book_id(&epub_info.title),
                 epub_info,
@@ -258,5 +283,5 @@ pub async fn scan_books(
     }
 
     info!("Found {} books!", books.len());
-    Ok(books)
+    Ok((books, library_md5s))
 }

src/file_watcher.rs

@@ -21,6 +21,7 @@ pub struct FileWatcher {
     time_config: TimeConfig,
     min_pages_per_day: Option<u32>,
     min_time_per_day: Option<u32>,
+    include_all_stats: bool,
 }
 
 impl FileWatcher {
@@ -35,6 +36,7 @@ impl FileWatcher {
         time_config: TimeConfig,
         min_pages_per_day: Option<u32>,
         min_time_per_day: Option<u32>,
+        include_all_stats: bool,
     ) -> Result<Self> {
         Ok(Self {
             books_path,
@@ -48,6 +50,7 @@ impl FileWatcher {
             time_config,
             min_pages_per_day,
             min_time_per_day,
+            include_all_stats,
         })
     }
 
@@ -114,6 +117,7 @@ impl FileWatcher {
         let time_config_clone = self.time_config.clone();
         let min_pages_per_day_clone = self.min_pages_per_day;
         let min_time_per_day_clone = self.min_time_per_day;
+        let include_all_stats_clone = self.include_all_stats;
 
         // Spawn delayed rebuild task
         let rebuild_task = tokio::task::spawn_blocking(move || {
@@ -142,6 +146,7 @@ impl FileWatcher {
                         time_config_clone.clone(),
                         min_pages_per_day_clone,
                         min_time_per_day_clone,
+                        include_all_stats_clone,
                     );
 
                     match site_generator.generate().await {

src/main.rs

@@ -90,8 +90,14 @@ struct Cli {
     #[arg(long, display_order = 14)]
     min_time_per_day: Option<String>,
 
+    /// Include statistics for all books in the database, not just those in --books-path.
+    /// By default, when --books-path is provided, statistics are filtered to only include
+    /// books present in that directory. Use this flag to include all statistics.
+    #[arg(long, default_value = "false", display_order = 15)]
+    include_all_stats: bool,
+
     /// Print GitHub repository URL
-    #[arg(long, display_order = 15)]
+    #[arg(long, display_order = 16)]
     github: bool,
 }
 
@@ -132,6 +138,7 @@ fn parse_time_to_seconds(time_str: &str) -> Result<Option<u32>> {
 async fn main() -> Result<()> {
     env_logger::builder()
         .filter_level(log::LevelFilter::Info)
+        .parse_default_env()
         .init();
     let cli = Cli::parse();
 
@@ -258,6 +265,7 @@ async fn main() -> Result<()> {
         time_config.clone(),
         cli.min_pages_per_day,
         min_time_per_day,
+        cli.include_all_stats,
     );
 
     // Generate initial site
@@ -283,6 +291,7 @@ async fn main() -> Result<()> {
             time_config.clone(),
             cli.min_pages_per_day,
             min_time_per_day,
+            cli.include_all_stats,
         ).await?;
 
         // Run file watcher
@@ -302,6 +311,7 @@ async fn main() -> Result<()> {
             time_config.clone(),
             cli.min_pages_per_day,
             min_time_per_day,
+            cli.include_all_stats,
         ).await?;
 
         // Start web server

src/site_generator.rs

@@ -1,7 +1,7 @@
 use crate::models::*;
 use crate::templates::*;
 use crate::statistics_parser::StatisticsParser;
-use crate::statistics::{BookStatistics};
+use crate::statistics::{BookStatistics, StatisticsCalculator};
 use crate::book_scanner::{scan_books, MetadataLocation};
 use anyhow::{Result, Context};
 use askama::Template;
@@ -28,6 +28,7 @@ pub struct SiteGenerator {
     time_config: TimeConfig,
     min_pages_per_day: Option<u32>,
     min_time_per_day: Option<u32>,
+    include_all_stats: bool,
 }
 
 impl SiteGenerator {
@@ -42,6 +43,7 @@ impl SiteGenerator {
         time_config: TimeConfig,
         min_pages_per_day: Option<u32>,
         min_time_per_day: Option<u32>,
+        include_all_stats: bool,
     ) -> Self {
         Self {
             output_dir,
@@ -54,6 +56,7 @@ impl SiteGenerator {
             time_config,
             min_pages_per_day,
             min_time_per_day,
+            include_all_stats,
         }
     }
 
@@ -74,10 +77,11 @@ impl SiteGenerator {
         info!("Generating static site in: {:?}", self.output_dir);
 
         // Scan books if books_path is provided
-        let books = if let Some(ref books_path) = self.books_path {
+        // Also returns the set of MD5 hashes for all books (for statistics filtering)
+        let (books, library_md5s) = if let Some(ref books_path) = self.books_path {
             scan_books(books_path, &self.metadata_location).await?
         } else {
-            Vec::new()
+            (Vec::new(), std::collections::HashSet::new())
         };
 
         // After loading statistics if path is provided
@@ -95,6 +99,12 @@ impl SiteGenerator {
                     );
                 }
 
+                // Filter statistics to library books only (unless --include-all-stats is set)
+                // This is skipped if no books_path is provided (can't filter without a library)
+                if !self.include_all_stats && !books.is_empty() {
+                    StatisticsCalculator::filter_to_library(&mut data, &library_md5s);
+                }
+                
                 crate::statistics::StatisticsCalculator::populate_completions(&mut data, &self.time_config);
                 Some(data)
             } else {

src/statistics.rs

@@ -1,5 +1,6 @@
 use chrono::{NaiveDate, Duration, Datelike};
-use std::collections::HashMap;
+use std::collections::{HashMap, HashSet};
+use log::debug;
 
 use crate::models::*;
 use crate::read_completion_analyzer::{ReadCompletionDetector, CompletionConfig};
@@ -391,6 +392,36 @@ impl StatisticsCalculator {
             *valid_book_dates.get(&key).unwrap_or(&false)
         });
     }
+
+    /// Filter statistics to only include books present in the library.
+    /// The library_md5s set contains MD5 hashes of books in the scanned library.
+    /// This filters out statistics for deleted books or books in other directories.
+    pub fn filter_to_library(stats_data: &mut StatisticsData, library_md5s: &HashSet<String>) {
+        let original_count = stats_data.books.len();
+        
+        // 1. Get IDs of books to keep (those whose MD5 is in library_md5s)
+        let mut ids_to_keep: HashSet<i64> = HashSet::new();
+        stats_data.books.retain(|book| {
+            if library_md5s.contains(&book.md5) {
+                ids_to_keep.insert(book.id);
+                true
+            } else {
+                debug!("Filtering out statistics for book not in library: '{}' by {} (md5: {})", 
+                    book.title, book.authors, book.md5);
+                false
+            }
+        });
+        
+        // 2. Filter page_stats to only include entries for kept books
+        stats_data.page_stats.retain(|stat| ids_to_keep.contains(&stat.id_book));
+        
+        // 3. Update stats_by_md5 map to only include kept books
+        stats_data.stats_by_md5.retain(|md5, _| library_md5s.contains(md5));
+        
+        let filtered_count = original_count - stats_data.books.len();
+        log::info!("Filtered statistics to {} books present in library ({} excluded)", 
+            stats_data.books.len(), filtered_count);
+    }
 }