feat: implement output collapse pipeline and track savings in session stats

Author: fajarhide

src/cli/stats.rs

@@ -181,6 +181,24 @@ pub fn run(args: &[String], store: &Store) -> Result<()> {
         .bright_magenta()
     );
 
+    // Collapse savings
+    let collapse_stats = store.collapse_aggregate(since);
+    if let Ok((events, total_original, total_collapsed)) = collapse_stats
+        && events > 0
+    {
+        println!(
+            "  {:<20} {}",
+            "Collapse:".bright_black(),
+            format!(
+                "{} → {} lines across {} events",
+                format_number(total_original),
+                format_number(total_collapsed),
+                events
+            )
+            .bright_green()
+        );
+    }
+
     // Filter breakdown
     let filters = store.filter_breakdown(since)?;
     if !filters.is_empty() {

src/hooks/pipe.rs

@@ -4,7 +4,7 @@ use std::io::{Read, Write};
 use std::sync::{Arc, Mutex};
 use std::time::Instant;
 
-use crate::pipeline::{SessionState, classifier, composer, scorer, toml_filter};
+use crate::pipeline::{SessionState, classifier, collapse, composer, scorer, toml_filter};
 use crate::store::sqlite::Store;
 use crate::store::transcript::{Transcript, TranscriptEntry};
 
@@ -131,8 +131,13 @@ pub fn run_inner<R: Read, W: Write, E: Write>(
             )
         } else {
             let c = classifier::classify(&input_text);
+
+            // Pre-processing: collapse repetitive lines before scoring
+            let collapse_result = collapse::collapse(&input_text, &c);
+            let effective_input = collapse_result.collapsed_lines.join("\n");
+
             let scored_segments =
-                scorer::score_segments(&input_text, &c, active_session_opt.as_deref());
+                scorer::score_segments(&effective_input, &c, active_session_opt.as_deref());
             drop(active_session_opt);
 
             let compose_config = composer::ComposeConfig::default();
@@ -194,6 +199,7 @@ pub fn run_inner<R: Read, W: Write, E: Write>(
             rewind_hash: rewind_hash_opt,
             segments_kept: kept_count,
             segments_dropped: dropped_count,
+            collapse_savings: None,
         };
 
         s.record_distillation(&s_id, &result, command_name.unwrap_or(""));

src/hooks/post_tool.rs

@@ -1,6 +1,6 @@
 use crate::distillers;
 use crate::pipeline::toml_filter;
-use crate::pipeline::{DistillResult, Route, SessionState, classifier, composer, scorer};
+use crate::pipeline::{DistillResult, Route, SessionState, classifier, collapse, composer, scorer};
 use crate::store::sqlite::Store;
 use serde::{Deserialize, Serialize};
 use std::sync::{Arc, Mutex};
@@ -117,19 +117,23 @@ pub fn process_payload(
         // Fallback to Rust distiller pipeline
         let ctype = classifier::classify(&content);
 
+        // Pre-processing: collapse repetitive lines before scoring
+        let collapse_result = collapse::collapse(&content, &ctype);
+        let effective_input = collapse_result.collapsed_lines.join("\n");
+
         let scored_segments = if let Some(ref lock) = session {
             if let Ok(state) = lock.lock() {
-                scorer::score_segments(&content, &ctype, Some(&*state))
+                scorer::score_segments(&effective_input, &ctype, Some(&*state))
             } else {
-                scorer::score_segments(&content, &ctype, None)
+                scorer::score_segments(&effective_input, &ctype, None)
             }
         } else {
-            scorer::score_segments(&content, &ctype, None)
+            scorer::score_segments(&effective_input, &ctype, None)
         };
 
         let distiller = distillers::get_distiller(&ctype);
         let active_ctype = distiller.content_type();
-        let output = distiller.distill(&scored_segments, &content);
+        let output = distiller.distill(&scored_segments, &effective_input);
         (output, format!("{:?}", active_ctype), Some(active_ctype))
     };
 
@@ -220,6 +224,7 @@ pub fn process_payload(
             },
             segments_kept: 0,
             segments_dropped: 0,
+            collapse_savings: None,
         };
         let session_id = session
             .as_ref()