feat: add GIXmake size limit detection and adaptive retry (fix-gixmake-fails)

- Add IndexCreationError type with size limit detection in fastga_integration.rs
- Extend run_batch_alignment_with_budget() with adaptive retry for GIXmake failures
- Apply halving backoff when index creation fails due to size limits (≥48MB)
- Add helpful error messages in other batch functions suggesting manual fixes
- Document GIXmake practical limits: ≤40MB safe, ≥48MB often fail
- Support up to 5 automatic restarts with progressively smaller batches

Fixes silent GIXmake failures that occurred when batch sizes exceeded ~48MB
of sequence data, typically affecting batches with ≥4 yeast-sized genomes.

Author: ekg

src/batch_align.rs

@@ -3,13 +3,44 @@
 //! When aligning many genomes, the aligner's resource usage can exceed available
 //! disk or memory. This module partitions genomes into batches based on total
 //! sequence data size, runs the aligner on each batch pair, and aggregates results.
+//!
+//! ## GIXmake Index Size Limits
+//!
+//! FastGA's GIXmake indexer has practical size limits for k-mer index creation:
+//! - **Safe range**: Batches with ≤40MB of sequence data typically succeed
+//! - **Failure threshold**: Batches with ≥48MB often fail silently during index creation
+//! - **Recommendation**: For yeast-sized genomes (~12MB each), keep batches to ≤3 genomes
+//!
+//! When GIXmake failures occur, `run_batch_alignment_with_budget()` automatically
+//! reduces batch size and retries. Other batch functions provide error messages
+//! suggesting manual --batch-bytes adjustment or switching to wfmash (no size limit).
 
 use anyhow::{Context, Result};
 use std::collections::{HashMap, HashSet};
 use std::fs::File;
 use std::io::{BufRead, BufReader, Write};
 use std::path::{Path, PathBuf};
 
+/// Reasons why a batch alignment attempt might need to restart
+#[derive(Debug)]
+pub enum RestartReason {
+    /// Disk budget exceeded during alignment
+    BudgetExceeded,
+    /// GIXmake index creation failed due to size limits
+    IndexSizeLimitExceeded { batch_size_mb: u64 },
+}
+
+impl std::fmt::Display for RestartReason {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        match self {
+            RestartReason::BudgetExceeded => write!(f, "disk budget exceeded"),
+            RestartReason::IndexSizeLimitExceeded { batch_size_mb } => {
+                write!(f, "GIXmake index size limit exceeded ({}MB)", batch_size_mb)
+            }
+        }
+    }
+}
+
 /// Trait for aligner-specific batch operations.
 ///
 /// The batch loop calls these methods in order:
@@ -758,6 +789,7 @@ pub fn run_batch_alignment_with_budget(
     let largest_genome = genome_bp.iter().copied().max().unwrap_or(0);
     let mut max_batch_bp = initial_batch_bp;
     let mut restarts = 0u32;
+    let mut restart_reason = RestartReason::BudgetExceeded; // Default, updated on GIXmake failure
 
     let temp_base = if let Some(dir) = tempdir {
         PathBuf::from(dir)
@@ -828,7 +860,36 @@ pub fn run_batch_alignment_with_budget(
 
         // Phase 2: For each target batch, prepare, align all queries, cleanup
         for i in 0..num_batches {
-            aligner.prepare_target(i, config.quiet)?;
+            // Try to prepare target batch - this may fail due to GIXmake size limits
+            match aligner.prepare_target(i, config.quiet) {
+                Ok(()) => {
+                    // Success, continue with batch processing
+                },
+                Err(e) => {
+                    // Check if this is a GIXmake size limit error that should trigger restart
+                    if let Some(error_chain) = e.chain().find_map(|err| {
+                        err.downcast_ref::<crate::fastga_integration::IndexCreationError>()
+                    }) {
+                        if let crate::fastga_integration::IndexCreationError::SizeLimitExceeded { batch_size_mb, .. } = error_chain {
+                            eprintln!(
+                                "[gixmake] Index creation failed for batch {} ({}MB) - batch too large for GIXmake",
+                                i + 1, batch_size_mb
+                            );
+
+                            // Set restart reason and clean up
+                            restart_reason = RestartReason::IndexSizeLimitExceeded {
+                                batch_size_mb: *batch_size_mb
+                            };
+                            aligner.cleanup_all()?;
+                            budget_exceeded = true; // Reuse existing restart mechanism
+                            break;
+                        }
+                    }
+
+                    // Not a GIXmake size limit error - propagate the original error
+                    return Err(e);
+                }
+            }
 
             // Budget check after prepare_target — the index is the dominant cost
             let (exceeded, current, _) =
@@ -929,12 +990,24 @@ pub fn run_batch_alignment_with_budget(
         // Budget was exceeded — apply halving backoff
         restarts += 1;
         if restarts as usize > MAX_RESTARTS {
-            anyhow::bail!(
-                "Exceeded max restarts ({}) while trying to fit within disk budget {}. \
-                 Use --zstd to halve index size or increase --max-disk.",
-                MAX_RESTARTS,
-                format_bytes(disk_budget),
-            );
+            match restart_reason {
+                RestartReason::BudgetExceeded => {
+                    anyhow::bail!(
+                        "Exceeded max restarts ({}) while trying to fit within disk budget {}. \
+                         Use --zstd to halve index size or increase --max-disk.",
+                        MAX_RESTARTS,
+                        format_bytes(disk_budget),
+                    );
+                }
+                RestartReason::IndexSizeLimitExceeded { batch_size_mb } => {
+                    anyhow::bail!(
+                        "Exceeded max restarts ({}) due to GIXmake index size limits (last failed batch: {}MB). \
+                         Try reducing --batch-bytes further or use a different aligner (wfmash) that doesn't have this limit.",
+                        MAX_RESTARTS,
+                        batch_size_mb
+                    );
+                }
+            }
         }
 
         let old_batch_bp = max_batch_bp;
@@ -952,13 +1025,30 @@ pub fn run_batch_alignment_with_budget(
             }
         }
 
-        eprintln!(
-            "[budget] Restart {}/{}: reducing batch from {} to {}",
-            restarts,
-            MAX_RESTARTS,
-            format_bytes(old_batch_bp),
-            format_bytes(max_batch_bp),
-        );
+        match restart_reason {
+            RestartReason::BudgetExceeded => {
+                eprintln!(
+                    "[budget] Restart {}/{}: reducing batch from {} to {} (disk budget exceeded)",
+                    restarts,
+                    MAX_RESTARTS,
+                    format_bytes(old_batch_bp),
+                    format_bytes(max_batch_bp),
+                );
+            }
+            RestartReason::IndexSizeLimitExceeded { batch_size_mb } => {
+                eprintln!(
+                    "[gixmake] Restart {}/{}: reducing batch from {} to {} (GIXmake index size limit, {}MB batch failed)",
+                    restarts,
+                    MAX_RESTARTS,
+                    format_bytes(old_batch_bp),
+                    format_bytes(max_batch_bp),
+                    batch_size_mb
+                );
+            }
+        }
+
+        // Reset restart reason for next iteration
+        restart_reason = RestartReason::BudgetExceeded;
     }
 }
 
@@ -1041,7 +1131,24 @@ pub fn run_batch_alignment_generic(
 
     // Phase 2: For each target batch, prepare, align all queries, cleanup
     for i in 0..num_batches {
-        aligner.prepare_target(i, config.quiet)?;
+        // Try to prepare target batch - provide helpful error for GIXmake size limit failures
+        if let Err(e) = aligner.prepare_target(i, config.quiet) {
+            // Check if this is a GIXmake size limit error
+            if let Some(error_chain) = e.chain().find_map(|err| {
+                err.downcast_ref::<crate::fastga_integration::IndexCreationError>()
+            }) {
+                if let crate::fastga_integration::IndexCreationError::SizeLimitExceeded { batch_size_mb, suggested_limit, .. } = error_chain {
+                    return Err(anyhow::anyhow!(
+                        "GIXmake index creation failed for batch {} ({}MB). \
+                         This batch size exceeds FastGA's index limit. \
+                         Try --batch-bytes {}M or use run_batch_alignment_with_budget() for automatic retry.",
+                        i + 1, batch_size_mb, suggested_limit
+                    ));
+                }
+            }
+            // Not a GIXmake error - propagate original error
+            return Err(e);
+        }
 
         for j in 0..num_batches {
             if !config.quiet {
@@ -1172,7 +1279,24 @@ pub fn run_batch_alignment_by_count(
     let num_batches = batch_files.len();
 
     for i in 0..num_batches {
-        aligner.prepare_target(i, config.quiet)?;
+        // Try to prepare target batch - provide helpful error for GIXmake size limit failures
+        if let Err(e) = aligner.prepare_target(i, config.quiet) {
+            // Check if this is a GIXmake size limit error
+            if let Some(error_chain) = e.chain().find_map(|err| {
+                err.downcast_ref::<crate::fastga_integration::IndexCreationError>()
+            }) {
+                if let crate::fastga_integration::IndexCreationError::SizeLimitExceeded { batch_size_mb, suggested_limit, .. } = error_chain {
+                    return Err(anyhow::anyhow!(
+                        "GIXmake index creation failed for batch {} ({}MB). \
+                         This batch size exceeds FastGA's index limit. \
+                         Try --batch-bytes {}M or use the disk budget mode for automatic retry.",
+                        i + 1, batch_size_mb, suggested_limit
+                    ));
+                }
+            }
+            // Not a GIXmake error - propagate original error
+            return Err(e);
+        }
 
         for j in 0..num_batches {
             if !config.quiet {

src/fastga_integration.rs

@@ -11,6 +11,36 @@ use std::sync::atomic::{AtomicBool, Ordering};
 use std::sync::Arc;
 use tempfile::NamedTempFile;
 
+/// Error types specific to GIXmake index creation
+#[derive(Debug)]
+pub enum IndexCreationError {
+    /// Index size likely exceeds GIXmake's internal limits (≥48MB sequence data)
+    SizeLimitExceeded {
+        batch_size_mb: u64,
+        suggested_limit: u64,
+        underlying_error: String
+    },
+    /// Other index creation failure
+    Other(String),
+}
+
+impl std::fmt::Display for IndexCreationError {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        match self {
+            IndexCreationError::SizeLimitExceeded { batch_size_mb, suggested_limit, underlying_error } => {
+                write!(f,
+                    "GIXmake index creation failed: batch size {}MB likely exceeds FastGA's index size limit. \
+                     Try --batch-bytes {}M or smaller. Original error: {}",
+                    batch_size_mb, suggested_limit, underlying_error
+                )
+            }
+            IndexCreationError::Other(msg) => write!(f, "Index creation failed: {}", msg),
+        }
+    }
+}
+
+impl std::error::Error for IndexCreationError {}
+
 /// Get the preferred temp directory for FastGA operations.
 /// Priority: explicit override > TMPDIR env var > current directory.
 ///
@@ -292,11 +322,48 @@ impl FastGAIntegration {
                 gdb_base,
                 self.config.adaptive_seed_cutoff.unwrap_or(10) as i32,
             )
-            .map_err(|e| anyhow::anyhow!("Failed to create index: {e}"))?;
+            .map_err(|e| {
+                // Try to determine batch size from GDB file for better error messages
+                let batch_size_mb = Self::estimate_batch_size_mb(gdb_base).unwrap_or(0);
+
+                // Check if this looks like a GIXmake size limit failure
+                if Self::is_likely_size_limit_error(&e.to_string(), batch_size_mb) {
+                    anyhow::anyhow!(IndexCreationError::SizeLimitExceeded {
+                        batch_size_mb,
+                        suggested_limit: std::cmp::max(32, batch_size_mb * 3 / 4), // Suggest 25% reduction
+                        underlying_error: e.to_string()
+                    })
+                } else {
+                    anyhow::anyhow!(IndexCreationError::Other(e.to_string()))
+                }
+            })?;
 
         Ok(())
     }
 
+    /// Estimate the batch size in MB from the GDB file size
+    fn estimate_batch_size_mb(gdb_base: &str) -> Result<u64> {
+        let gdb_path = format!("{}.1gdb", gdb_base);
+        let metadata = std::fs::metadata(&gdb_path)?;
+        // GDB file size is roughly proportional to sequence data size
+        // Use a conservative multiplier to estimate original sequence size
+        Ok((metadata.len() / (1024 * 1024)) * 3) // Rough approximation
+    }
+
+    /// Check if an error message indicates a likely GIXmake size limit failure
+    fn is_likely_size_limit_error(error_msg: &str, batch_size_mb: u64) -> bool {
+        // GIXmake size limit failures typically occur at ≥48MB sequence data
+        if batch_size_mb >= 40 {
+            // Look for typical patterns in GIXmake/FastGA failure messages
+            error_msg.contains("GIXmake")
+                || error_msg.contains("code None")
+                || error_msg.contains("failed with code")
+                || (error_msg.is_empty() && batch_size_mb >= 48) // Silent failures
+        } else {
+            false
+        }
+    }
+
     /// Compress ktab index files using zstd seekable format
     /// This reduces disk usage by ~2x and can improve I/O performance
     /// Level: 1-19 (higher = smaller files but slower compression)