add EAGER_DELETE_COMPACT when Trigger compaction when 30% deleted

Signed-off-by: eric-epsilla <[email protected]>

Author: eric-epsilla

VERSION

@@ -1 +1 @@
-0.3.21
+0.3.22

engine/config/config.hpp

@@ -34,13 +34,17 @@ struct Config {
 
   // Compaction configuration (inspired by Qdrant and Milvus best practices)
   std::atomic<bool> AutoCompaction{true};        // Enable automatic compaction (default: true)
-  std::atomic<double> CompactionThreshold{0.2};  // Trigger compaction when 20% deleted (default: 0.2)
+  std::atomic<double> CompactionThreshold{0.3};  // Trigger compaction when 30% deleted (default: 0.3)
   std::atomic<int> CompactionInterval{3600};     // Check interval in seconds (default: 1 hour)
   std::atomic<int> MinVectorsForCompaction{1000}; // Minimum vectors to trigger compaction (default: 1000)
   std::atomic<int> CompactionMaxDuration{1800};   // Maximum compaction duration in seconds (default: 30 min)
 
   // Memory management configuration
   std::atomic<int> InitialTableCapacity{1000};   // Initial table capacity (default: 1000, was 150000)
+
+  // Eager compaction configuration: run Table::Compact() immediately after large soft-deletes
+  std::atomic<bool> EagerCompactionOnDelete{true}; // Default: true
+  // std::atomic<bool> EagerCompactionOnDelete{false}; --- IGNORE
 
   // Constructor to initialize thread counts based on hardware
   Config() {
@@ -127,6 +131,16 @@ struct Config {
       }
     }
 
+    // Eager compaction toggle after delete (optional)
+    const char* env_eager_delete_compact = std::getenv("EAGER_DELETE_COMPACT");
+    if (env_eager_delete_compact != nullptr) {
+      std::string env_value(env_eager_delete_compact);
+      std::transform(env_value.begin(), env_value.end(), env_value.begin(), ::tolower);
+      bool eager = (env_value == "true" || env_value == "1" || env_value == "yes");
+      EagerCompactionOnDelete.store(eager, std::memory_order_release);
+      printf("[Config] Using EAGER_DELETE_COMPACT=%s from environment\n", eager ? "true" : "false");
+    }
+
     // Check environment variable for initial table capacity
     const char* env_initial_capacity = std::getenv("INITIAL_TABLE_CAPACITY");
     if (env_initial_capacity != nullptr) {
@@ -218,6 +232,9 @@ struct Config {
         InitialTableCapacity.store(capacity, std::memory_order_release);
       }
     }
+    if (json.HasMember("EagerCompactionOnDelete")) {
+      EagerCompactionOnDelete.store(json.GetBool("EagerCompactionOnDelete"), std::memory_order_release);
+    }
   }
 
   // Setter method for SoftDelete mode
@@ -245,6 +262,7 @@ struct Config {
     config.SetInt("MinVectorsForCompaction", MinVectorsForCompaction.load(std::memory_order_acquire));
     config.SetInt("CompactionMaxDuration", CompactionMaxDuration.load(std::memory_order_acquire));
     config.SetInt("InitialTableCapacity", InitialTableCapacity.load(std::memory_order_acquire));
+    config.SetBool("EagerCompactionOnDelete", EagerCompactionOnDelete.load(std::memory_order_acquire));
     return config;
   }
 };

engine/db/table.cpp

@@ -374,18 +374,35 @@ Status Table::Delete(
                  ", deletion_ratio=" + std::to_string(post_deletion_ratio));
     logger_.Debug("[Table] TableSegment deletion completed successfully: " + status.message());
 
-    // Trigger incremental compaction if enabled
-    if (compactor_) {
-      if (post_deletion_ratio > 0.2) {  // 20% threshold for incremental compaction
-        logger_.Debug("[Table] Deletion ratio " + std::to_string(post_deletion_ratio) + 
-                     " exceeds 20% threshold, triggering incremental compaction");
-        compactor_->TriggerCompaction();
-      } else {
-        logger_.Debug("[Table] Deletion ratio " + std::to_string(post_deletion_ratio) + 
-                     " below 20% threshold, no compaction needed");
+    // Eager compaction: immediately compact if enabled and threshold exceeded
+    bool eager = vectordb::globalConfig.EagerCompactionOnDelete.load();
+    double threshold = vectordb::globalConfig.CompactionThreshold.load();
+    if (eager && post_deletion_ratio >= threshold) {
+      logger_.Info("[Table] Eager compaction enabled and deletion ratio " + std::to_string(post_deletion_ratio) +
+                   " >= threshold " + std::to_string(threshold) + ", flushing WAL and compacting now...");
+      // Ensure WAL is flushed so compaction is crash-safe
+      auto wal_status = FlushWAL();
+      if (!wal_status.ok()) {
+        logger_.Warning("[Table] WAL flush failed before eager compaction: " + wal_status.message());
+      }
+      auto compact_status = Compact(threshold);
+      if (!compact_status.ok()) {
+        logger_.Warning("[Table] Eager compaction failed: " + compact_status.message());
       }
     } else {
-      logger_.Debug("[Table] No compactor available, skipping compaction check");
+      // Trigger incremental compaction if available (background), using a fixed 20% hint as before
+      if (compactor_) {
+        if (post_deletion_ratio > 0.2) {
+          logger_.Debug("[Table] Deletion ratio " + std::to_string(post_deletion_ratio) +
+                       " exceeds 20% threshold, triggering incremental compaction");
+          compactor_->TriggerCompaction();
+        } else {
+          logger_.Debug("[Table] Deletion ratio " + std::to_string(post_deletion_ratio) +
+                       " below 20% threshold, no compaction needed");
+        }
+      } else {
+        logger_.Debug("[Table] No compactor available, skipping compaction check");
+      }
     }
   } else {
     logger_.Error("[Table] TableSegment deletion failed: " + status.message());