enhance: skip fully filtered windows in sindi sparse search (#1782)

* enhance: skip fully filtered windows in sindi sparse search

Skip the window scoring when the bitset filters out every document in
the window. Keep posting cursors in sync when a window is skipped so
cumulative posting offsets stay correct for subsequent windows.

Signed-off-by: chasingegg <gaoc96@qq.com>

* test: extend sindi window filter skip test to BM25

Signed-off-by: chasingegg <gaoc96@qq.com>

* chore: trigger ci

Signed-off-by: chasingegg <gaoc96@qq.com>

---------

Signed-off-by: chasingegg <gaoc96@qq.com>

Author: chasingegg

src/index/sparse/sindi_inverted_index.h

@@ -932,6 +932,18 @@ class SindiInvertedIndex : public DimMapInvertedIndex<DataType, AllowIncremental
         const uint32_t wnnz_bits = 32 - __builtin_clz(window_size_);
         const uint32_t wnnz_mask = (1u << wnnz_bits) - 1;
 
+        // Skip a window when every document in it is filtered out by the bitset.
+        const auto window_all_filtered = [&bitset](size_t docid_start, size_t window_doc_count) {
+            if (bitset.empty()) {
+                return false;
+            }
+            const size_t end = docid_start + window_doc_count;
+            if (end > bitset.size()) {
+                return false;
+            }
+            return bitset.range_all_filtered(docid_start, end);
+        };
+
         // Initialize posting list cursors for each query term
         // Cursors track position in posting lists and handle window-by-window iteration
         std::vector<PostingCursor> cursors;
@@ -971,6 +983,17 @@ class SindiInvertedIndex : public DimMapInvertedIndex<DataType, AllowIncremental
                 const size_t docid_start = window_size_ * widx;
                 const uint32_t curr_window_size =
                     std::min(window_size_, static_cast<uint32_t>(this->nr_rows_ - docid_start));
+                if (window_all_filtered(docid_start, curr_window_size)) {
+                    // Keep posting cursors in sync even when the window is skipped, so the
+                    // cumulative posting offsets stay correct for subsequent windows.
+                    for (auto& cur : cursors) {
+                        if (cur.wnnz_buf == nullptr || cur.wnnz_buf_sz == 0) {
+                            continue;
+                        }
+                        cur.advance_window(widx);
+                    }
+                    continue;
+                }
                 std::fill_n(wscores_final, curr_window_size, 0);
 
                 float curr_max_score = 0.0f;
@@ -1019,6 +1042,17 @@ class SindiInvertedIndex : public DimMapInvertedIndex<DataType, AllowIncremental
                 const size_t docid_start = window_size_ * widx;
                 const uint32_t curr_window_size =
                     std::min(window_size_, static_cast<uint32_t>(this->nr_rows_ - docid_start));
+                if (window_all_filtered(docid_start, curr_window_size)) {
+                    // Keep posting cursors in sync even when the window is skipped, so the
+                    // cumulative posting offsets stay correct for subsequent windows.
+                    for (auto& cur : cursors) {
+                        if (cur.wnnz_buf == nullptr || cur.wnnz_buf_sz == 0) {
+                            continue;
+                        }
+                        cur.advance_window(widx);
+                    }
+                    continue;
+                }
                 std::fill_n(wscores_final, curr_window_size, 0);
 
                 float curr_max_score = 0.0f;

tests/ut/test_sparse.cc

@@ -1711,6 +1711,74 @@ TEST_CASE("Test SINDI Index Window Size", "[sparse][sindi]") {
     REQUIRE(recall >= 0.85);
 }
 
+TEST_CASE("Test SINDI Index Search with Window Filter Skip", "[sparse][sindi]") {
+    auto nb = 3000;
+    auto dim = 500;
+    auto topk = 10;
+    int64_t nq = 5;
+    auto doc_sparsity = 0.97f;
+    auto query_sparsity = 0.99f;
+    constexpr int32_t window_size = 1024;
+
+    auto metric = GENERATE(knowhere::metric::IP, knowhere::metric::BM25);
+
+    auto version = knowhere::Version::GetMaximumVersion().VersionNumber();
+    auto sparse_dataset_gen = [&](int nr, float sparsity) -> knowhere::DataSetPtr {
+        if (metric == knowhere::metric::BM25) {
+            return GenSparseDataSetWithMaxVal(nr, dim, sparsity, 256, true);
+        }
+        return GenSparseDataSet(nr, dim, sparsity);
+    };
+    auto train_ds = sparse_dataset_gen(nb, doc_sparsity);
+    auto query_ds = sparse_dataset_gen(nq, query_sparsity);
+
+    // Filter out the leading `filtered_docs` documents so that one or more whole windows
+    // are skipped. Skipping leading windows exercises the posting-cursor sync path.
+    auto filtered_docs = GENERATE(1024, 2048);
+    CAPTURE(filtered_docs);
+    auto bitset_data = GenerateBitsetWithFirstTbitsSet(nb, filtered_docs);
+    knowhere::BitsetView bitset(bitset_data.data(), nb);
+
+    knowhere::Json build_json;
+    build_json[knowhere::meta::DIM] = dim;
+    build_json[knowhere::meta::METRIC_TYPE] = metric;
+    build_json[knowhere::indexparam::INVERTED_INDEX_ALGO] = "SINDI";
+    build_json["sindi_window_size"] = window_size;
+    build_json[knowhere::meta::BM25_K1] = 1.2;
+    build_json[knowhere::meta::BM25_B] = 0.75;
+    build_json[knowhere::meta::BM25_AVGDL] = 100;
+
+    knowhere::Json search_json;
+    search_json[knowhere::meta::TOPK] = topk;
+    search_json[knowhere::meta::METRIC_TYPE] = metric;
+    search_json[knowhere::meta::BM25_K1] = 1.2;
+    search_json[knowhere::meta::BM25_B] = 0.75;
+    search_json[knowhere::meta::BM25_AVGDL] = 100;
+
+    auto expected = knowhere::BruteForce::SearchSparse(train_ds, query_ds, search_json, bitset);
+    REQUIRE(expected.has_value());
+
+    auto idx = knowhere::IndexFactory::Instance()
+                   .Create<knowhere::sparse_u32_f32>(knowhere::IndexEnum::INDEX_SPARSE_INVERTED_INDEX, version)
+                   .value();
+    REQUIRE(idx.Build(train_ds, build_json) == knowhere::Status::success);
+
+    auto results = idx.Search(query_ds, search_json, bitset);
+    REQUIRE(results.has_value());
+    REQUIRE(GetKNNRecall(*expected.value(), *results.value()) >= 0.99f);
+
+    auto* ids = results.value()->GetIds();
+    auto k = results.value()->GetDim();
+    for (int64_t i = 0; i < nq; ++i) {
+        for (int64_t j = 0; j < k; ++j) {
+            if (ids[i * k + j] == -1) {
+                break;
+            }
+            REQUIRE(!bitset.test(ids[i * k + j]));
+        }
+    }
+}
+
 TEST_CASE("Test SINDI Index Search Algo Mismatch", "[sparse][sindi]") {
     auto nb = 500;
     auto dim = 300;