refactor raw storage; impl bin for lemur

Signed-off-by: SpadeA <tangchenjie1210@gmail.com>

Author: SpadeA-Tang

include/knowhere/index/emb_list_strategy.h

@@ -14,6 +14,7 @@
 
 #include <cstring>
 #include <memory>
+#include <optional>
 #include <string>
 
 #include "knowhere/bitsetview.h"
@@ -47,6 +48,37 @@ struct EmbListMetricInfo {
     bool is_cosine;
 };
 
+// Some strategies build ANN data whose physical type differs from the raw vectors.
+// This target tells IndexNode whether that ANN data can use the current index node
+// or needs a separate index with a different data type.
+enum class EmbListAnnIndexTarget {
+    // Build the ANN dataset directly into this IndexNode's own ANN index. This is used when
+    // the ANN dataset has the same data type as the current index node, such as TokenANN and MUVERA.
+    BaseIndex,
+    // Build the ANN dataset into a separately allocated Index<IndexNode>. This is used when
+    // the strategy's ANN representation has a different data type from the current index node,
+    // such as bin1 LEMUR producing fp32 learned representations.
+    SeparateIndex,
+};
+
+enum class EmbListAnnIndexDataType {
+    // Valid for BaseIndex: the current IndexNode data type is reused.
+    // SeparateIndex must request a concrete data type instead.
+    SameAsBaseIndex,
+    Fp32,
+    Fp16,
+    Bf16,
+    Int8,
+    Bin1,
+};
+
+struct EmbListAnnIndexSpec {
+    EmbListAnnIndexTarget target;
+    EmbListAnnIndexDataType data_type;
+    // nullopt means the ANN index uses the raw sub metric parsed from the outer emb-list metric.
+    std::optional<std::string> ann_metric_type;
+};
+
 /**
  * @brief Parse metric type from config into a shared struct.
  *
@@ -206,6 +238,16 @@ class EmbListStrategy {
         return false;
     }
 
+    /**
+     * @brief Describe how the strategy's ANN dataset should be indexed.
+     *
+     * BaseIndex means the ANN data is indexed by this IndexNode itself. SeparateIndex means IndexNode must create
+     * a child Index<IndexNode> with the requested data type. nullopt ann_metric_type means the ANN index uses the
+     * raw sub metric parsed from the outer emb-list metric; a value means the strategy overrides it.
+     */
+    [[nodiscard]] virtual EmbListAnnIndexSpec
+    AnnIndexSpec(const BaseConfig& config) const = 0;
+
     /**
      * @brief Execute search with full control over the search flow.
      *

include/knowhere/index/index_node.h

@@ -13,6 +13,7 @@
 #define INDEX_NODE_H
 
 #include <functional>
+#include <memory>
 #include <mutex>
 #include <queue>
 #include <utility>
@@ -46,13 +47,11 @@ struct OpContext;
 class ThreadPool;
 #endif
 
-namespace faiss {
-class IndexFlat;
-}  // namespace faiss
-
 namespace knowhere {
 
 class Interrupt;
+class EmbListRawStorage;
+struct EmbListSeparateAnnIndexHolder;
 
 class IndexNode : public Object {
  public:
@@ -318,6 +317,23 @@ class IndexNode : public Object {
     virtual int64_t
     Count() const = 0;
 
+    // Returns the ANN index that currently serves vector-level metadata/search.
+    // EmbList strategies normally use this node itself, but a strategy may build
+    // a child ANN index when its ANN representation has a different data type.
+    virtual IndexNode*
+    AnnIndexNode();
+
+    virtual const IndexNode*
+    AnnIndexNode() const;
+
+    virtual int64_t
+    CountForSearchBitset() const {
+        if (emb_list_strategy_ != nullptr && emb_list_strategy_->GetDocCount() >= 0) {
+            return emb_list_strategy_->GetDocCount();
+        }
+        return Count();
+    }
+
     virtual std::string
     Type() const = 0;
 
@@ -496,6 +512,10 @@ class IndexNode : public Object {
     SearchEmbListIfNeed(const DataSetPtr dataset, std::unique_ptr<Config> config, const BitsetView& bitset,
                         milvus::OpContext* op_context = nullptr) const;
 
+    virtual expected<DataSetPtr>
+    SearchEmbListAnnIndex(const DataSetPtr dataset, std::unique_ptr<Config> config, const BitsetView& bitset,
+                          milvus::OpContext* op_context = nullptr) const;
+
     /**
      * @brief Returns the code size (in bytes) of a single query vector, which varies depending on the data type (e.g.,
      * fp32, bf16, etc).
@@ -619,10 +639,10 @@ class IndexNode : public Object {
 
  protected:
     /**
-     * @brief Compute distances using emb_list_raw_index_ (raw vector storage).
+     * @brief Compute distances using emb_list raw vector storage.
      *
-     * Used by CalcDistByIDs implementations when emb_list_raw_index_ is present
-     * (MUVERA/LEMUR strategies). The raw index stores original vectors indexed by
+     * Used by CalcDistByIDs implementations when emb_list raw storage is present
+     * (MUVERA/LEMUR strategies). The raw storage keeps original vectors indexed by
      * global vector IDs, so no ID translation is needed.
      *
      * @param pool Thread pool for parallel computation
@@ -637,14 +657,14 @@ class IndexNode : public Object {
     std::string el_metric_type_;
     EmbListStrategyPtr emb_list_strategy_;  // emb_list encoding strategy (tokenann/muvera)
     // Raw vector storage for EmbList strategies (MUVERA/LEMUR) that encode documents
-    // into different representations for ANN search. Since the base index holds encoded
-    // vectors (not raw), this IndexFlat stores original vectors for exact distance
-    // computation during MaxSim reranking.
-    // Baseline type so that the same shared_ptr can hold either the knowhere
-    // Jaccard-aware IndexFlat subclass (fresh build path, if ever needed) or
-    // a plain ::faiss::IndexFlat{,IP,L2} restored by the deserialization
-    // factory in cppcontrib/knowhere/impl/index_read.cpp.
-    std::shared_ptr<::faiss::IndexFlat> emb_list_raw_index_;
+    // into different representations for ANN search. The ANN index may be this node's
+    // BaseIndex or a SeparateIndex, so raw storage keeps the original vectors available
+    // for exact distance computation during MaxSim reranking.
+    std::shared_ptr<EmbListRawStorage> emb_list_raw_storage_;
+    // Optional child ANN index used when a strategy's ANN representation has a different
+    // data type from this IndexNode, for example binary LEMUR producing fp32 vectors.
+    std::shared_ptr<EmbListSeparateAnnIndexHolder> emb_list_separate_ann_index_;
+    std::string emb_list_raw_metric_type_;
 
 #if defined(NOT_COMPILE_FOR_SWIG) && !defined(KNOWHERE_WITH_LIGHT)
     struct PrometheusMetrics {