[Feature] C++ Extension: Introduce DocNode, NodeTransform, TextSplitterBase, SentenseSplitter

.github/workflows/main.yml

@@ -721,6 +721,7 @@ jobs:
   cpp_ext_test:
     name: C++ Extension Test (${{ matrix.os }})
     needs: [ clone ]
+    if:  always()
     runs-on: ${{ matrix.os }}
     strategy:
       fail-fast: false

.github/workflows/publish_release.yml

@@ -188,12 +188,6 @@ jobs:
           name: repo-with-docs
           path: ./repo_artifact
 
-      - name: Install Python dev headers (Ubuntu only)
-        if: startsWith(matrix.os, 'ubuntu')
-        run: |
-          sudo apt-get update
-          sudo apt-get install -y python3-dev
-
       - name: Extract repo-with-docs
         run: |
           set -ex

.gitignore

@@ -7,6 +7,7 @@ test/
 dist/
 tmp/
 build
+.cache/
 *.lock
 *.db
 mkdocs.yml

csrc/CMakeLists.txt

@@ -1,23 +1,39 @@
 cmake_minimum_required(VERSION 3.16)
 project(LazyLLMCPP LANGUAGES CXX)
 
-set(CMAKE_CXX_STANDARD 11)
+set(CMAKE_CXX_STANDARD 17)
 set(CMAKE_CXX_STANDARD_REQUIRED ON)
 set(CMAKE_POSITION_INDEPENDENT_CODE ON)
+add_compile_options(
+    -Werror
+    -Wshadow
+)
 
-find_package(Python3 COMPONENTS Interpreter Development.Module REQUIRED)
-find_package(pybind11 CONFIG REQUIRED)
+# Third party libs
+include(cmake/third_party.cmake)
 
 # Config lazyllm_core lib with pure cpp code.
-file(GLOB_RECURSE LAZYLLM_CORE_SOURCES CONFIGURE_DEPENDS "${CMAKE_CURRENT_SOURCE_DIR}/src/*.cpp")
+file(GLOB_RECURSE LAZYLLM_CORE_SOURCES CONFIGURE_DEPENDS
+    "${CMAKE_CURRENT_SOURCE_DIR}/core/src/*.cpp")
 add_library(lazyllm_core STATIC ${LAZYLLM_CORE_SOURCES})
-target_include_directories(lazyllm_core PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/include)
+target_include_directories(lazyllm_core PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/core/include)
+target_link_libraries(lazyllm_core PUBLIC xxhash)
+target_link_libraries(lazyllm_core PUBLIC sentencepiece)
+target_include_directories(lazyllm_core PUBLIC ${sentencepiece_SOURCE_DIR}/src)
+
+# Config lazyllm_adaptor lib which maintains callback invocations.
+file(GLOB_RECURSE LAZYLLM_ADAPTOR_SOURCES CONFIGURE_DEPENDS
+    "${CMAKE_CURRENT_SOURCE_DIR}/adaptor/*.cpp")
+add_library(lazyllm_adaptor STATIC ${LAZYLLM_ADAPTOR_SOURCES})
+target_include_directories(lazyllm_adaptor PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/adaptor)
+target_link_libraries(lazyllm_adaptor PUBLIC pybind11::headers Python3::Python lazyllm_core)
 
 # Config lazyllm_cpp lib with binding infomations.
-set(LAZYLLM_BINDING_SOURCES binding/lazyllm.cpp binding/doc.cpp)
+file(GLOB_RECURSE LAZYLLM_BINDING_SOURCES CONFIGURE_DEPENDS
+    "${CMAKE_CURRENT_SOURCE_DIR}/binding/*.cpp")
 set(INTERFACE_TARGET_NAME lazyllm_cpp)
 pybind11_add_module(${INTERFACE_TARGET_NAME} ${LAZYLLM_BINDING_SOURCES})
-target_link_libraries(${INTERFACE_TARGET_NAME} PRIVATE lazyllm_core)
+target_link_libraries(${INTERFACE_TARGET_NAME} PRIVATE lazyllm_core lazyllm_adaptor)
 
 if (CMAKE_BUILD_TYPE STREQUAL "Debug")
     # SHOW_SYMBOL

csrc/adaptor/adaptor.cpp

@@ -0,0 +1,2 @@
+#include "adaptor_base_wrapper.hpp"
+#include "document_store.hpp"

csrc/adaptor/adaptor_base_wrapper.hpp

@@ -0,0 +1,37 @@
+#pragma once
+
+#include <memory>
+#include <mutex>
+#include <string>
+#include <unordered_map>
+#include <vector>
+
+#include <pybind11/pybind11.h>
+
+#include "adaptor_base.hpp"
+
+
+namespace lazyllm {
+
+class LAZYLLM_HIDDEN AdaptorBaseWrapper : public AdaptorBase {
+    pybind11::object _py_obj;
+public:
+    AdaptorBaseWrapper(const pybind11::object &obj) : _py_obj(obj) {}
+    virtual ~AdaptorBaseWrapper() = default;
+
+    std::any call(
+        const std::string& func_name,
+        const std::unordered_map<std::string, std::any>& args) const override final
+    {
+        pybind11::gil_scoped_acquire gil;
+        pybind11::object func = pybind11::getattr(_py_obj, func_name.c_str(), pybind11::none());
+        return call_impl(func_name, func, args);
+    }
+
+    virtual std::any call_impl(
+        const std::string& func_name,
+        const pybind11::object& func,
+        const std::unordered_map<std::string, std::any>& args) const = 0;
+};
+
+}

csrc/adaptor/document_store.hpp

@@ -0,0 +1,119 @@
+#pragma once
+
+#include <memory>
+#include <string>
+#include <unordered_map>
+#include <vector>
+
+#include <pybind11/pybind11.h>
+#include <pybind11/stl.h>
+
+#include "adaptor_base_wrapper.hpp"
+#include "doc_node.hpp"
+
+namespace lazyllm {
+
+struct NodeGroup {
+    enum class Type {
+        ORIGINAL, CHUNK, SUMMARY, IMAGE_INFO, QUESTION_ANSWER, OTHER
+    };
+    std::string _parent;
+    std::string _display_name;
+    Type _type;
+    NodeGroup(
+        const std::string& parent,
+        const std::string& display_name,
+        const Type& type = Type::ORIGINAL) :
+            _parent(parent), _display_name(display_name), _type(type) {}
+};
+
+class LAZYLLM_HIDDEN DocumentStore : public AdaptorBaseWrapper {
+public:
+    DocumentStore() = delete;
+    explicit DocumentStore(
+        const pybind11::object& store,
+        const std::unordered_map<std::string, NodeGroup> &map) :
+            AdaptorBaseWrapper(store), _node_groups_map(map) {}
+
+    // Cache-aware factory to avoid rebuilding adaptor for the same Python store.
+    static std::shared_ptr<DocumentStore> from_store(
+        const pybind11::object& store, const std::unordered_map<std::string, NodeGroup>& map) {
+        if (store.is_none()) return nullptr;
+
+        pybind11::gil_scoped_acquire gil;
+        PyObject *key = store.ptr();
+        auto &cache = store_cache();
+        auto it = cache.find(key);
+        if (it != cache.end()) {
+            if (auto existing = it->second.lock())
+                return existing;
+        }
+        auto created = std::make_shared<DocumentStore>(store, map);
+        cache[key] = created;
+        return created;
+    }
+
+    DocNode::Children get_node_children(const DocNode* node) const {
+        DocNode::Children out;
+        auto& kb_id = std::any_cast<std::string&>(node->_p_global_metadata->at(std::string(RAG_KEY_KB_ID)));
+        auto& doc_id = std::any_cast<std::string&>(node->_p_global_metadata->at(std::string(RAG_KEY_DOC_ID)));
+        auto& group_name = node->get_group_name();
+        for(auto& [current_group_name, group] : _node_groups_map) {
+            if (group._parent != group_name) continue;
+            if (!std::any_cast<bool>(call("is_group_active", {{"group", current_group_name}}))) continue;
+            auto nodes_in_group = std::any_cast<std::vector<DocNode*>>(call("get_nodes", {
+                {"group_name", current_group_name},
+                {"kb_id", kb_id},
+                {"doc_ids", std::vector<std::string>({doc_id})}
+            }));
+
+            std::vector<DocNode*> children;
+            children.reserve(nodes_in_group.size());
+            for (auto* n : nodes_in_group)
+                if (n->get_parent_node() == node) children.push_back(n);
+            out[current_group_name] = children;
+        }
+        return out;
+    }
+
+private:
+    std::unordered_map<std::string, NodeGroup> _node_groups_map;
+
+    std::any call_impl(
+        const std::string& func_name,
+        const pybind11::object& func,
+        const std::unordered_map<std::string, std::any>& args) const override
+    {
+        if (func_name == "is_group_active") {
+            return func(args.at("group")).cast<bool>();
+        }
+        else if (func_name == "get_node") {
+            return func(
+                pybind11::arg("group_name") = std::any_cast<std::string>(args.at("group_name")),
+                pybind11::arg("uids") = std::vector<std::string>({std::any_cast<std::string>(args.at("uid"))}),
+                pybind11::arg("kb_id") = std::any_cast<std::string>(args.at("kb_id")),
+                pybind11::arg("display") = true
+            ).cast<pybind11::list>()[0].cast<DocNode*>();
+        }
+        else if (func_name == "get_nodes") {
+            return func(
+                pybind11::arg("group_name") = std::any_cast<std::string>(args.at("group_name")),
+                pybind11::arg("kb_id") = std::any_cast<std::string>(args.at("kb_id")),
+                pybind11::arg("doc_ids") = std::vector<std::string>({std::any_cast<std::string>(args.at("doc_id"))})
+            ).cast<std::vector<DocNode*>>();
+        }
+        else if (func_name == "get_node_children") {
+            return get_node_children(std::any_cast<DocNode*>(args.at("node")));
+        }
+
+        throw std::runtime_error("Unknown DocumentStore function: " + func_name);
+    }
+
+    // Cache by Python object identity to ensure one wrapper per store instance.
+    static std::unordered_map<PyObject *, std::weak_ptr<DocumentStore>> &store_cache() {
+        static std::unordered_map<PyObject *, std::weak_ptr<DocumentStore>> cache;
+        return cache;
+    }
+};
+
+} // namespace lazyllm

csrc/binding/doc.cpp → csrc/binding/export_add_doc_str.cpp

@@ -28,6 +28,6 @@ void addDocStr(py::object obj, std::string docs) {
     }
 }
 
-void exportDoc(py::module& m) {
+void exportAddDocStr(py::module& m) {
     m.def("add_doc", &addDocStr, "Add docstring to a function or method", py::arg("obj"), py::arg("docs"));
 }