fix(global_index): compatibility for legacy lumina index type (#247) (#249)

Author: lszskye

CMakeLists.txt

@@ -32,7 +32,7 @@ if(NOT CMAKE_BUILD_TYPE)
 endif()
 
 project(paimon
-        VERSION 0.1.2
+        VERSION 0.1.3
         DESCRIPTION "Paimon C++ Project")
 
 string(TOUPPER "${CMAKE_BUILD_TYPE}" UPPERCASE_BUILD_TYPE)

docs/source/conf.py

@@ -113,7 +113,7 @@
 # The master toctree document.
 master_doc = "index"
 
-version = "0.1.2"
+version = "0.1.3"
 
 html_theme_options = {
     "show_toc_level": 2,

src/paimon/common/global_index/global_indexer_factory.cpp

@@ -28,7 +28,9 @@ const char GlobalIndexerFactory::GLOBAL_INDEX_IDENTIFIER_SUFFIX[] = "-global";
 
 Result<std::unique_ptr<GlobalIndexer>> GlobalIndexerFactory::Get(
     const std::string& identifier, const std::map<std::string, std::string>& options) {
-    std::string global_index_identifier = identifier + GLOBAL_INDEX_IDENTIFIER_SUFFIX;
+    // Compatibility: "lumina-vector-ann" was the old identifier for lumina global index.
+    std::string final_identifier = (identifier == "lumina-vector-ann" ? "lumina" : identifier);
+    std::string global_index_identifier = final_identifier + GLOBAL_INDEX_IDENTIFIER_SUFFIX;
     auto factory_creator = FactoryCreator::GetInstance();
     if (factory_creator == nullptr) {
         assert(false);

src/paimon/common/global_index/global_indexer_factory_test.cpp

@@ -38,4 +38,17 @@ TEST(GlobalIndexerFactoryTest, TestNonExist) {
                          GlobalIndexerFactory::Get("nonexist", options));
     ASSERT_FALSE(indexer);
 }
+
+TEST(GlobalIndexerFactoryTest, TestLuminaVectorAnnCompatibility) {
+    // "lumina-vector-ann" should be treated as "lumina" for backward compatibility.
+    // Both identifiers should produce the same result (either both succeed or both return nullptr
+    // depending on whether the lumina module is linked).
+    std::map<std::string, std::string> options;
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<GlobalIndexer> lumina_vector_ann_indexer,
+                         GlobalIndexerFactory::Get("lumina-vector-ann", options));
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr<GlobalIndexer> lumina_indexer,
+                         GlobalIndexerFactory::Get("lumina", options));
+
+    ASSERT_EQ(static_cast<bool>(lumina_vector_ann_indexer), static_cast<bool>(lumina_indexer));
+}
 }  // namespace paimon::test

test/inte/global_index_test.cpp

@@ -2232,6 +2232,15 @@ TEST_P(GlobalIndexTest, TestLuceneWriteCommitScanReadIndexWithScore) {
                                  /*pre_filter=*/std::nullopt)));
         ASSERT_TRUE(index_result->ToString().find("row ids: {0,1,2}") != std::string::npos);
     }
+    {
+        std::optional<RoaringBitmap64> pre_filter = RoaringBitmap64::From({1, 2, 3});
+        ASSERT_OK_AND_ASSIGN(
+            auto index_result,
+            index_reader->VisitFullTextSearch(std::make_shared<FullTextSearch>(
+                "f0",
+                /*limit=*/10, "document", FullTextSearch::SearchType::MATCH_ALL, pre_filter)));
+        ASSERT_TRUE(index_result->ToString().find("row ids: {1,2}") != std::string::npos);
+    }
     {
         ASSERT_OK_AND_ASSIGN(auto index_result,
                              index_reader->VisitFullTextSearch(std::make_shared<FullTextSearch>(

third_party/lumina/OptionsReference.md

@@ -1,2 +1,2 @@
-tag: v0.1.0
-bd7ac880cf34af4d267fcaf16773627cad122463
+tag: v0.2.1
+c88ce90ed44b7037e3a307a36627cbd030e5eb60

third_party/lumina/VERSION

@@ -14,7 +14,6 @@
  * limitations under the License.
  */
 
-
 #pragma once
 
 #include <cstdint>

third_party/lumina/include/lumina/api/Dataset.h

@@ -15,6 +15,7 @@
  */
 
 #pragma once
+#include <cstdint>
 #include <string_view>
 namespace lumina::core {
 
@@ -56,6 +57,7 @@ constexpr std::string_view kEncodingType = "encoding.type"; // Encoding type
 constexpr std::string_view kEncodingRawf32 = "rawf32";
 constexpr std::string_view kEncodingSQ8 = "sq8";
 constexpr std::string_view kEncodingPQ = "pq";
+constexpr std::string_view kEncodingRabitQ = "rabitq";
 constexpr std::string_view kEncodingDummy = "dummy";
 
 // IO options
@@ -88,6 +90,7 @@ constexpr std::string_view kExtensionPrefix = "extension.";
 constexpr std::string_view kExtensionSearchWithFilter = "extension.search_with_filter";
 constexpr std::string_view kExtensionCkptThreshold = "extension.build.ckpt.threshold";
 constexpr std::string_view kExtensionCkptCount = "extension.build.ckpt.count";
+constexpr std::string_view kExtensionGetVector = "extension.search.get_vector";
 
 /* constexpr std::string_view kExtensionFilterDsl = "filter.dsl"; */
 /* constexpr std::string_view kExtensionFilterTags = "filter.tags"; */