27
27
#include " impl/odescent_graph_builder.h"
28
28
#include " impl/pruning_strategy.h"
29
29
#include " index/iterator_filter.h"
30
+ #include " logger.h"
31
+ #include " serialization.h"
32
+ #include " stream_reader.h"
33
+ #include " typing.h"
30
34
#include " utils/standard_heap.h"
31
35
#include " utils/util_functions.h"
36
+ #include " vsag/options.h"
32
37
33
38
namespace vsag {
34
39
@@ -607,7 +612,7 @@ HGraph::RangeSearch(const DatasetPtr& query,
607
612
}
608
613
609
614
void
610
- HGraph::serialize_basic_info (StreamWriter& writer) const {
615
+ HGraph::serialize_basic_info_v0_14 (StreamWriter& writer) const {
611
616
StreamWriter::WriteObj (writer, this ->use_reorder_ );
612
617
StreamWriter::WriteObj (writer, this ->dim_ );
613
618
StreamWriter::WriteObj (writer, this ->metric_ );
@@ -629,12 +634,120 @@ HGraph::serialize_basic_info(StreamWriter& writer) const {
629
634
}
630
635
}
631
636
637
+ void
638
+ HGraph::deserialize_basic_info_v0_14 (StreamReader& reader) {
639
+ StreamReader::ReadObj (reader, this ->use_reorder_ );
640
+ StreamReader::ReadObj (reader, this ->dim_ );
641
+ StreamReader::ReadObj (reader, this ->metric_ );
642
+ uint64_t max_level;
643
+ StreamReader::ReadObj (reader, max_level);
644
+ for (uint64_t i = 0 ; i < max_level; ++i) {
645
+ this ->route_graphs_ .emplace_back (this ->generate_one_route_graph ());
646
+ }
647
+ StreamReader::ReadObj (reader, this ->entry_point_id_ );
648
+ StreamReader::ReadObj (reader, this ->ef_construct_ );
649
+ StreamReader::ReadObj (reader, this ->mult_ );
650
+ InnerIdType capacity;
651
+ StreamReader::ReadObj (reader, capacity);
652
+ this ->max_capacity_ .store (capacity);
653
+ StreamReader::ReadVector (reader, this ->label_table_ ->label_table_ );
654
+
655
+ uint64_t size;
656
+ StreamReader::ReadObj (reader, size);
657
+ for (uint64_t i = 0 ; i < size; ++i) {
658
+ LabelType key;
659
+ StreamReader::ReadObj (reader, key);
660
+ InnerIdType value;
661
+ StreamReader::ReadObj (reader, value);
662
+ this ->label_table_ ->label_remap_ .emplace (key, value);
663
+ }
664
+ }
665
+
666
+ #define TO_JSON (json_obj, var ) json_obj[#var] = this ->var##_;
667
+
668
+ #define TO_JSON_BASE64 (json_obj, var ) json_obj[#var] = base64_encode_obj(this ->var##_);
669
+
670
+ #define TO_JSON_ATOMIC (json_obj, var ) json_obj[#var] = this ->var##_.load();
671
+
672
+ JsonType
673
+ HGraph::serialize_basic_info () const {
674
+ JsonType jsonify_basic_info;
675
+ TO_JSON (jsonify_basic_info, use_reorder);
676
+ TO_JSON (jsonify_basic_info, dim);
677
+ TO_JSON (jsonify_basic_info, metric);
678
+ TO_JSON (jsonify_basic_info, entry_point_id);
679
+ TO_JSON (jsonify_basic_info, ef_construct);
680
+ // logger::debug("mult: {}", this->mult_);
681
+ TO_JSON_BASE64 (jsonify_basic_info, mult);
682
+ TO_JSON_ATOMIC (jsonify_basic_info, max_capacity);
683
+ jsonify_basic_info[" max_level" ] = this ->route_graphs_ .size ();
684
+
685
+ return jsonify_basic_info;
686
+ }
687
+
688
+ #define FROM_JSON (json_obj, var ) this ->var##_ = (json_obj)[#var];
689
+
690
+ #define FROM_JSON_BASE64 (json_obj, var ) base64_decode_obj((json_obj)[#var], this ->var##_);
691
+
692
+ #define FROM_JSON_ATOMIC (json_obj, var ) this ->var##_.store((json_obj)[#var]);
693
+
694
+ void
695
+ HGraph::deserialize_basic_info (JsonType jsonify_basic_info) {
696
+ FROM_JSON (jsonify_basic_info, use_reorder);
697
+ FROM_JSON (jsonify_basic_info, dim);
698
+ FROM_JSON (jsonify_basic_info, metric);
699
+ FROM_JSON (jsonify_basic_info, entry_point_id);
700
+ FROM_JSON (jsonify_basic_info, ef_construct);
701
+ FROM_JSON_BASE64 (jsonify_basic_info, mult);
702
+ // logger::debug("mult: {}", this->mult_);
703
+ FROM_JSON_ATOMIC (jsonify_basic_info, max_capacity);
704
+
705
+ uint64_t max_level = jsonify_basic_info[" max_level" ];
706
+ for (uint64_t i = 0 ; i < max_level; ++i) {
707
+ this ->route_graphs_ .emplace_back (this ->generate_one_route_graph ());
708
+ }
709
+ }
710
+
711
+ void
712
+ HGraph::serialize_label_info (StreamWriter& writer) const {
713
+ StreamWriter::WriteVector (writer, this ->label_table_ ->label_table_ );
714
+ uint64_t size = this ->label_table_ ->label_remap_ .size ();
715
+ StreamWriter::WriteObj (writer, size);
716
+ for (const auto & pair : this ->label_table_ ->label_remap_ ) {
717
+ auto key = pair.first ;
718
+ StreamWriter::WriteObj (writer, key);
719
+ StreamWriter::WriteObj (writer, pair.second );
720
+ }
721
+ }
722
+
723
+ void
724
+ HGraph::deserialize_label_info (StreamReader& reader) const {
725
+ StreamReader::ReadVector (reader, this ->label_table_ ->label_table_ );
726
+ uint64_t size;
727
+ StreamReader::ReadObj (reader, size);
728
+ for (uint64_t i = 0 ; i < size; ++i) {
729
+ LabelType key;
730
+ StreamReader::ReadObj (reader, key);
731
+ InnerIdType value;
732
+ StreamReader::ReadObj (reader, value);
733
+ this ->label_table_ ->label_remap_ .emplace (key, value);
734
+ }
735
+ }
736
+
632
737
void
633
738
HGraph::Serialize (StreamWriter& writer) const {
634
739
if (this ->ignore_reorder_ ) {
635
740
this ->use_reorder_ = false ;
636
741
}
637
- this ->serialize_basic_info (writer);
742
+
743
+ // basic info moved to metadata since version 0.15
744
+ // only for test
745
+ if (Options::Instance ().new_version ()) {
746
+ this ->serialize_label_info (writer);
747
+ } else {
748
+ this ->serialize_basic_info_v0_14 (writer);
749
+ }
750
+
638
751
this ->basic_flatten_codes_ ->Serialize (writer);
639
752
this ->bottom_graph_ ->Serialize (writer);
640
753
if (this ->use_reorder_ ) {
@@ -646,11 +759,32 @@ HGraph::Serialize(StreamWriter& writer) const {
646
759
if (this ->extra_info_size_ > 0 && this ->extra_infos_ != nullptr ) {
647
760
this ->extra_infos_ ->Serialize (writer);
648
761
}
762
+
763
+ // serialize footer (introduce since v0.15)
764
+ if (Options::Instance ().new_version ()) {
765
+ auto metadata = std::make_shared<Metadata>();
766
+ auto jsonify_basic_info = this ->serialize_basic_info ();
767
+ metadata->Set (" basic_info" , jsonify_basic_info);
768
+ logger::debug (jsonify_basic_info.dump ());
769
+ auto footer = std::make_shared<Footer>(metadata);
770
+ footer->Write (writer);
771
+ }
649
772
}
650
773
651
774
void
652
775
HGraph::Deserialize (StreamReader& reader) {
653
- this ->deserialize_basic_info (reader);
776
+ // try to deserialize footer (only in new version)
777
+ auto footer = Footer::Parse (reader);
778
+ if (footer != nullptr ) {
779
+ logger::debug (" parse with new version format" );
780
+ auto metadata = footer->GetMetadata ();
781
+ this ->deserialize_basic_info (metadata->Get (" basic_info" ));
782
+ this ->deserialize_label_info (reader);
783
+ } else {
784
+ logger::debug (" parse with v0.14 version format" );
785
+ this ->deserialize_basic_info_v0_14 (reader);
786
+ }
787
+
654
788
this ->basic_flatten_codes_ ->Deserialize (reader);
655
789
this ->bottom_graph_ ->Deserialize (reader);
656
790
if (this ->use_reorder_ ) {
@@ -676,35 +810,6 @@ HGraph::Deserialize(StreamReader& reader) {
676
810
}
677
811
}
678
812
679
- void
680
- HGraph::deserialize_basic_info (StreamReader& reader) {
681
- StreamReader::ReadObj (reader, this ->use_reorder_ );
682
- StreamReader::ReadObj (reader, this ->dim_ );
683
- StreamReader::ReadObj (reader, this ->metric_ );
684
- uint64_t max_level;
685
- StreamReader::ReadObj (reader, max_level);
686
- for (uint64_t i = 0 ; i < max_level; ++i) {
687
- this ->route_graphs_ .emplace_back (this ->generate_one_route_graph ());
688
- }
689
- StreamReader::ReadObj (reader, this ->entry_point_id_ );
690
- StreamReader::ReadObj (reader, this ->ef_construct_ );
691
- StreamReader::ReadObj (reader, this ->mult_ );
692
- InnerIdType capacity;
693
- StreamReader::ReadObj (reader, capacity);
694
- this ->max_capacity_ .store (capacity);
695
- StreamReader::ReadVector (reader, this ->label_table_ ->label_table_ );
696
-
697
- uint64_t size;
698
- StreamReader::ReadObj (reader, size);
699
- for (uint64_t i = 0 ; i < size; ++i) {
700
- LabelType key;
701
- StreamReader::ReadObj (reader, key);
702
- InnerIdType value;
703
- StreamReader::ReadObj (reader, value);
704
- this ->label_table_ ->label_remap_ .emplace (key, value);
705
- }
706
- }
707
-
708
813
float
709
814
HGraph::CalcDistanceById (const float * query, int64_t id) const {
710
815
auto flat = this ->basic_flatten_codes_ ;
0 commit comments