Merge pull request #125 from mpetri/doc-ret-fixes

Fixes and additions to the document retrieval benchmark suite

Author: simongog

benchmark/document_retrieval/index.config

@@ -6,5 +6,7 @@
 #   * TYPE      : Corresponding type.
 #   * LATEX_NAME: LaTeX name for output in the benchmark report.
 GREEDY;doc_list_index_greedy<>;GREEDY
+#GREEDY-RRR;doc_list_index_greedy<csa_wt<wt_huff<rrr_vector<63>>,1000000,1000000>,wt_int<rrr_vector<63>>>;GREEDY-RRR
 #QPROBING;doc_list_index_qprobing<>;QPROBING
 SADA;doc_list_index_sada<csa_sada<enc_vector<>, 32, 1000000, text_order_sa_sampling<sd_vector<>>>>;SADA
+SORT;doc_list_index_sort<>;SORT

benchmark/document_retrieval/index_int.config

@@ -7,3 +7,4 @@
 #   * LATEX_NAME: LaTeX name for output in the benchmark report.
 GREEDYINT;doc_list_index_greedy<csa_wt<wt_int<rrr_vector<63>>,1000000,1000000>>;GREEDY-I
 SADAINT;doc_list_index_sada<csa_sada_int<enc_vector<>, 32, 1000000, text_order_sa_sampling<sd_vector<>>>>;SADA-I
+SORTINT;doc_list_index_sort<csa_sada_int<enc_vector<>, 32, 1000000, text_order_sa_sampling<sd_vector<>>>>;SORT-I

benchmark/document_retrieval/src/doc_list_index.hpp

@@ -17,5 +17,6 @@ construct(t_index& idx, const std::string& file, sdsl::cache_config& config, uin
 #include "doc_list_index_sada.hpp"
 #include "doc_list_index_greedy.hpp"
 #include "doc_list_index_qprobing.hpp"
+#include "doc_list_index_sort.hpp"
 
 #endif

benchmark/document_retrieval/src/doc_list_index_greedy.hpp

@@ -49,7 +49,7 @@ class doc_list_index_greedy
                     return m_ep-m_sp+1;
                 }
 
-// Constructors for an empty result and for a result in the interval [sp, ep]:
+                // Constructors for an empty result and for a result in the interval [sp, ep]:
                 result(size_type sp, size_type ep,list_type&& l) : list_type(l), m_sp(1), m_ep(0) {}
                 result() : m_sp(1), m_ep(0) {}
                 result(size_type sp, size_type ep) : m_sp(sp), m_ep(ep) {}
@@ -66,15 +66,15 @@ class doc_list_index_greedy
 
     protected:
         size_type m_doc_cnt; // number of documents in the collection
-        csa_type  m_csa;     // CSA built from the collection text
+        csa_type  m_csa_full;     // CSA built from the collection text
         wtd_type  m_wtd;     // wtd build from the collection text
     public:
 
-//! Default constructor
+        //! Default constructor
         doc_list_index_greedy() { }
 
         doc_list_index_greedy(std::string file_name, sdsl::cache_config& cconfig, uint8_t num_bytes) {
-            construct(m_csa, file_name, cconfig, num_bytes);
+            construct(m_csa_full, file_name, cconfig, num_bytes);
 
             const char* KEY_TEXT = key_text_trait<WIDTH>::KEY_TEXT;
             std::string text_file = cache_file_name(KEY_TEXT, cconfig);
@@ -108,31 +108,31 @@ class doc_list_index_greedy
             structure_tree_node* child = structure_tree::add_child(v, name, util::class_name(*this));
             size_type written_bytes = 0;
             written_bytes += write_member(m_doc_cnt, out, child, "doc_cnt");
-            written_bytes += m_csa.serialize(out, child, "csa");
+            written_bytes += m_csa_full.serialize(out, child, "csa_full");
             written_bytes += m_wtd.serialize(out, child, "wtd");
             structure_tree::add_size(child, written_bytes);
             return written_bytes;
         }
 
         void load(std::istream& in) {
             read_member(m_doc_cnt, in);
-            m_csa.load(in);
+            m_csa_full.load(in);
             m_wtd.load(in);
         }
 
         void swap(doc_list_index_greedy& dr) {
             if (this != &dr) {
                 std::swap(m_doc_cnt, dr.m_doc_cnt);
-                m_csa.swap(dr.m_csa);
+                m_csa_full.swap(dr.m_csa_full);
                 m_wtd.swap(dr.m_wtd);
             }
         }
 
-//! Search for the k documents which contain the search term most frequent
+        //! Search for the k documents which contain the search term most frequent
         template<class t_pat_iter>
         size_type search(t_pat_iter begin, t_pat_iter end, result& res, size_t k) const {
             size_type sp=1, ep=0;
-            if (0 == backward_search(m_csa, 0, m_csa.size()-1, begin, end, sp, ep)) {
+            if (0 == backward_search(m_csa_full, 0, m_csa_full.size()-1, begin, end, sp, ep)) {
                 res = result();
                 return 0;
             } else {
@@ -143,7 +143,7 @@ class doc_list_index_greedy
         }
 
     private:
-//! Construct the doc_border bitvector by streaming the text file
+        //! Construct the doc_border bitvector by streaming the text file
         void
         construct_doc_border(const std::string& text_file, bit_vector& doc_border) {
             int_vector_buffer<WIDTH> text_buf(text_file);

benchmark/document_retrieval/src/doc_list_index_sada.hpp

@@ -99,7 +99,7 @@ class doc_list_index_sada
 
     private:
         size_type              m_doc_cnt;           // number of documents in the collection
-        csa_full_type          m_full_csa;          // CSA build from the collection text
+        csa_full_type          m_csa_full;          // CSA build from the collection text
         vector<int_vector<>>   m_doc_isa;           // array of inverse SAs. m_doc_isa[i] contains the ISA of document i
         range_min_type         m_rminq;             // range minimum data structure build over an array Cprev
         range_max_type         m_rmaxq;             // range maximum data structure build over an array Cnext
@@ -116,7 +116,7 @@ class doc_list_index_sada
         doc_list_index_sada() { }
 
         doc_list_index_sada(std::string file_name, sdsl::cache_config& cconfig, uint8_t num_bytes) {
-            construct(m_full_csa, file_name, cconfig, num_bytes);
+            construct(m_csa_full, file_name, cconfig, num_bytes);
 
             const char* KEY_TEXT = key_text_trait<WIDTH>::KEY_TEXT;
             std::string text_file = cache_file_name(KEY_TEXT, cconfig);
@@ -154,14 +154,14 @@ class doc_list_index_sada
         }
 
         size_type word_cnt()const {
-            return m_full_csa.size()-doc_cnt();
+            return m_csa_full.size()-doc_cnt();
         }
 
         size_type serialize(std::ostream& out, structure_tree_node* v=NULL, std::string name="")const {
             structure_tree_node* child = structure_tree::add_child(v, name, util::class_name(*this));
             size_type written_bytes = 0;
             written_bytes += write_member(m_doc_cnt, out, child, "doc_cnt");
-            written_bytes += m_full_csa.serialize(out, child, "full_csa");
+            written_bytes += m_csa_full.serialize(out, child, "csa_full");
             written_bytes += serialize_vector(m_doc_isa, out, child, "doc_isa");
             written_bytes += m_rminq.serialize(out, child, "rminq");
             written_bytes += m_rmaxq.serialize(out, child, "rmaxq");
@@ -176,7 +176,7 @@ class doc_list_index_sada
 
         void load(std::istream& in) {
             read_member(m_doc_cnt, in);
-            m_full_csa.load(in);
+            m_csa_full.load(in);
             m_doc_isa.resize(m_doc_cnt);
             load_vector(m_doc_isa, in);
             m_rminq.load(in);
@@ -195,7 +195,7 @@ class doc_list_index_sada
         void swap(doc_list_index_sada& dr) {
             if (this != &dr) {
                 std::swap(m_doc_cnt, dr.m_doc_cnt);
-                m_full_csa.swap(dr.m_full_csa);
+                m_csa_full.swap(dr.m_csa_full);
                 m_doc_isa.swap(dr.m_doc_isa);
                 m_rminq.swap(dr.m_rminq);
                 m_rmaxq.swap(dr.m_rmaxq);
@@ -218,7 +218,7 @@ class doc_list_index_sada
                result& res,
                size_t k) const {
             size_type sp=1, ep=0;
-            if (0 == backward_search(m_full_csa, 0, m_full_csa.size()-1, begin, end, sp, ep)) {
+            if (0 == backward_search(m_csa_full, 0, m_csa_full.size()-1, begin, end, sp, ep)) {
                 res = result();
                 return 0;
             } else {
@@ -249,46 +249,64 @@ class doc_list_index_sada
                 m_doc_rmax_marked[doc]        = 0;  //                                 get_lex_largest_suffixes
 
                 if (suffix_1 == suffix_2) {  // if pattern occurs exactly once
-                    res.push_back({doc,1});  // add the #occurrence
+                    res.push_back( {doc,1}); // add the #occurrence
                 } else {
                     size_type doc_begin = doc ? m_doc_border_select(doc) + 1 : 0;
                     size_type doc_sp    = m_doc_isa[doc][ suffix_1 - doc_begin ];
                     size_type doc_ep    = m_doc_isa[doc][ suffix_2 - doc_begin ];
                     if (doc_sp > doc_ep) {
                         std::swap(doc_sp, doc_ep);
                     }
-                    res.push_back({doc, doc_ep - doc_sp + 1});
+                    res.push_back( {doc, doc_ep - doc_sp + 1});
                 }
             }
         }
 
-        void get_lex_smallest_suffixes(size_type sp, size_type ep, vector<size_type>& suffixes)const {
-            if (sp > ep)
-                return;
-            size_type min_idx = m_rminq(sp, ep);
-            size_type suffix  = m_full_csa[min_idx];
-            size_type doc     = m_doc_border_rank(suffix+1);
-
-            if (!m_doc_rmin_marked[doc]) {
-                suffixes.push_back(suffix);
-                m_doc_rmin_marked[doc] = 1;
-                get_lex_smallest_suffixes(sp, min_idx - 1, suffixes); // min_idx != 0, since `\0` is appended to string
-                get_lex_smallest_suffixes(min_idx+1, ep, suffixes);
+        void get_lex_smallest_suffixes(size_type sp, size_type ep, vector<size_type>& suffixes) const {
+            using lex_range_t = std::pair<size_type,size_type>;
+            std::stack<lex_range_t> stack;
+            stack.emplace(sp,ep);
+            while (!stack.empty()) {
+                auto range = stack.top();
+                stack.pop();
+                size_type rsp = std::get<0>(range);
+                size_type rep = std::get<1>(range);
+                if (rsp <= rep) {
+                    size_type min_idx = m_rminq(rsp,rep);
+                    size_type suffix  = m_csa_full[min_idx];
+                    size_type doc     = m_doc_border_rank(suffix+1);
+
+                    if (!m_doc_rmin_marked[doc]) {
+                        suffixes.push_back(suffix);
+                        m_doc_rmin_marked[doc] = 1;
+                        stack.emplace(min_idx+1,rep);
+                        stack.emplace(rsp,min_idx-1); // min_idx != 0, since `\0` is appended to string
+                    }
+                }
             }
         }
 
-        void get_lex_largest_suffixes(size_type sp, size_type ep, vector<size_type>& suffixes)const {
-            if (sp > ep)
-                return;
-            size_type max_idx = m_rmaxq(sp, ep);
-            size_type suffix  = m_full_csa[max_idx];
-            size_type doc     = m_doc_border_rank(suffix+1);
-
-            if (!m_doc_rmax_marked[doc]) {
-                suffixes.push_back(suffix);
-                m_doc_rmax_marked[doc] = 1;
-                get_lex_largest_suffixes(max_idx+1, ep, suffixes);
-                get_lex_largest_suffixes(sp, max_idx - 1, suffixes); // max_idx != 0, since `\0` is appended to string
+        void get_lex_largest_suffixes(size_type sp, size_type ep, vector<size_type>& suffixes) const {
+            using lex_range_t = std::pair<size_type,size_type>;
+            std::stack<lex_range_t> stack;
+            stack.emplace(sp,ep);
+            while (!stack.empty()) {
+                auto range = stack.top();
+                stack.pop();
+                size_type rsp = std::get<0>(range);
+                size_type rep = std::get<1>(range);
+                if (rsp <= rep) {
+                    size_type max_idx = m_rmaxq(rsp,rep);
+                    size_type suffix  = m_csa_full[max_idx];
+                    size_type doc     = m_doc_border_rank(suffix+1);
+
+                    if (!m_doc_rmax_marked[doc]) {
+                        suffixes.push_back(suffix);
+                        m_doc_rmax_marked[doc] = 1;
+                        stack.emplace(rsp,max_idx - 1); // max_idx != 0, since `\0` is appended to string
+                        stack.emplace(max_idx+1,rep);
+                    }
+                }
             }
         }