segmentwriter can ignore store

Author: rebasedming

src/indexer/index_writer.rs

@@ -203,7 +203,7 @@ fn index_documents<D: Document>(
     segment_updater: &SegmentUpdater,
     mut delete_cursor: DeleteCursor,
 ) -> crate::Result<()> {
-    let mut segment_writer = SegmentWriter::for_segment(memory_budget, segment.clone())?;
+    let mut segment_writer = SegmentWriter::for_segment(memory_budget, segment.clone(), false)?;
     for document_group in grouped_document_iterator {
         for doc in document_group {
             segment_writer.add_document(doc)?;

src/indexer/segment_writer.rs

@@ -73,6 +73,7 @@ pub struct SegmentWriter {
     per_field_text_analyzers: Vec<TextAnalyzer>,
     term_buffer: Term,
     schema: Schema,
+    ignore_store: bool,
 }
 
 impl SegmentWriter {
@@ -85,7 +86,11 @@ impl SegmentWriter {
     ///   behavior as a memory limit.
     /// - segment: The segment being written
     /// - schema
-    pub fn for_segment(memory_budget_in_bytes: usize, segment: Segment) -> crate::Result<Self> {
+    pub fn for_segment(
+        memory_budget_in_bytes: usize,
+        segment: Segment,
+        ignore_store: bool,
+    ) -> crate::Result<Self> {
         let schema = segment.schema();
         let tokenizer_manager = segment.index().tokenizers().clone();
         let tokenizer_manager_fast_field = segment.index().fast_field_tokenizer().clone();
@@ -130,6 +135,7 @@ impl SegmentWriter {
             per_field_text_analyzers,
             term_buffer: Term::with_capacity(16),
             schema,
+            ignore_store,
         })
     }
 
@@ -156,6 +162,7 @@ impl SegmentWriter {
             &self.fieldnorms_writer,
             self.segment_serializer,
             mapping.as_ref(),
+            self.ignore_store,
         )?;
         let doc_opstamps = remap_doc_opstamps(self.doc_opstamps, mapping.as_ref());
         Ok(doc_opstamps)
@@ -420,6 +427,7 @@ fn remap_and_write(
     fieldnorms_writer: &FieldNormsWriter,
     mut serializer: SegmentSerializer,
     doc_id_map: Option<&DocIdMapping>,
+    ignore_store: bool,
 ) -> crate::Result<()> {
     debug!("remap-and-write");
     if let Some(fieldnorms_serializer) = serializer.extract_fieldnorms_serializer() {
@@ -429,7 +437,6 @@ fn remap_and_write(
         .segment()
         .open_read(SegmentComponent::FieldNorms)?;
     let fieldnorm_readers = FieldNormReaders::open(fieldnorm_data)?;
-    let uses_store = schema.fields().any(|(_, entry)| entry.is_stored());
     serialize_postings(
         ctx,
         schema,
@@ -442,7 +449,7 @@ fn remap_and_write(
     fast_field_writers.serialize(serializer.get_fast_field_write(), doc_id_map)?;
 
     // finalize temp docstore and create version, which reflects the doc_id_map
-    if uses_store {
+    if !ignore_store {
         if let Some(doc_id_map) = doc_id_map {
             debug!("resort-docstore");
             let store_write = serializer

src/indexer/single_segment_index_writer.rs

@@ -17,7 +17,7 @@ pub struct SingleSegmentIndexWriter<D: Document = TantivyDocument> {
 impl<D: Document> SingleSegmentIndexWriter<D> {
     pub fn new(index: Index, mem_budget: usize) -> crate::Result<Self> {
         let segment = index.new_segment();
-        let segment_writer = SegmentWriter::for_segment(mem_budget, segment.clone())?;
+        let segment_writer = SegmentWriter::for_segment(mem_budget, segment.clone(), false)?;
         Ok(Self {
             segment_writer,
             segment,

src/postings/mod.rs

@@ -230,7 +230,7 @@ pub(crate) mod tests {
 
         {
             let mut segment_writer =
-                SegmentWriter::for_segment(15_000_000, segment.clone()).unwrap();
+                SegmentWriter::for_segment(15_000_000, segment.clone(), false).unwrap();
             {
                 // checking that position works if the field has two values
                 let op = AddOperation {