new serialisation format

Author: mkeskells

bsi/build.gradle.kts

@@ -13,6 +13,11 @@ tasks.test {
     systemProperty("kryo.unsafe", "false")
     useJUnitPlatform()
     failFast = true
+
+    // Define the memory requirements of tests, to prevent issues in CI while OK locally
+    minHeapSize = "2G"
+    maxHeapSize = "2G"
+
     testLogging {
         // We exclude 'passed' events
         events( "skipped", "failed")

bsi/src/main/java/org/roaringbitmap/bsi/longlong/Roaring64BitmapSliceIndex.java

@@ -254,24 +254,31 @@ public void deserialize(ByteBuffer buffer) throws IOException {
     ebm.deserialize(buffer);
     this.ebM = ebm;
     // read back
-    buffer.position(buffer.position() + ebm.getSizeInBytes());
     int bitDepth = buffer.getInt();
     Roaring64Bitmap[] ba = new Roaring64Bitmap[bitDepth];
     for (int i = 0; i < bitDepth; i++) {
       Roaring64Bitmap rb = new Roaring64Bitmap();
       rb.deserialize(buffer);
       ba[i] = rb;
-      buffer.position(buffer.position() + rb.getSizeInBytes());
     }
     this.bA = ba;
   }
 
   public int serializedSizeInBytes() {
-    int size = 0;
+    long size = 0;
     for (Roaring64Bitmap rb : this.bA) {
-      size += rb.getSizeInBytes();
+      size += rb.serializedSizeInBytes();
+    }
+    size += 8 + //minValue
+            8 + //maxValue
+            1 + //runOptimized
+            4 + //bitDepth
+            this.ebM.serializedSizeInBytes();
+    if (size <= Integer.MAX_VALUE) {
+      return (int) size;
+    } else {
+      throw new IllegalStateException("the serialized size is larger than Integer.MAX_VALUE");
     }
-    return 8 + 8 + 1 + 4 + this.ebM.getSizeInBytes() + size;
   }
 
   /**

roaringbitmap/src/main/java/org/roaringbitmap/art/Art.java

@@ -1,14 +1,18 @@
 package org.roaringbitmap.art;
 
 import org.roaringbitmap.ArraysShim;
+import org.roaringbitmap.longlong.HighLowContainer;
 import org.roaringbitmap.longlong.LongUtils;
 
 import java.io.DataInput;
 import java.io.DataOutput;
 import java.io.IOException;
 import java.nio.ByteBuffer;
+import java.nio.ByteOrder;
 import java.util.Arrays;
 
+import static java.nio.ByteOrder.LITTLE_ENDIAN;
+
 /**
  * See: https://db.in.tum.de/~leis/papers/ART.pdf a cpu cache friendly main memory data structure.
  * At our case, the LeafNode's key is always 48 bit size. The high 48 bit keys here are compared
@@ -365,24 +369,8 @@ public LeafNode last() {
     return getExtremeLeaf(true);
   }
 
-  public void serializeArt(DataOutput dataOutput) throws IOException {
-    dataOutput.writeLong(Long.reverseBytes(keySize));
-    serialize(root, dataOutput);
-  }
-
-  public void deserializeArt(DataInput dataInput) throws IOException {
-    keySize = Long.reverseBytes(dataInput.readLong());
-    root = deserialize(dataInput);
-  }
-
-  public void serializeArt(ByteBuffer byteBuffer) throws IOException {
-    byteBuffer.putLong(keySize);
-    serialize(root, byteBuffer);
-  }
-
-  public void deserializeArt(ByteBuffer byteBuffer) throws IOException {
-    keySize = byteBuffer.getLong();
-    root = deserialize(byteBuffer);
+  public long getKeySize() {
+    return keySize;
   }
 
   public LeafNodeIterator leafNodeIterator(boolean reverse, Containers containers) {
@@ -393,116 +381,51 @@ public LeafNodeIterator leafNodeIteratorFrom(long bound, boolean reverse, Contai
     return new LeafNodeIterator(this, reverse, containers, bound);
   }
 
-  private void serialize(Node node, DataOutput dataOutput) throws IOException {
-    if (node instanceof BranchNode) {
-      BranchNode branchNode = (BranchNode)node;
-      // serialize the internal node itself first
-      branchNode.serialize(dataOutput);
-      // then all the internal node's children
-      int nexPos = branchNode.getNextLargerPos(BranchNode.ILLEGAL_IDX);
-      while (nexPos != BranchNode.ILLEGAL_IDX) {
-        // serialize all the not null child node
-        Node child = branchNode.getChild(nexPos);
-        serialize(child, dataOutput);
-        nexPos = branchNode.getNextLargerPos(nexPos);
-      }
-    } else {
-      // serialize the leaf node
-      node.serialize(dataOutput);
+  public void serializeArt(DataOutput dataOutput, HighLowContainer highLow) throws IOException {
+    dataOutput.writeLong(Long.reverseBytes(keySize));
+    if (keySize != 0L) {
+      root.serialize(dataOutput, highLow);
     }
   }
-
-  private void serialize(Node node, ByteBuffer byteBuffer) throws IOException {
-    if (node instanceof BranchNode) {
-      BranchNode branchNode = (BranchNode)node;
-      // serialize the internal node itself first
-      branchNode.serialize(byteBuffer);
-      // then all the internal node's children
-      int nexPos = branchNode.getNextLargerPos(BranchNode.ILLEGAL_IDX);
-      while (nexPos != BranchNode.ILLEGAL_IDX) {
-        // serialize all the not null child node
-        Node child = branchNode.getChild(nexPos);
-        serialize(child, byteBuffer);
-        nexPos = branchNode.getNextLargerPos(nexPos);
+  public void serializeArt(ByteBuffer buffer, HighLowContainer highLow) throws IOException {
+    ByteOrder originalOrder = buffer.order();
+    buffer.order(LITTLE_ENDIAN);
+    try {
+      buffer.putLong(keySize);
+      if (keySize != 0L) {
+        root.serialize(buffer, highLow);
       }
-    } else {
-      // serialize the leaf node
-      node.serialize(byteBuffer);
+    } finally {
+      buffer.order(originalOrder);
     }
   }
 
-  private Node deserialize(DataInput dataInput) throws IOException {
-    Node oneNode = Node.deserialize(dataInput);
-    if (oneNode == null) {
-      return null;
-    }
-    if (oneNode instanceof LeafNode) {
-      return oneNode;
-    } else {
-      BranchNode branch = (BranchNode) oneNode;
-      // internal node
-      int count = branch.count;
-      // all the not null child nodes
-      Node[] children = new Node[count];
-      for (int i = 0; i < count; i++) {
-        Node child = deserialize(dataInput);
-        children[i] = child;
-      }
-      branch.replaceChildren(children);
-      return branch;
+  public void deserializeArt(DataInput dataInput, HighLowContainer highLow) throws IOException {
+    keySize = Long.reverseBytes(dataInput.readLong());
+    if (keySize != 0L) {
+      root = Node.deserialize(dataInput, highLow);
     }
   }
 
-  private Node deserialize(ByteBuffer byteBuffer) throws IOException {
-    Node oneNode = Node.deserialize(byteBuffer);
-    if (oneNode == null) {
-      return null;
-    }
-    if (oneNode instanceof LeafNode) {
-      return oneNode;
-    } else {
-      BranchNode branchNode = (BranchNode) oneNode;
-      // internal node
-      int count = branchNode.count;
-      // all the not null child nodes
-      Node[] children = new Node[count];
-      for (int i = 0; i < count; i++) {
-        Node child = deserialize(byteBuffer);
-        children[i] = child;
+  public void deserializeArt(ByteBuffer buffer, HighLowContainer highLow) throws IOException {
+    ByteOrder originalOrder = buffer.order();
+    buffer.order(LITTLE_ENDIAN);
+    try {
+      keySize = buffer.getLong();
+      if (keySize != 0L) {
+        root = Node.deserialize(buffer, highLow);
       }
-      branchNode.replaceChildren(children);
-      return branchNode;
+    } finally {
+      buffer.order(originalOrder);
     }
   }
 
-  public long serializeSizeInBytes() {
-    return serializeSizeInBytes(this.root) + 8;
-  }
-
-  public long getKeySize() {
-    return keySize;
-  }
-
-  private long serializeSizeInBytes(Node node) {
-    if (node instanceof BranchNode) {
-      BranchNode branchNode = (BranchNode) node;
-      // serialize the internal node itself first
-      int currentNodeSize = branchNode.serializeSizeInBytes();
-      // then all the internal node's children
-      long childrenTotalSize = 0L;
-      int nexPos = branchNode.getNextLargerPos(BranchNode.ILLEGAL_IDX);
-      while (nexPos != BranchNode.ILLEGAL_IDX) {
-        // serialize all the not null child node
-        Node child = branchNode.getChild(nexPos);
-        long childSize = serializeSizeInBytes(child);
-        nexPos = branchNode.getNextLargerPos(nexPos);
-        childrenTotalSize += childSize;
-      }
-      return currentNodeSize + childrenTotalSize;
-    } else {
-      // serialize the leaf node
-      int nodeSize = node.serializeSizeInBytes();
-      return nodeSize;
+  public long serializeSizeInBytes(HighLowContainer highLow) {
+    long size = 8; // 8 bytes for the keySize
+    if (!isEmpty()) {
+      size += root.serializeSizeInBytes(highLow);
     }
+    return size;
   }
+
 }