[core] Rename 'deletion-vectors.version' to 'deletion-vectors.bitmap64' (apache#5552)

Author: JingsongLi

docs/layouts/shortcodes/generated/core_configuration.html

@@ -297,16 +297,16 @@
             <td>The target size of deletion vector index file.</td>
         </tr>
         <tr>
-            <td><h5>deletion-vectors.enabled</h5></td>
+            <td><h5>deletion-vectors.bitmap64</h5></td>
             <td style="word-wrap: break-word;">false</td>
             <td>Boolean</td>
-            <td>Whether to enable deletion vectors mode. In this mode, index files containing deletion vectors are generated when data is written, which marks the data for deletion. During read operations, by applying these index files, merging can be avoided.</td>
+            <td>Enable 64 bit bitmap implementation. Note that only 64 bit bitmap implementation is compatible with Iceberg.</td>
         </tr>
         <tr>
-            <td><h5>deletion-vectors.version</h5></td>
-            <td style="word-wrap: break-word;">1</td>
-            <td>Integer</td>
-            <td>The version of deletion vector, currently support v1 and v2, default version is 1.</td>
+            <td><h5>deletion-vectors.enabled</h5></td>
+            <td style="word-wrap: break-word;">false</td>
+            <td>Boolean</td>
+            <td>Whether to enable deletion vectors mode. In this mode, index files containing deletion vectors are generated when data is written, which marks the data for deletion. During read operations, by applying these index files, merging can be avoided.</td>
         </tr>
         <tr>
             <td><h5>dynamic-bucket.assigner-parallelism</h5></td>

docs/static/rest-catalog-open-api.yaml

@@ -232,6 +232,11 @@ paths:
           in: query
           schema:
             type: string
+        - name: tableNamePattern
+          description: A sql LIKE pattern (% and _) for table names.
+          in: query
+          schema:
+            type: string
       responses:
         "200":
           description: OK
@@ -305,6 +310,11 @@ paths:
           in: query
           schema:
             type: string
+        - name: tableNamePattern
+          description: A sql LIKE pattern (% and _) for table names.
+          in: query
+          schema:
+            type: string
       responses:
         "200":
           description: OK
@@ -658,6 +668,11 @@ paths:
           in: query
           schema:
             type: string
+        - name: partitionNamePattern
+          description: A sql LIKE pattern (% and _) for partition names.
+          in: query
+          schema:
+            type: string
       responses:
         "200":
           description: OK
@@ -896,6 +911,11 @@ paths:
           in: query
           schema:
             type: string
+        - name: viewNamePattern
+          description: A sql LIKE pattern (% and _) for view names.
+          in: query
+          schema:
+            type: string
       responses:
         "200":
           description: OK
@@ -969,6 +989,11 @@ paths:
           in: query
           schema:
             type: string
+        - name: viewNamePattern
+          description: A sql LIKE pattern (% and _) for view names.
+          in: query
+          schema:
+            type: string
       responses:
         "200":
           description: OK

paimon-common/src/main/java/org/apache/paimon/CoreOptions.java

@@ -1510,12 +1510,12 @@ public class CoreOptions implements Serializable {
                     .defaultValue(MemorySize.ofMebiBytes(2))
                     .withDescription("The target size of deletion vector index file.");
 
-    public static final ConfigOption<Integer> DELETION_VECTOR_VERSION =
-            key("deletion-vectors.version")
-                    .intType()
-                    .defaultValue(1)
+    public static final ConfigOption<Boolean> DELETION_VECTOR_BITMAP64 =
+            key("deletion-vectors.bitmap64")
+                    .booleanType()
+                    .defaultValue(false)
                     .withDescription(
-                            "The version of deletion vector, currently support v1 and v2, default version is 1.");
+                            "Enable 64 bit bitmap implementation. Note that only 64 bit bitmap implementation is compatible with Iceberg.");
 
     public static final ConfigOption<Boolean> DELETION_FORCE_PRODUCE_CHANGELOG =
             key("delete.force-produce-changelog")
@@ -2629,8 +2629,8 @@ public MemorySize deletionVectorIndexFileTargetSize() {
         return options.get(DELETION_VECTOR_INDEX_FILE_TARGET_SIZE);
     }
 
-    public int deletionVectorVersion() {
-        return options.get(DELETION_VECTOR_VERSION);
+    public boolean deletionVectorBitmap64() {
+        return options.get(DELETION_VECTOR_BITMAP64);
     }
 
     public FileIndexOptions indexColumnsOptions() {

paimon-core/src/main/java/org/apache/paimon/AbstractFileStore.java

@@ -243,7 +243,7 @@ public IndexFileHandler newIndexFileHandler() {
                         bucketMode() == BucketMode.BUCKET_UNAWARE
                                 ? options.deletionVectorIndexFileTargetSize()
                                 : MemorySize.ofBytes(Long.MAX_VALUE),
-                        options.deletionVectorVersion()));
+                        options.deletionVectorBitmap64()));
     }
 
     @Override

paimon-core/src/main/java/org/apache/paimon/deletionvectors/Bitmap64DeletionVector.java

@@ -22,6 +22,8 @@
 import org.apache.paimon.utils.Preconditions;
 import org.apache.paimon.utils.RoaringBitmap32;
 
+import java.io.DataOutputStream;
+import java.io.IOException;
 import java.nio.ByteBuffer;
 import java.nio.ByteOrder;
 import java.util.Objects;
@@ -35,8 +37,6 @@
  */
 public class Bitmap64DeletionVector implements DeletionVector {
 
-    public static final int VERSION = 2;
-
     public static final int MAGIC_NUMBER = 1681511377;
     public static final int LENGTH_SIZE_BYTES = 4;
     public static final int CRC_SIZE_BYTES = 4;
@@ -90,12 +90,7 @@ public long getCardinality() {
     }
 
     @Override
-    public int version() {
-        return VERSION;
-    }
-
-    @Override
-    public byte[] serializeToBytes() {
+    public int serializeTo(DataOutputStream out) throws IOException {
         roaringBitmap.runLengthEncode(); // run-length encode the bitmap before serializing
         int bitmapDataLength = computeBitmapDataLength(roaringBitmap); // magic bytes + bitmap
         byte[] bytes = new byte[LENGTH_SIZE_BYTES + bitmapDataLength + CRC_SIZE_BYTES];
@@ -106,18 +101,8 @@ public byte[] serializeToBytes() {
         int crc = computeChecksum(bytes, bitmapDataLength);
         buffer.putInt(crcOffset, crc);
         buffer.rewind();
-        return bytes;
-    }
-
-    public static DeletionVector deserializeFromBytes(byte[] bytes) {
-        ByteBuffer buffer = ByteBuffer.wrap(bytes);
-        int bitmapDataLength = readBitmapDataLength(buffer, bytes.length);
-        OptimizedRoaringBitmap64 bitmap = deserializeBitmap(bytes, bitmapDataLength);
-        int crc = computeChecksum(bytes, bitmapDataLength);
-        int crcOffset = LENGTH_SIZE_BYTES + bitmapDataLength;
-        int expectedCrc = buffer.getInt(crcOffset);
-        Preconditions.checkArgument(crc == expectedCrc, "Invalid CRC");
-        return new Bitmap64DeletionVector(bitmap);
+        out.write(bytes);
+        return bytes.length;
     }
 
     public static DeletionVector deserializeFromBitmapDataBytes(byte[] bytes) {
@@ -182,7 +167,6 @@ private static int computeChecksum(byte[] bytes, int bitmapDataLength) {
 
     protected static int toLittleEndianInt(int bigEndianInt) {
         byte[] bytes = ByteBuffer.allocate(4).putInt(bigEndianInt).array();
-
         return ByteBuffer.wrap(bytes).order(ByteOrder.LITTLE_ENDIAN).getInt();
     }
 

paimon-core/src/main/java/org/apache/paimon/deletionvectors/BitmapDeletionVector.java

@@ -25,15 +25,14 @@
 import java.io.IOException;
 import java.nio.ByteBuffer;
 import java.util.Objects;
+import java.util.zip.CRC32;
 
 /**
  * A {@link DeletionVector} based on {@link RoaringBitmap32}, it only supports files with row count
  * not exceeding {@link RoaringBitmap32#MAX_VALUE}.
  */
 public class BitmapDeletionVector implements DeletionVector {
 
-    public static final int VERSION = 1;
-
     public static final int MAGIC_NUMBER = 1581511376;
     public static final int MAGIC_NUMBER_SIZE_BYTES = 4;
 
@@ -85,36 +84,22 @@ public long getCardinality() {
     }
 
     @Override
-    public int version() {
-        return VERSION;
-    }
-
-    @Override
-    public byte[] serializeToBytes() {
+    public int serializeTo(DataOutputStream out) {
         try (ByteArrayOutputStream bos = new ByteArrayOutputStream();
                 DataOutputStream dos = new DataOutputStream(bos)) {
             dos.writeInt(MAGIC_NUMBER);
             roaringBitmap.serialize(dos);
-            return bos.toByteArray();
+            byte[] data = bos.toByteArray();
+            int size = data.length;
+            out.writeInt(size);
+            out.write(data);
+            out.writeInt(calculateChecksum(data));
+            return size;
         } catch (Exception e) {
             throw new RuntimeException("Unable to serialize deletion vector", e);
         }
     }
 
-    public static DeletionVector deserializeFromBytes(byte[] bytes) {
-        try {
-            ByteBuffer buffer = ByteBuffer.wrap(bytes);
-            int magicNum = buffer.getInt();
-            if (magicNum == MAGIC_NUMBER) {
-                return deserializeFromByteBuffer(buffer);
-            } else {
-                throw new RuntimeException("Invalid magic number: " + magicNum);
-            }
-        } catch (IOException e) {
-            throw new RuntimeException("Unable to deserialize deletion vector", e);
-        }
-    }
-
     /**
      * Note: the result is read only, do not call any modify operation outside.
      *
@@ -153,4 +138,10 @@ public boolean equals(Object o) {
     public int hashCode() {
         return Objects.hashCode(roaringBitmap);
     }
+
+    public static int calculateChecksum(byte[] bytes) {
+        CRC32 crc = new CRC32();
+        crc.update(bytes);
+        return (int) crc.getValue();
+    }
 }