Skip to content

Commit 2c8aded

Browse files
JingsongLiJingsongLi
committed
[core] Introduce 'deletion-vectors.modifiable' to limit dv alter
1 parent 7461e3b commit 2c8aded

File tree

5 files changed

+82
-4
lines changed

5 files changed

+82
-4
lines changed

‎paimon-api/src/main/java/org/apache/paimon/CoreOptions.java‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1709,6 +1709,12 @@ public InlineElement getDescription() {
17091709
+ " vectors are generated when data is written, which marks the data for deletion."
17101710
+ " During read operations, by applying these index files, merging can be avoided.");
17111711

1712+
public static final ConfigOption<Boolean> DELETION_VECTORS_MODIFIABLE =
1713+
key("deletion-vectors.modifiable")
1714+
.booleanType()
1715+
.defaultValue(false)
1716+
.withDescription("Whether to enable modifying deletion vectors mode.");
1717+
17121718
public static final ConfigOption<MemorySize> DELETION_VECTOR_INDEX_FILE_TARGET_SIZE =
17131719
key("deletion-vector.index-file.target-size")
17141720
.memoryType()

‎paimon-core/src/main/java/org/apache/paimon/schema/SchemaManager.java‎

Lines changed: 30 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -78,6 +78,8 @@
7878

7979
import static org.apache.paimon.CoreOptions.AGG_FUNCTION;
8080
import static org.apache.paimon.CoreOptions.BUCKET_KEY;
81+
import static org.apache.paimon.CoreOptions.DELETION_VECTORS_ENABLED;
82+
import static org.apache.paimon.CoreOptions.DELETION_VECTORS_MODIFIABLE;
8183
import static org.apache.paimon.CoreOptions.DISTINCT;
8284
import static org.apache.paimon.CoreOptions.FIELDS_PREFIX;
8385
import static org.apache.paimon.CoreOptions.IGNORE_RETRACT;
@@ -297,7 +299,10 @@ public static TableSchema generateTableSchema(
297299
SetOption setOption = (SetOption) change;
298300
if (hasSnapshots.get()) {
299301
checkAlterTableOption(
300-
setOption.key(), oldOptions.get(setOption.key()), setOption.value());
302+
oldOptions,
303+
setOption.key(),
304+
oldOptions.get(setOption.key()),
305+
setOption.value());
301306
}
302307
newOptions.put(setOption.key(), setOption.value());
303308
} else if (change instanceof RemoveOption) {
@@ -1072,7 +1077,7 @@ public void deleteSchema(long schemaId) {
10721077
}
10731078

10741079
public static void checkAlterTableOption(
1075-
String key, @Nullable String oldValue, String newValue) {
1080+
Map<String, String> options, String key, @Nullable String oldValue, String newValue) {
10761081
if (CoreOptions.IMMUTABLE_OPTIONS.contains(key)) {
10771082
throw new UnsupportedOperationException(
10781083
String.format("Change '%s' is not supported yet.", key));
@@ -1092,6 +1097,29 @@ public static void checkAlterTableOption(
10921097
throw new UnsupportedOperationException("Cannot change bucket to -1.");
10931098
}
10941099
}
1100+
1101+
if (DELETION_VECTORS_ENABLED.key().equals(key)) {
1102+
boolean dvModifiable =
1103+
Boolean.parseBoolean(
1104+
options.getOrDefault(
1105+
DELETION_VECTORS_MODIFIABLE.key(),
1106+
DELETION_VECTORS_MODIFIABLE.defaultValue().toString()));
1107+
if (!dvModifiable) {
1108+
boolean oldDv =
1109+
oldValue == null
1110+
? DELETION_VECTORS_ENABLED.defaultValue()
1111+
: Boolean.parseBoolean(oldValue);
1112+
boolean newDv = Boolean.parseBoolean(newValue);
1113+
1114+
if (oldDv != newDv) {
1115+
throw new UnsupportedOperationException(
1116+
String.format(
1117+
"Cannot change deletion vectors mode from %s to %s. If modifying table deletion-vectors mode without full-compaction, this may result in data duplication. "
1118+
+ "If you are confident, you can set table option '%s' = 'true' to allow deletion vectors modification.",
1119+
oldDv, newDv, DELETION_VECTORS_MODIFIABLE.key()));
1120+
}
1121+
}
1122+
}
10951123
}
10961124

10971125
public static void checkResetTableOption(String key) {

‎paimon-core/src/main/java/org/apache/paimon/table/AbstractFileStoreTable.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -318,7 +318,7 @@ private void checkImmutability(Map<String, String> dynamicOptions) {
318318
(k, newValue) -> {
319319
String oldValue = oldOptions.get(k);
320320
if (!Objects.equals(oldValue, newValue)) {
321-
SchemaManager.checkAlterTableOption(k, oldValue, newValue);
321+
SchemaManager.checkAlterTableOption(oldOptions, k, oldValue, newValue);
322322
}
323323
});
324324
}

‎paimon-core/src/test/java/org/apache/paimon/schema/SchemaManagerTest.java‎

Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -69,6 +69,8 @@
6969
import java.util.stream.IntStream;
7070
import java.util.stream.Stream;
7171

72+
import static org.apache.paimon.CoreOptions.DELETION_VECTORS_ENABLED;
73+
import static org.apache.paimon.CoreOptions.DELETION_VECTORS_MODIFIABLE;
7274
import static org.apache.paimon.utils.FailingFileIO.retryArtificialException;
7375
import static org.assertj.core.api.Assertions.assertThat;
7476
import static org.assertj.core.api.Assertions.assertThatCode;
@@ -748,4 +750,45 @@ public void testUpdateRowTypeInArrayAndMap() throws Exception {
748750
1, "v", new ArrayType(new MapType(DataTypes.INT(), innerType))));
749751
assertThat(manager.latest().get().logicalRowType()).isEqualTo(outerType);
750752
}
753+
754+
@Test
755+
public void testAlterDeletionVectorsMode() throws Exception {
756+
// create table
757+
Schema schema =
758+
new Schema(
759+
rowType.getFields(),
760+
Collections.emptyList(),
761+
Collections.emptyList(),
762+
options,
763+
"");
764+
Path tableRoot = new Path(tempDir.toString(), "table");
765+
SchemaManager manager = new SchemaManager(LocalFileIO.create(), tableRoot);
766+
manager.createTable(schema);
767+
768+
// write table
769+
FileStoreTable table = FileStoreTableFactory.create(LocalFileIO.create(), tableRoot);
770+
String commitUser = UUID.randomUUID().toString();
771+
TableWriteImpl<?> write =
772+
table.newWrite(commitUser).withIOManager(IOManager.create(tempDir + "/io"));
773+
TableCommitImpl commit = table.newCommit(commitUser);
774+
write.write(GenericRow.of(1, 10L, BinaryString.fromString("apple")));
775+
commit.commit(1, write.prepareCommit(false, 1));
776+
write.close();
777+
commit.close();
778+
779+
// assert exception in alter table
780+
assertThatThrownBy(
781+
() ->
782+
manager.commitChanges(
783+
SchemaChange.setOption(
784+
DELETION_VECTORS_ENABLED.key(), "true")))
785+
.hasMessageContaining(
786+
"If modifying table deletion-vectors mode without full-compaction, this may result in data duplication.");
787+
788+
// assert not exception when set option
789+
manager.commitChanges(SchemaChange.setOption(DELETION_VECTORS_MODIFIABLE.key(), "true"));
790+
manager.commitChanges(SchemaChange.setOption(DELETION_VECTORS_ENABLED.key(), "true"));
791+
table = FileStoreTableFactory.create(LocalFileIO.create(), tableRoot);
792+
assertThat(table.options().get(DELETION_VECTORS_ENABLED.key())).isEqualTo("true");
793+
}
751794
}

‎paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/AbstractFlinkTableFactory.java‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -206,7 +206,8 @@ Table buildPaimonTable(DynamicTableFactory.Context context) {
206206
(key, newValue) -> {
207207
String oldValue = origin.getOptions().get(key);
208208
if (!Objects.equals(oldValue, newValue)) {
209-
SchemaManager.checkAlterTableOption(key, oldValue, newValue);
209+
SchemaManager.checkAlterTableOption(
210+
origin.getOptions(), key, oldValue, newValue);
210211
}
211212
});
212213
Map<String, String> newOptions = new HashMap<>();

0 commit comments

Comments
 (0)