Skip to content

Commit 3136296

Browse files
committed
[core] Add compactIndexFiles to ManifestEntryChanges.changedPartitions
1 parent ded6aeb commit 3136296

File tree

3 files changed

+18
-11
lines changed

3 files changed

+18
-11
lines changed

paimon-common/src/main/java/org/apache/paimon/utils/ListUtils.java

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -46,4 +46,11 @@ public static <T> List<T> toList(Iterator<T> iterator) {
4646
}
4747
return result;
4848
}
49+
50+
public static <E> List<E> union(List<? extends E> list1, List<? extends E> list2) {
51+
ArrayList<E> result = new ArrayList<>(list1.size() + list2.size());
52+
result.addAll(list1);
53+
result.addAll(list2);
54+
return result;
55+
}
4956
}

paimon-core/src/main/java/org/apache/paimon/operation/FileStoreCommitImpl.java

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -352,7 +352,8 @@ public int commit(ManifestCommittable committable, boolean checkAppendFiles) {
352352
changedPartitions(
353353
changes.appendTableFiles,
354354
changes.compactTableFiles,
355-
changes.appendIndexFiles)));
355+
changes.appendIndexFiles,
356+
changes.compactIndexFiles)));
356357
if (discardDuplicate) {
357358
Set<FileEntry.Identifier> baseIdentifiers =
358359
baseEntries.stream()
@@ -897,7 +898,7 @@ CommitResult tryCommitOnce(
897898
// latestSnapshotId is different from the snapshot id we've checked for conflicts,
898899
// so we have to check again
899900
List<BinaryRow> changedPartitions =
900-
changedPartitions(deltaFiles, Collections.emptyList(), indexFiles);
901+
changedPartitions(deltaFiles, emptyList(), indexFiles, emptyList());
901902
if (retryResult != null && retryResult.latestSnapshot != null) {
902903
baseDataFiles = new ArrayList<>(retryResult.baseDataFiles);
903904
List<SimpleFileEntry> incremental =

paimon-core/src/main/java/org/apache/paimon/operation/commit/ManifestEntryChanges.java

Lines changed: 8 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@
2525
import org.apache.paimon.manifest.ManifestEntry;
2626
import org.apache.paimon.table.sink.CommitMessage;
2727
import org.apache.paimon.table.sink.CommitMessageImpl;
28+
import org.apache.paimon.utils.ListUtils;
2829

2930
import java.util.ArrayList;
3031
import java.util.HashSet;
@@ -165,17 +166,15 @@ public String toString() {
165166
public static List<BinaryRow> changedPartitions(
166167
List<ManifestEntry> appendTableFiles,
167168
List<ManifestEntry> compactTableFiles,
168-
List<IndexManifestEntry> appendIndexFiles) {
169+
List<IndexManifestEntry> appendIndexFiles,
170+
List<IndexManifestEntry> compactIndexFiles) {
169171
Set<BinaryRow> changedPartitions = new HashSet<>();
170-
for (ManifestEntry appendTableFile : appendTableFiles) {
171-
changedPartitions.add(appendTableFile.partition());
172+
for (ManifestEntry file : ListUtils.union(appendTableFiles, compactTableFiles)) {
173+
changedPartitions.add(file.partition());
172174
}
173-
for (ManifestEntry compactTableFile : compactTableFiles) {
174-
changedPartitions.add(compactTableFile.partition());
175-
}
176-
for (IndexManifestEntry appendIndexFile : appendIndexFiles) {
177-
if (appendIndexFile.indexFile().indexType().equals(DELETION_VECTORS_INDEX)) {
178-
changedPartitions.add(appendIndexFile.partition());
175+
for (IndexManifestEntry file : ListUtils.union(appendIndexFiles, compactIndexFiles)) {
176+
if (file.indexFile().indexType().equals(DELETION_VECTORS_INDEX)) {
177+
changedPartitions.add(file.partition());
179178
}
180179
}
181180
return new ArrayList<>(changedPartitions);

0 commit comments

Comments
 (0)