Skip to content

Commit 50b89f2

Browse files
JingsongLiJingsongLi
committed
[core] Fix listFilesIterative bug and avoid using Files.walk in SimpleTableTestBase
1 parent a4effef commit 50b89f2

File tree

3 files changed

+40
-55
lines changed

3 files changed

+40
-55
lines changed

‎paimon-common/src/main/java/org/apache/paimon/fs/FileIO.java‎

Lines changed: 11 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -154,22 +154,18 @@ public FileStatus next() throws IOException {
154154
}
155155

156156
private void maybeUnpackDirectory() throws IOException {
157-
if (!files.isEmpty()) {
158-
return;
159-
}
160-
if (directories.isEmpty()) {
161-
return;
162-
}
163-
FileStatus[] statuses = listStatus(directories.remove());
164-
for (FileStatus f : statuses) {
165-
if (!f.isDir()) {
166-
files.add(f);
167-
continue;
168-
}
169-
if (!recursive) {
170-
continue;
157+
while (files.isEmpty() && !directories.isEmpty()) {
158+
FileStatus[] statuses = listStatus(directories.remove());
159+
for (FileStatus f : statuses) {
160+
if (!f.isDir()) {
161+
files.add(f);
162+
continue;
163+
}
164+
if (!recursive) {
165+
continue;
166+
}
167+
directories.add(f.getPath());
171168
}
172-
directories.add(f.getPath());
173169
}
174170
}
175171

‎paimon-common/src/main/java/org/apache/paimon/fs/local/LocalFileIO.java‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -381,5 +381,10 @@ public Path getPath() {
381381
public long getModificationTime() {
382382
return file.lastModified();
383383
}
384+
385+
@Override
386+
public String toString() {
387+
return "{" + "file=" + file + ", length=" + length + ", scheme='" + scheme + '\'' + '}';
388+
}
384389
}
385390
}

‎paimon-core/src/test/java/org/apache/paimon/table/SimpleTableTestBase.java‎

Lines changed: 24 additions & 40 deletions
Original file line numberDiff line numberDiff line change
@@ -79,9 +79,6 @@
7979
import org.junit.jupiter.params.ParameterizedTest;
8080
import org.junit.jupiter.params.provider.ValueSource;
8181

82-
import java.io.File;
83-
import java.nio.file.Files;
84-
import java.nio.file.Paths;
8582
import java.util.ArrayList;
8683
import java.util.Arrays;
8784
import java.util.Collections;
@@ -793,10 +790,8 @@ public void testRollbackToSnapshotSkipNonExistentSnapshot() throws Exception {
793790
assertThat(result)
794791
.containsExactlyInAnyOrder("0|0|0|binary|varbinary|mapKey:mapVal|multiset");
795792

796-
List<java.nio.file.Path> files =
797-
Files.walk(new File(table.location().toUri().getPath()).toPath())
798-
.collect(Collectors.toList());
799-
assertThat(files.size()).isEqualTo(14);
793+
FileStatus[] files = table.fileIO().listFiles(table.location(), true);
794+
assertThat(files).hasSize(8);
800795
}
801796

802797
// All tags are after the rollback snapshot
@@ -819,25 +814,16 @@ public void testRollbackToSnapshotCase0() throws Exception {
819814
assertThat(result)
820815
.containsExactlyInAnyOrder("0|0|0|binary|varbinary|mapKey:mapVal|multiset");
821816

822-
List<java.nio.file.Path> files =
823-
Files.walk(new File(table.location().toUri().getPath()).toPath())
824-
.collect(Collectors.toList());
825-
assertThat(files.size()).isEqualTo(15);
826-
// table-path
827-
// table-path/snapshot
817+
FileStatus[] files = table.fileIO().listFiles(table.location(), true);
818+
assertThat(files).hasSize(8);
828819
// table-path/snapshot/LATEST
829820
// table-path/snapshot/EARLIEST
830821
// table-path/snapshot/snapshot-1
831-
// table-path/pt=0
832-
// table-path/pt=0/bucket-0
833822
// table-path/pt=0/bucket-0/data-0.orc
834-
// table-path/manifest
835823
// table-path/manifest/manifest-list-1
836824
// table-path/manifest/manifest-0
837825
// table-path/manifest/manifest-list-0
838-
// table-path/schema
839826
// table-path/schema/schema-0
840-
// table-path/tag
841827
}
842828

843829
// One tag is at the rollback snapshot and others are after it
@@ -871,10 +857,8 @@ public void testRollbackToSnapshotCase1() throws Exception {
871857
assertThat(result)
872858
.containsExactlyInAnyOrder("0|0|0|binary|varbinary|mapKey:mapVal|multiset");
873859

874-
List<java.nio.file.Path> files =
875-
Files.walk(new File(table.location().toUri().getPath()).toPath())
876-
.collect(Collectors.toList());
877-
assertThat(files.size()).isEqualTo(16);
860+
FileStatus[] files = table.fileIO().listFiles(table.location(), true);
861+
assertThat(files).hasSize(9);
878862
// case 0 plus 1:
879863
// table-path/tag/tag-test3
880864
}
@@ -912,17 +896,14 @@ public void testRollbackToSnapshotCase2() throws Exception {
912896
assertThat(result)
913897
.containsExactlyInAnyOrder("0|0|0|binary|varbinary|mapKey:mapVal|multiset");
914898

915-
List<java.nio.file.Path> files =
916-
Files.walk(new File(table.location().toUri().getPath()).toPath())
917-
.collect(Collectors.toList());
918-
assertThat(files.size()).isEqualTo(23);
919-
// case 0 plus 7:
899+
FileStatus[] files = table.fileIO().listFiles(table.location(), true);
900+
assertThat(files).hasSize(14);
901+
// case 0 plus 6:
920902
// table-path/manifest/manifest-list-2
921903
// table-path/manifest/manifest-list-3
922904
// table-path/manifest/manifest-1
923905
// table-path/snapshot/snapshot-2
924-
// table-path/pt=1
925-
// table-path/pt=1/bucket-0
906+
// table-path/tag/tag-test3
926907
// table-path/pt=1/bucket-0/data-0.orc
927908
}
928909

@@ -968,10 +949,8 @@ public void testRollbackToTag(boolean expire) throws Exception {
968949
assertThat(result)
969950
.containsExactlyInAnyOrder("0|0|0|binary|varbinary|mapKey:mapVal|multiset");
970951

971-
List<java.nio.file.Path> files =
972-
Files.walk(new File(table.location().toUri().getPath()).toPath())
973-
.collect(Collectors.toList());
974-
assertThat(files.size()).isEqualTo(16);
952+
FileStatus[] files = table.fileIO().listFiles(table.location(), true);
953+
assertThat(files).hasSize(9);
975954
// rollback snapshot case 0 plus 1:
976955
// table-path/tag/tag-test1
977956
}
@@ -1432,13 +1411,18 @@ public void testAsyncExpireExecutionMode() throws Exception {
14321411
store.manifestFileFactory().create());
14331412

14341413
List<Path> unusedFileList =
1435-
Files.walk(Paths.get(tempDir.toString()))
1436-
.filter(Files::isRegularFile)
1437-
.filter(p -> !p.getFileName().toString().startsWith("snapshot"))
1438-
.filter(p -> !p.getFileName().toString().startsWith("schema"))
1439-
.filter(p -> !p.getFileName().toString().equals(LATEST))
1440-
.filter(p -> !p.getFileName().toString().equals(EARLIEST))
1441-
.map(p -> new Path(TraceableFileIO.SCHEME + "://" + p.toString()))
1414+
Arrays.stream(table.fileIO().listFiles(table.location(), true))
1415+
.map(FileStatus::getPath)
1416+
.filter(p -> !p.getName().startsWith("snapshot"))
1417+
.filter(p -> !p.getName().startsWith("schema"))
1418+
.filter(p -> !p.getName().equals(LATEST))
1419+
.filter(p -> !p.getName().equals(EARLIEST))
1420+
.map(
1421+
p ->
1422+
new Path(
1423+
TraceableFileIO.SCHEME
1424+
+ ":"
1425+
+ p.toString().replace("file:", "")))
14421426
.filter(p -> !filesInUse.contains(p))
14431427
.collect(Collectors.toList());
14441428

0 commit comments

Comments
 (0)