Skip to content

Commit 88be864

Browse files
JingsongLiJingsongLi
committed
[rest] Partition methods should check table first in RESTCatalog
1 parent de7a50d commit 88be864

File tree

4 files changed

+82
-90
lines changed

4 files changed

+82
-90
lines changed

‎paimon-core/src/main/java/org/apache/paimon/catalog/AbstractCatalog.java‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -166,8 +166,7 @@ public void createPartition(Identifier identifier, Map<String, String> partition
166166
FileStoreTable table = (FileStoreTable) getTable(tableIdentifier);
167167

168168
if (table.partitionKeys().isEmpty() || !table.coreOptions().partitionedTableInMetastore()) {
169-
throw new UnsupportedOperationException(
170-
"The table is not partitioned table in metastore.");
169+
return;
171170
}
172171

173172
MetastoreClient.Factory metastoreFactory =

‎paimon-core/src/main/java/org/apache/paimon/catalog/Catalog.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -247,11 +247,11 @@ void createPartition(Identifier identifier, Map<String, String> partitionSpec)
247247
* Drop the partition of the specify table.
248248
*
249249
* @param identifier path of the table to drop partition
250-
* @param partitions the partition to be deleted
250+
* @param partition the partition to be deleted
251251
* @throws TableNotExistException if the table does not exist
252252
* @throws PartitionNotExistException if the partition does not exist
253253
*/
254-
void dropPartition(Identifier identifier, Map<String, String> partitions)
254+
void dropPartition(Identifier identifier, Map<String, String> partition)
255255
throws TableNotExistException, PartitionNotExistException;
256256

257257
/**

‎paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalog.java‎

Lines changed: 71 additions & 79 deletions
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,6 @@
3131
import org.apache.paimon.fs.FileIO;
3232
import org.apache.paimon.fs.Path;
3333
import org.apache.paimon.manifest.PartitionEntry;
34-
import org.apache.paimon.operation.FileStoreCommit;
3534
import org.apache.paimon.operation.Lock;
3635
import org.apache.paimon.options.CatalogOptions;
3736
import org.apache.paimon.options.Options;
@@ -65,7 +64,7 @@
6564
import org.apache.paimon.table.FileStoreTableFactory;
6665
import org.apache.paimon.table.Table;
6766
import org.apache.paimon.table.object.ObjectTable;
68-
import org.apache.paimon.table.sink.BatchWriteBuilder;
67+
import org.apache.paimon.table.sink.BatchTableCommit;
6968
import org.apache.paimon.types.RowType;
7069
import org.apache.paimon.utils.Pair;
7170
import org.apache.paimon.utils.Preconditions;
@@ -85,9 +84,9 @@
8584
import java.util.Optional;
8685
import java.util.Set;
8786
import java.util.concurrent.ScheduledExecutorService;
88-
import java.util.stream.Collectors;
8987

90-
import static org.apache.paimon.CoreOptions.createCommitUser;
88+
import static org.apache.paimon.CoreOptions.METASTORE_PARTITIONED_TABLE;
89+
import static org.apache.paimon.CoreOptions.PARTITION_DEFAULT_NAME;
9190
import static org.apache.paimon.catalog.CatalogUtils.checkNotSystemDatabase;
9291
import static org.apache.paimon.catalog.CatalogUtils.checkNotSystemTable;
9392
import static org.apache.paimon.catalog.CatalogUtils.isSystemDatabase;
@@ -360,6 +359,12 @@ public void dropTable(Identifier identifier, boolean ignoreIfNotExists)
360359
@Override
361360
public void createPartition(Identifier identifier, Map<String, String> partitionSpec)
362361
throws TableNotExistException {
362+
Table table = getTable(identifier);
363+
Options options = Options.fromMap(table.options());
364+
if (!options.get(METASTORE_PARTITIONED_TABLE)) {
365+
return;
366+
}
367+
363368
try {
364369
CreatePartitionRequest request = new CreatePartitionRequest(identifier, partitionSpec);
365370
client.post(
@@ -376,27 +381,77 @@ public void createPartition(Identifier identifier, Map<String, String> partition
376381
}
377382

378383
@Override
379-
public void dropPartition(Identifier identifier, Map<String, String> partitions)
384+
public void dropPartition(Identifier identifier, Map<String, String> partition)
380385
throws TableNotExistException, PartitionNotExistException {
381386
checkNotSystemTable(identifier, "dropPartition");
382-
dropPartitionMetadata(identifier, partitions);
387+
383388
Table table = getTable(identifier);
384-
cleanPartitionsInFileSystem(table, partitions);
389+
Options options = Options.fromMap(table.options());
390+
if (options.get(METASTORE_PARTITIONED_TABLE)) {
391+
try {
392+
client.delete(
393+
resourcePaths.partitions(
394+
identifier.getDatabaseName(), identifier.getTableName()),
395+
new DropPartitionRequest(partition),
396+
headers());
397+
} catch (NoSuchResourceException ignore) {
398+
throw new PartitionNotExistException(identifier, partition);
399+
} catch (ForbiddenException e) {
400+
throw new TableNoPermissionException(identifier, e);
401+
}
402+
}
403+
404+
try (BatchTableCommit commit =
405+
table.newBatchWriteBuilder().withOverwrite(partition).newCommit()) {
406+
commit.commit(Collections.emptyList());
407+
} catch (Exception e) {
408+
throw new RuntimeException(e);
409+
}
385410
}
386411

387412
@Override
388413
public List<PartitionEntry> listPartitions(Identifier identifier)
389414
throws TableNotExistException {
390-
FileStoreTable table = (FileStoreTable) getTable(identifier);
391-
boolean whetherSupportListPartitions =
392-
Boolean.parseBoolean(
393-
table.options().get(CoreOptions.METASTORE_PARTITIONED_TABLE.key()));
394-
if (whetherSupportListPartitions) {
395-
RowType rowType = table.schema().logicalPartitionType();
396-
return listPartitionsFromServer(identifier, rowType);
397-
} else {
398-
return getTable(identifier).newReadBuilder().newScan().listPartitionEntries();
415+
Table table = getTable(identifier);
416+
Options options = Options.fromMap(table.options());
417+
if (!options.get(METASTORE_PARTITIONED_TABLE)) {
418+
return table.newReadBuilder().newScan().listPartitionEntries();
399419
}
420+
421+
ListPartitionsResponse response;
422+
try {
423+
response =
424+
client.get(
425+
resourcePaths.partitions(
426+
identifier.getDatabaseName(), identifier.getTableName()),
427+
ListPartitionsResponse.class,
428+
headers());
429+
} catch (NoSuchResourceException e) {
430+
throw new TableNotExistException(identifier);
431+
} catch (ForbiddenException e) {
432+
throw new TableNoPermissionException(identifier, e);
433+
}
434+
435+
if (response == null || response.getPartitions() == null) {
436+
return Collections.emptyList();
437+
}
438+
439+
RowType partitionType = table.rowType().project(table.partitionKeys());
440+
InternalRowSerializer serializer = new InternalRowSerializer(partitionType);
441+
String defaultName = options.get(PARTITION_DEFAULT_NAME);
442+
List<PartitionEntry> result = new ArrayList<>();
443+
for (PartitionResponse partition : response.getPartitions()) {
444+
GenericRow row =
445+
convertSpecToInternalRow(partition.getSpec(), partitionType, defaultName);
446+
result.add(
447+
new PartitionEntry(
448+
serializer.toBinaryRow(row).copy(),
449+
partition.getRecordCount(),
450+
partition.getFileSizeInBytes(),
451+
partition.getFileCount(),
452+
partition.getLastFileCreationTime()));
453+
}
454+
return result;
400455
}
401456

402457
@Override
@@ -444,41 +499,6 @@ private Table getDataOrFormatTable(Identifier identifier) throws TableNotExistEx
444499
return table;
445500
}
446501

447-
private List<PartitionEntry> listPartitionsFromServer(Identifier identifier, RowType rowType)
448-
throws TableNotExistException {
449-
try {
450-
ListPartitionsResponse response =
451-
client.get(
452-
resourcePaths.partitions(
453-
identifier.getDatabaseName(), identifier.getTableName()),
454-
ListPartitionsResponse.class,
455-
headers());
456-
if (response != null && response.getPartitions() != null) {
457-
return response.getPartitions().stream()
458-
.map(p -> convertToPartitionEntry(p, rowType))
459-
.collect(Collectors.toList());
460-
} else {
461-
return Collections.emptyList();
462-
}
463-
} catch (NoSuchResourceException e) {
464-
throw new TableNotExistException(identifier);
465-
} catch (ForbiddenException e) {
466-
throw new TableNoPermissionException(identifier, e);
467-
}
468-
}
469-
470-
private void cleanPartitionsInFileSystem(Table table, Map<String, String> partitions) {
471-
FileStoreTable fileStoreTable = (FileStoreTable) table;
472-
try (FileStoreCommit commit =
473-
fileStoreTable
474-
.store()
475-
.newCommit(
476-
createCommitUser(fileStoreTable.coreOptions().toConfiguration()))) {
477-
commit.dropPartitions(
478-
Collections.singletonList(partitions), BatchWriteBuilder.COMMIT_IDENTIFIER);
479-
}
480-
}
481-
482502
private GetTableResponse getTableResponse(Identifier identifier) throws TableNotExistException {
483503
try {
484504
return client.get(
@@ -492,23 +512,6 @@ private GetTableResponse getTableResponse(Identifier identifier) throws TableNot
492512
}
493513
}
494514

495-
private boolean dropPartitionMetadata(Identifier identifier, Map<String, String> partitions)
496-
throws TableNoPermissionException, PartitionNotExistException {
497-
try {
498-
DropPartitionRequest request = new DropPartitionRequest(partitions);
499-
client.delete(
500-
resourcePaths.partitions(
501-
identifier.getDatabaseName(), identifier.getTableName()),
502-
request,
503-
headers());
504-
return true;
505-
} catch (NoSuchResourceException ignore) {
506-
throw new PartitionNotExistException(identifier, partitions);
507-
} catch (ForbiddenException e) {
508-
throw new TableNoPermissionException(identifier, e);
509-
}
510-
}
511-
512515
private static Map<String, String> configHeaders(Map<String, String> properties) {
513516
return RESTUtil.extractPrefixMap(properties, "header.");
514517
}
@@ -540,17 +543,6 @@ private ScheduledExecutorService tokenRefreshExecutor() {
540543
return refreshExecutor;
541544
}
542545

543-
private PartitionEntry convertToPartitionEntry(PartitionResponse partition, RowType rowType) {
544-
InternalRowSerializer serializer = new InternalRowSerializer(rowType);
545-
GenericRow row = convertSpecToInternalRow(partition.getSpec(), rowType, null);
546-
return new PartitionEntry(
547-
serializer.toBinaryRow(row).copy(),
548-
partition.getRecordCount(),
549-
partition.getFileSizeInBytes(),
550-
partition.getFileCount(),
551-
partition.getLastFileCreationTime());
552-
}
553-
554546
private static FileIO getFileIOFromOptions(CatalogContext context) {
555547
try {
556548
Options options = context.options();

‎paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogTest.java‎

Lines changed: 8 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -34,7 +34,6 @@
3434
import org.apache.paimon.rest.responses.ListDatabasesResponse;
3535
import org.apache.paimon.rest.responses.ListPartitionsResponse;
3636
import org.apache.paimon.rest.responses.ListTablesResponse;
37-
import org.apache.paimon.rest.responses.PartitionResponse;
3837
import org.apache.paimon.schema.SchemaChange;
3938
import org.apache.paimon.table.Table;
4039

@@ -332,10 +331,12 @@ public void testDropTableWhenTableNotExistAndIgnoreIfNotExistsIsFalse() throws E
332331
@Test
333332
public void testCreatePartition() throws Exception {
334333
String databaseName = MockRESTMessage.databaseName();
334+
GetTableResponse response = MockRESTMessage.getTableResponse();
335+
mockResponse(mapper.writeValueAsString(response), 200);
336+
335337
Map<String, String> partitionSpec = new HashMap<>();
336338
partitionSpec.put("p1", "v1");
337-
PartitionResponse response = MockRESTMessage.partitionResponse();
338-
mockResponse(mapper.writeValueAsString(response), 200);
339+
mockResponse(mapper.writeValueAsString(MockRESTMessage.partitionResponse()), 200);
339340
assertDoesNotThrow(
340341
() ->
341342
restCatalog.createPartition(
@@ -386,11 +387,12 @@ public void testDropPartition() throws Exception {
386387
@Test
387388
public void testDropPartitionWhenPartitionNoExist() throws Exception {
388389
String databaseName = MockRESTMessage.databaseName();
390+
GetTableResponse response = MockRESTMessage.getTableResponseEnablePartition();
391+
mockResponse(mapper.writeValueAsString(response), 200);
392+
389393
Map<String, String> partitionSpec = new HashMap<>();
390-
GetTableResponse response = MockRESTMessage.getTableResponse();
391394
partitionSpec.put(response.getSchema().primaryKeys().get(0), "1");
392395
mockResponse(mapper.writeValueAsString(""), 404);
393-
mockResponse(mapper.writeValueAsString(response), 200);
394396
assertThrows(
395397
Catalog.PartitionNotExistException.class,
396398
() ->
@@ -418,7 +420,6 @@ public void testDropPartitionWhenTableNoExist() throws Exception {
418420
Map<String, String> partitionSpec = new HashMap<>();
419421
GetTableResponse response = MockRESTMessage.getTableResponse();
420422
partitionSpec.put(response.getSchema().primaryKeys().get(0), "1");
421-
mockResponse(mapper.writeValueAsString(""), 200);
422423
mockResponse("", 404);
423424
assertThrows(
424425
Catalog.TableNotExistException.class,
@@ -442,7 +443,7 @@ public void testListPartitionsWhenMetastorePartitionedIsTrue() throws Exception
442443
@Test
443444
public void testListPartitionsFromFile() throws Exception {
444445
String databaseName = MockRESTMessage.databaseName();
445-
GetTableResponse response = MockRESTMessage.getTableResponse();
446+
GetTableResponse response = MockRESTMessage.getTableResponseEnablePartition();
446447
mockResponse(mapper.writeValueAsString(response), 200);
447448
mockResponse(mapper.writeValueAsString(response), 200);
448449
List<PartitionEntry> partitionEntries =

0 commit comments

Comments
 (0)