Skip to content

Commit cd7ec48

Browse files
evanvdiaevanvdia
committed
Add changes to populate the datasource metadata details
1 parent 58df37d commit cd7ec48

File tree

11 files changed

+208
-26
lines changed

11 files changed

+208
-26
lines changed

‎presto-base-jdbc/src/main/java/com/facebook/presto/plugin/jdbc/JdbcInputInfo.java

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,40 @@
1+
/*
2+
* Licensed under the Apache License, Version 2.0 (the "License");
3+
* you may not use this file except in compliance with the License.
4+
* You may obtain a copy of the License at
5+
*
6+
* http://www.apache.org/licenses/LICENSE-2.0
7+
*
8+
* Unless required by applicable law or agreed to in writing, software
9+
* distributed under the License is distributed on an "AS IS" BASIS,
10+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11+
* See the License for the specific language governing permissions and
12+
* limitations under the License.
13+
*/
14+
15+
package com.facebook.presto.plugin.jdbc;
16+
17+
import com.fasterxml.jackson.annotation.JsonProperty;
18+
19+
import static java.util.Objects.requireNonNull;
20+
21+
public class JdbcInputInfo
22+
{
23+
private final String tableLocation;
24+
private final String tableName;
25+
26+
public JdbcInputInfo(
27+
@JsonProperty("tableLocation") String tableLocation,
28+
@JsonProperty("tableName") String tableName)
29+
30+
{
31+
this.tableLocation = requireNonNull(tableLocation, "tableLocation is null");
32+
this.tableName = requireNonNull(tableName, "tableName is null");
33+
}
34+
35+
@JsonProperty
36+
public String getTableLocation()
37+
{
38+
return tableLocation;
39+
}
40+
}

‎presto-base-jdbc/src/main/java/com/facebook/presto/plugin/jdbc/JdbcMetadata.java

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -55,14 +55,16 @@ public class JdbcMetadata
5555
private final JdbcMetadataCache jdbcMetadataCache;
5656
private final JdbcClient jdbcClient;
5757
private final boolean allowDropTable;
58+
private final BaseJdbcConfig baseJdbcConfig;
5859

5960
private final AtomicReference<Runnable> rollbackAction = new AtomicReference<>();
6061

61-
public JdbcMetadata(JdbcMetadataCache jdbcMetadataCache, JdbcClient jdbcClient, boolean allowDropTable)
62+
public JdbcMetadata(JdbcMetadataCache jdbcMetadataCache, JdbcClient jdbcClient, boolean allowDropTable, BaseJdbcConfig baseJdbcConfig)
6263
{
6364
this.jdbcMetadataCache = requireNonNull(jdbcMetadataCache, "jdbcMetadataCache is null");
6465
this.jdbcClient = requireNonNull(jdbcClient, "client is null");
6566
this.allowDropTable = allowDropTable;
67+
this.baseJdbcConfig = requireNonNull(baseJdbcConfig, "baseJdbcConfig is null");
6668
}
6769

6870
@Override
@@ -188,8 +190,9 @@ public Optional<ConnectorOutputMetadata> finishCreateTable(ConnectorSession sess
188190
{
189191
JdbcOutputTableHandle handle = (JdbcOutputTableHandle) tableHandle;
190192
jdbcClient.commitCreateTable(session, JdbcIdentity.from(session), handle);
193+
String url = baseJdbcConfig.getConnectionUrl();
191194
clearRollback();
192-
return Optional.empty();
195+
return Optional.of(new JdbcOutputMetaData(url));
193196
}
194197

195198
private void setRollback(Runnable action)

‎presto-base-jdbc/src/main/java/com/facebook/presto/plugin/jdbc/JdbcMetadataFactory.java

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -22,18 +22,20 @@ public class JdbcMetadataFactory
2222
private final JdbcMetadataCache jdbcMetadataCache;
2323
private final JdbcClient jdbcClient;
2424
private final boolean allowDropTable;
25+
private final BaseJdbcConfig baseJdbcConfig;
2526

2627
@Inject
27-
public JdbcMetadataFactory(JdbcMetadataCache jdbcMetadataCache, JdbcClient jdbcClient, JdbcMetadataConfig config)
28+
public JdbcMetadataFactory(JdbcMetadataCache jdbcMetadataCache, JdbcClient jdbcClient, JdbcMetadataConfig config, BaseJdbcConfig baseJdbcConfig)
2829
{
2930
this.jdbcMetadataCache = requireNonNull(jdbcMetadataCache, "jdbcMetadataCache is null");
3031
this.jdbcClient = requireNonNull(jdbcClient, "jdbcClient is null");
3132
requireNonNull(config, "config is null");
3233
this.allowDropTable = config.isAllowDropTable();
34+
this.baseJdbcConfig = requireNonNull(baseJdbcConfig, "baseJdbcConfig is null");
3335
}
3436

3537
public JdbcMetadata create()
3638
{
37-
return new JdbcMetadata(jdbcMetadataCache, jdbcClient, allowDropTable);
39+
return new JdbcMetadata(jdbcMetadataCache, jdbcClient, allowDropTable, baseJdbcConfig);
3840
}
3941
}

‎presto-base-jdbc/src/main/java/com/facebook/presto/plugin/jdbc/JdbcOutputMetaData.java

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,39 @@
1+
/*
2+
* Licensed under the Apache License, Version 2.0 (the "License");
3+
* you may not use this file except in compliance with the License.
4+
* You may obtain a copy of the License at
5+
*
6+
* http://www.apache.org/licenses/LICENSE-2.0
7+
*
8+
* Unless required by applicable law or agreed to in writing, software
9+
* distributed under the License is distributed on an "AS IS" BASIS,
10+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11+
* See the License for the specific language governing permissions and
12+
* limitations under the License.
13+
*/
14+
15+
package com.facebook.presto.plugin.jdbc;
16+
17+
import com.facebook.presto.spi.connector.ConnectorOutputMetadata;
18+
import com.fasterxml.jackson.annotation.JsonCreator;
19+
import com.fasterxml.jackson.annotation.JsonProperty;
20+
21+
import static java.util.Objects.requireNonNull;
22+
23+
public class JdbcOutputMetaData
24+
implements ConnectorOutputMetadata
25+
{
26+
private final String tableLocation;
27+
28+
@JsonCreator
29+
public JdbcOutputMetaData(@JsonProperty("tableLocation") String tableLocation)
30+
{
31+
this.tableLocation = requireNonNull(tableLocation, "tableLocation is null");
32+
}
33+
34+
@JsonProperty
35+
public String getInfo()
36+
{
37+
return tableLocation;
38+
}
39+
}

‎presto-base-jdbc/src/test/java/com/facebook/presto/plugin/jdbc/TestJdbcMetadata.java

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -66,7 +66,7 @@ public void setUp()
6666
database = new TestingDatabase();
6767
ListeningExecutorService executor = listeningDecorator(newCachedThreadPool(daemonThreadsNamed("test-%s")));
6868
jdbcMetadataCache = new JdbcMetadataCache(executor, database.getJdbcClient(), new JdbcMetadataCacheStats(), OptionalLong.of(0), OptionalLong.of(0), 100);
69-
metadata = new JdbcMetadata(jdbcMetadataCache, database.getJdbcClient(), false);
69+
metadata = new JdbcMetadata(jdbcMetadataCache, database.getJdbcClient(), false, new BaseJdbcConfig());
7070
tableHandle = metadata.getTableHandle(SESSION, new SchemaTableName("example", "numbers"));
7171
}
7272

@@ -261,7 +261,7 @@ public void testDropTableTable()
261261
assertEquals(e.getErrorCode(), PERMISSION_DENIED.toErrorCode());
262262
}
263263

264-
metadata = new JdbcMetadata(jdbcMetadataCache, database.getJdbcClient(), true);
264+
metadata = new JdbcMetadata(jdbcMetadataCache, database.getJdbcClient(), true, new BaseJdbcConfig());
265265
metadata.dropTable(SESSION, tableHandle);
266266

267267
try {

‎presto-hive-common/src/main/java/com/facebook/presto/hive/HiveConnectorOutputMetadata.java

Lines changed: 73 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,73 @@
1+
/*
2+
* Licensed under the Apache License, Version 2.0 (the "License");
3+
* you may not use this file except in compliance with the License.
4+
* You may obtain a copy of the License at
5+
*
6+
* http://www.apache.org/licenses/LICENSE-2.0
7+
*
8+
* Unless required by applicable law or agreed to in writing, software
9+
* distributed under the License is distributed on an "AS IS" BASIS,
10+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11+
* See the License for the specific language governing permissions and
12+
* limitations under the License.
13+
*/
14+
15+
package com.facebook.presto.hive;
16+
17+
import com.fasterxml.jackson.annotation.JsonCreator;
18+
import com.fasterxml.jackson.annotation.JsonProperty;
19+
20+
import java.util.List;
21+
import java.util.Objects;
22+
23+
public class HiveConnectorOutputMetadata
24+
{
25+
private final List<String> partitionNames;
26+
private final String tableLocation;
27+
28+
@JsonCreator
29+
public HiveConnectorOutputMetadata(
30+
@JsonProperty("partitionNames") List<String> partitionNames,
31+
@JsonProperty("tableLocation") String tableLocation)
32+
{
33+
this.partitionNames = partitionNames;
34+
this.tableLocation = tableLocation;
35+
}
36+
37+
@JsonProperty
38+
public List<String> getPartitionNames()
39+
{
40+
return partitionNames;
41+
}
42+
43+
@JsonProperty
44+
public String getTableLocation()
45+
{
46+
return tableLocation;
47+
}
48+
49+
@Override
50+
public boolean equals(Object o)
51+
{
52+
if (o == null || getClass() != o.getClass()) {
53+
return false;
54+
}
55+
HiveConnectorOutputMetadata that = (HiveConnectorOutputMetadata) o;
56+
return Objects.equals(partitionNames, that.partitionNames) && Objects.equals(tableLocation, that.tableLocation);
57+
}
58+
59+
@Override
60+
public int hashCode()
61+
{
62+
return Objects.hash(partitionNames, tableLocation);
63+
}
64+
65+
@Override
66+
public String toString()
67+
{
68+
return "ConnectorOutputInfo{" +
69+
"partitionNames=" + partitionNames +
70+
", tableLocation='" + tableLocation + '\'' +
71+
'}';
72+
}
73+
}

‎presto-hive/src/main/java/com/facebook/presto/hive/HiveInputInfo.java

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -24,14 +24,17 @@ public class HiveInputInfo
2424
// Code that serialize HiveInputInfo into log would often need the ability to limit the length of log entries.
2525
// This boolean field allows such code to mark the log entry as length limited.
2626
private final boolean truncated;
27+
private final String tableLocation;
2728

2829
@JsonCreator
2930
public HiveInputInfo(
3031
@JsonProperty("partitionIds") List<String> partitionIds,
31-
@JsonProperty("truncated") boolean truncated)
32+
@JsonProperty("truncated") boolean truncated,
33+
@JsonProperty("tableLocation") String tableLocation)
3234
{
3335
this.partitionIds = partitionIds;
3436
this.truncated = truncated;
37+
this.tableLocation = tableLocation;
3538
}
3639

3740
@JsonProperty
@@ -45,4 +48,10 @@ public boolean isTruncated()
4548
{
4649
return truncated;
4750
}
51+
52+
@JsonProperty
53+
public String getTableLocation()
54+
{
55+
return tableLocation;
56+
}
4857
}

‎presto-hive/src/main/java/com/facebook/presto/hive/HiveMetadata.java

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -794,7 +794,7 @@ public Optional<Object> getInfo(ConnectorTableLayoutHandle layoutHandle)
794794
tableLayoutHandle.getPartitions().get().stream()
795795
.map(hivePartition -> hivePartition.getPartitionId().getPartitionName())
796796
.collect(toList()),
797-
false));
797+
false, tableLayoutHandle.getTablePath()));
798798
}
799799

800800
return Optional.empty();
@@ -1742,7 +1742,7 @@ public Optional<ConnectorOutputMetadata> finishCreateTable(ConnectorSession sess
17421742
metastore.createTable(session, table, principalPrivileges, Optional.of(writeInfo.getWritePath()), false, tableStatistics, emptyList());
17431743

17441744
if (handle.getPartitionedBy().isEmpty()) {
1745-
return Optional.of(new HiveWrittenPartitions(ImmutableList.of(UNPARTITIONED_ID.getPartitionName())));
1745+
return Optional.of(new HiveWrittenPartitions(new HiveConnectorOutputMetadata(ImmutableList.of(UNPARTITIONED_ID.getPartitionName()), writeInfo.getTargetPath().toString())));
17461746
}
17471747

17481748
if (isRespectTableFormat(session)) {
@@ -1771,10 +1771,10 @@ public Optional<ConnectorOutputMetadata> finishCreateTable(ConnectorSession sess
17711771
partitionStatistics);
17721772
}
17731773

1774-
return Optional.of(new HiveWrittenPartitions(
1774+
return Optional.of(new HiveWrittenPartitions(new HiveConnectorOutputMetadata(
17751775
partitionUpdates.stream()
17761776
.map(PartitionUpdate::getName)
1777-
.collect(toList())));
1777+
.collect(toList()), writeInfo.getTargetPath().getName())));
17781778
}
17791779

17801780
public static boolean shouldCreateFilesForMissingBuckets(Table table, ConnectorSession session)
@@ -2182,11 +2182,11 @@ else if (partitionUpdate.getUpdateMode() == NEW || partitionUpdate.getUpdateMode
21822182
}
21832183
}
21842184

2185-
return Optional.of(new HiveWrittenPartitions(
2185+
return Optional.of(new HiveWrittenPartitions(new HiveConnectorOutputMetadata(
21862186
partitionUpdates.stream()
21872187
.map(PartitionUpdate::getName)
21882188
.map(name -> name.isEmpty() ? UNPARTITIONED_ID.getPartitionName() : name)
2189-
.collect(toList())));
2189+
.collect(toList()), table.getStorage().getLocation())));
21902190
}
21912191

21922192
/**

‎presto-hive/src/main/java/com/facebook/presto/hive/HiveWrittenPartitions.java

Lines changed: 5 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -16,26 +16,23 @@
1616
import com.facebook.presto.spi.connector.ConnectorOutputMetadata;
1717
import com.fasterxml.jackson.annotation.JsonCreator;
1818
import com.fasterxml.jackson.annotation.JsonProperty;
19-
import com.google.common.collect.ImmutableList;
20-
21-
import java.util.List;
2219

2320
import static java.util.Objects.requireNonNull;
2421

2522
public class HiveWrittenPartitions
2623
implements ConnectorOutputMetadata
2724
{
28-
private final List<String> partitionNames;
25+
private final HiveConnectorOutputMetadata connectorOutputInfo;
2926

3027
@JsonCreator
31-
public HiveWrittenPartitions(@JsonProperty("partitionNames") List<String> partitionNames)
28+
public HiveWrittenPartitions(@JsonProperty("connectorOutputInfo") HiveConnectorOutputMetadata connectorOutputInfo)
3229
{
33-
this.partitionNames = ImmutableList.copyOf(requireNonNull(partitionNames, "partitionNames is null"));
30+
this.connectorOutputInfo = requireNonNull(connectorOutputInfo, "connectorOutputInfo is null");
3431
}
3532

3633
@JsonProperty
37-
public List<String> getInfo()
34+
public HiveConnectorOutputMetadata getInfo()
3835
{
39-
return partitionNames;
36+
return connectorOutputInfo;
4037
}
4138
}

‎presto-iceberg/src/main/java/com/facebook/presto/iceberg/IcebergAbstractMetadata.java

Lines changed: 14 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@
2323
import com.facebook.presto.common.type.TimestampWithTimeZoneType;
2424
import com.facebook.presto.common.type.TypeManager;
2525
import com.facebook.presto.common.type.VarcharType;
26+
import com.facebook.presto.hive.HiveConnectorOutputMetadata;
2627
import com.facebook.presto.hive.HivePartition;
2728
import com.facebook.presto.hive.HiveWrittenPartitions;
2829
import com.facebook.presto.hive.NodeVersion;
@@ -570,9 +571,9 @@ private Optional<ConnectorOutputMetadata> finishInsert(ConnectorSession session,
570571
throw new PrestoException(ICEBERG_COMMIT_ERROR, "Failed to commit Iceberg update to table: " + writableTableHandle.getTableName(), e);
571572
}
572573

573-
return Optional.of(new HiveWrittenPartitions(commitTasks.stream()
574+
return Optional.of(new HiveWrittenPartitions(new HiveConnectorOutputMetadata(commitTasks.stream()
574575
.map(CommitTaskData::getPath)
575-
.collect(toImmutableList())));
576+
.collect(toImmutableList()), icebergTable.location())));
576577
}
577578

578579
private Optional<ConnectorOutputMetadata> finishWrite(ConnectorSession session, IcebergWritableTableHandle writableTableHandle, Collection<Slice> fragments, ChangelogOperation operationType)
@@ -617,9 +618,9 @@ private Optional<ConnectorOutputMetadata> finishWrite(ConnectorSession session,
617618
throw new PrestoException(ICEBERG_COMMIT_ERROR, "Failed to commit Iceberg update to table: " + writableTableHandle.getTableName(), e);
618619
}
619620

620-
return Optional.of(new HiveWrittenPartitions(commitTasks.stream()
621+
return Optional.of(new HiveWrittenPartitions(new HiveConnectorOutputMetadata(commitTasks.stream()
621622
.map(CommitTaskData::getPath)
622-
.collect(toImmutableList())));
623+
.collect(toImmutableList()), icebergTable.location())));
623624
}
624625

625626
private void handleInsertTask(CommitTaskData task, Table icebergTable, AppendFiles appendFiles, ImmutableSet.Builder<String> writtenFiles)
@@ -1323,4 +1324,13 @@ protected Optional<String> getDataLocationBasedOnWarehouseDataDir(SchemaTableNam
13231324
{
13241325
return Optional.empty();
13251326
}
1327+
1328+
@Override
1329+
public Optional<Object> getInfo(ConnectorTableLayoutHandle tableHandle)
1330+
{
1331+
IcebergTableLayoutHandle icebergTableHandle = (IcebergTableLayoutHandle) tableHandle;
1332+
return Optional.of(new IcebergInputInfo(
1333+
icebergTableHandle.getTable().getIcebergTableName().getSnapshotId(),
1334+
icebergTableHandle.getTable().getOutputPath().get()));
1335+
}
13261336
}

0 commit comments

Comments
 (0)