Skip to content

Commit 7f731d8

Browse files
pramodsatyapramodsatya
committed
Add hive configs for supported read and write formats
1 parent 1fc5687 commit 7f731d8

File tree

9 files changed

+83
-12
lines changed

9 files changed

+83
-12
lines changed

‎presto-hive-hadoop2/src/test/java/com/facebook/presto/hive/s3select/S3SelectTestHelper.java‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -199,7 +199,8 @@ public S3SelectTestHelper(String host,
199199
config.getRecursiveDirWalkerEnabled(),
200200
new ConfigBasedCacheQuotaRequirementProvider(cacheConfig),
201201
new HiveEncryptionInformationProvider(ImmutableSet.of()),
202-
new HivePartitionSkippabilityChecker());
202+
new HivePartitionSkippabilityChecker(),
203+
ImmutableList.of());
203204
pageSourceProvider = new HivePageSourceProvider(
204205
config,
205206
hdfsEnvironment,

‎presto-hive/src/main/java/com/facebook/presto/hive/HiveClientConfig.java‎

Lines changed: 29 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -222,6 +222,8 @@ public class HiveClientConfig
222222
private int parquetQuickStatsMaxConcurrentCalls = 500;
223223
private int quickStatsMaxConcurrentCalls = 100;
224224
private boolean legacyTimestampBucketing;
225+
private List<String> readFormats = ImmutableList.of();
226+
private String writeFormats;
225227

226228
@Min(0)
227229
public int getMaxInitialSplits()
@@ -464,7 +466,7 @@ public List<String> getResourceConfigFiles()
464466
@Config("hive.config.resources")
465467
public HiveClientConfig setResourceConfigFiles(String files)
466468
{
467-
this.resourceConfigFiles = Splitter.on(',').trimResults().omitEmptyStrings().splitToList(files);
469+
this.resourceConfigFiles = SPLITTER.splitToList(files);
468470
return this;
469471
}
470472

@@ -1831,4 +1833,30 @@ public HiveClientConfig setLegacyTimestampBucketing(boolean legacyTimestampBucke
18311833
this.legacyTimestampBucketing = legacyTimestampBucketing;
18321834
return this;
18331835
}
1836+
1837+
@Config("hive.read-formats")
1838+
@ConfigDescription("File formats supported for read operation.")
1839+
public HiveClientConfig setReadFormats(String formats)
1840+
{
1841+
this.readFormats = SPLITTER.splitToList(formats);
1842+
return this;
1843+
}
1844+
1845+
public List<String> getReadFormats()
1846+
{
1847+
return readFormats;
1848+
}
1849+
1850+
@Config("hive.write-formats")
1851+
@ConfigDescription("File formats supported for write operation.")
1852+
public HiveClientConfig setWriteFormats(String formats)
1853+
{
1854+
this.writeFormats = formats;
1855+
return this;
1856+
}
1857+
1858+
public String getWriteFormats()
1859+
{
1860+
return writeFormats;
1861+
}
18341862
}

‎presto-hive/src/main/java/com/facebook/presto/hive/HiveMetadata.java‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -213,6 +213,7 @@
213213
import static com.facebook.presto.hive.HiveSessionProperties.getTemporaryTableSchema;
214214
import static com.facebook.presto.hive.HiveSessionProperties.getTemporaryTableStorageFormat;
215215
import static com.facebook.presto.hive.HiveSessionProperties.getVirtualBucketCount;
216+
import static com.facebook.presto.hive.HiveSessionProperties.getWriteFormats;
216217
import static com.facebook.presto.hive.HiveSessionProperties.isBucketExecutionEnabled;
217218
import static com.facebook.presto.hive.HiveSessionProperties.isCollectColumnStatisticsOnWrite;
218219
import static com.facebook.presto.hive.HiveSessionProperties.isCreateEmptyBucketFiles;
@@ -1913,6 +1914,12 @@ private HiveInsertTableHandle beginInsertInternal(ConnectorSession session, Conn
19131914
SchemaTableName tableName = ((HiveTableHandle) tableHandle).getSchemaTableName();
19141915
Table table = metastore.getTable(metastoreContext, tableName.getSchemaName(), tableName.getTableName())
19151916
.orElseThrow(() -> new TableNotFoundException(tableName));
1917+
HiveStorageFormat tableStorageFormat = extractHiveStorageFormat(table);
1918+
List<String> writeFormats = getWriteFormats(session);
1919+
if (!writeFormats.isEmpty() && !writeFormats.contains(tableStorageFormat)) {
1920+
throw new PrestoException(NOT_SUPPORTED,
1921+
format("File format %s not supported for write operation.", tableStorageFormat));
1922+
}
19161923

19171924
tableWritabilityChecker.checkTableWritable(table);
19181925

@@ -1933,7 +1940,6 @@ private HiveInsertTableHandle beginInsertInternal(ConnectorSession session, Conn
19331940
.filter(columnHandle -> !columnHandle.isHidden())
19341941
.collect(toList());
19351942

1936-
HiveStorageFormat tableStorageFormat = extractHiveStorageFormat(table);
19371943
LocationHandle locationHandle;
19381944
boolean isTemporaryTable = table.getTableType().equals(TEMPORARY_TABLE);
19391945
boolean tempPathRequired = isTempPathRequired(

‎presto-hive/src/main/java/com/facebook/presto/hive/HiveSessionProperties.java‎

Lines changed: 14 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@
1717
import com.facebook.presto.spi.ConnectorSession;
1818
import com.facebook.presto.spi.PrestoException;
1919
import com.facebook.presto.spi.session.PropertyMetadata;
20+
import com.google.common.base.Splitter;
2021
import com.google.common.collect.ImmutableList;
2122
import io.airlift.units.DataSize;
2223
import io.airlift.units.Duration;
@@ -134,6 +135,7 @@ public final class HiveSessionProperties
134135
public static final String DYNAMIC_SPLIT_SIZES_ENABLED = "dynamic_split_sizes_enabled";
135136
public static final String SKIP_EMPTY_FILES = "skip_empty_files";
136137
public static final String LEGACY_TIMESTAMP_BUCKETING = "legacy_timestamp_bucketing";
138+
public static final String WRITE_FORMATS = "write_formats";
137139

138140
public static final String NATIVE_STATS_BASED_FILTER_REORDER_DISABLED = "native_stats_based_filter_reorder_disabled";
139141

@@ -660,7 +662,12 @@ public HiveSessionProperties(HiveClientConfig hiveClientConfig, OrcFileWriterCon
660662
NATIVE_STATS_BASED_FILTER_REORDER_DISABLED,
661663
"Native Execution only. Disable stats based filter reordering.",
662664
false,
663-
true));
665+
true),
666+
stringProperty(
667+
WRITE_FORMATS,
668+
"File formats supported for write operation.",
669+
hiveClientConfig.getWriteFormats(),
670+
false));
664671
}
665672

666673
public List<PropertyMetadata<?>> getSessionProperties()
@@ -1148,4 +1155,10 @@ public static boolean isLegacyTimestampBucketing(ConnectorSession session)
11481155
{
11491156
return session.getProperty(LEGACY_TIMESTAMP_BUCKETING, Boolean.class);
11501157
}
1158+
1159+
public static List<String> getWriteFormats(ConnectorSession session)
1160+
{
1161+
String formats = session.getProperty(WRITE_FORMATS, String.class);
1162+
return (formats == null) ? ImmutableList.of() : Splitter.on(',').trimResults().omitEmptyStrings().splitToList(formats);
1163+
}
11511164
}

‎presto-hive/src/main/java/com/facebook/presto/hive/HiveSplitManager.java‎

Lines changed: 15 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -151,6 +151,7 @@ public class HiveSplitManager
151151
private final CacheQuotaRequirementProvider cacheQuotaRequirementProvider;
152152
private final HiveEncryptionInformationProvider encryptionInformationProvider;
153153
private final PartitionSkippabilityChecker partitionSkippabilityChecker;
154+
private final List<String> readFormats;
154155

155156
@Inject
156157
public HiveSplitManager(
@@ -181,7 +182,8 @@ public HiveSplitManager(
181182
hiveClientConfig.getRecursiveDirWalkerEnabled(),
182183
cacheQuotaRequirementProvider,
183184
encryptionInformationProvider,
184-
partitionSkippabilityChecker);
185+
partitionSkippabilityChecker,
186+
hiveClientConfig.getReadFormats());
185187
}
186188

187189
public HiveSplitManager(
@@ -200,7 +202,8 @@ public HiveSplitManager(
200202
boolean recursiveDfsWalkerEnabled,
201203
CacheQuotaRequirementProvider cacheQuotaRequirementProvider,
202204
HiveEncryptionInformationProvider encryptionInformationProvider,
203-
PartitionSkippabilityChecker partitionSkippabilityChecker)
205+
PartitionSkippabilityChecker partitionSkippabilityChecker,
206+
List<String> readFormats)
204207
{
205208
this.hiveTransactionManager = requireNonNull(hiveTransactionManager, "hiveTransactionManager is null");
206209
this.namenodeStats = requireNonNull(namenodeStats, "namenodeStats is null");
@@ -219,6 +222,7 @@ public HiveSplitManager(
219222
this.cacheQuotaRequirementProvider = requireNonNull(cacheQuotaRequirementProvider, "cacheQuotaRequirementProvider is null");
220223
this.encryptionInformationProvider = requireNonNull(encryptionInformationProvider, "encryptionInformationProvider is null");
221224
this.partitionSkippabilityChecker = requireNonNull(partitionSkippabilityChecker, "partitionSkippabilityChecker is null");
225+
this.readFormats = requireNonNull(readFormats, "readFormats is null");
222226
}
223227

224228
@Override
@@ -250,6 +254,15 @@ public ConnectorSplitSource getSplits(
250254
session.getRuntimeStats());
251255
Table table = layout.getTable(metastore, metastoreContext);
252256

257+
if (!readFormats.isEmpty()) {
258+
StorageFormat storageFormat = table.getStorage().getStorageFormat();
259+
Optional<HiveStorageFormat> hiveStorageFormat = getHiveStorageFormat(storageFormat);
260+
if (hiveStorageFormat.isPresent() && !readFormats.contains(hiveStorageFormat.get())) {
261+
throw new HiveNotReadableException(tableName, Optional.empty(),
262+
format("File format %s not supported for read operation.", hiveStorageFormat.get()));
263+
}
264+
}
265+
253266
if (!isOfflineDataDebugModeEnabled(session)) {
254267
// verify table is not marked as non-readable
255268
String tableNotReadable = table.getParameters().get(OBJECT_NOT_READABLE);

‎presto-hive/src/test/java/com/facebook/presto/hive/AbstractTestHiveClient.java‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1067,7 +1067,8 @@ protected final void setup(String databaseName, HiveClientConfig hiveClientConfi
10671067
false,
10681068
new ConfigBasedCacheQuotaRequirementProvider(cacheConfig),
10691069
encryptionInformationProvider,
1070-
new HivePartitionSkippabilityChecker());
1070+
new HivePartitionSkippabilityChecker(),
1071+
ImmutableList.of());
10711072
pageSinkProvider = new HivePageSinkProvider(
10721073
getDefaultHiveFileWriterFactories(hiveClientConfig, metastoreClientConfig),
10731074
hdfsEnvironment,

‎presto-hive/src/test/java/com/facebook/presto/hive/AbstractTestHiveFileSystem.java‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -247,7 +247,8 @@ protected void setup(String host, int port, String databaseName, BiFunction<Hive
247247
config.getRecursiveDirWalkerEnabled(),
248248
new ConfigBasedCacheQuotaRequirementProvider(cacheConfig),
249249
new HiveEncryptionInformationProvider(ImmutableSet.of()),
250-
new HivePartitionSkippabilityChecker());
250+
new HivePartitionSkippabilityChecker(),
251+
ImmutableList.of());
251252
pageSinkProvider = new HivePageSinkProvider(
252253
getDefaultHiveFileWriterFactories(config, metastoreClientConfig),
253254
hdfsEnvironment,

‎presto-hive/src/test/java/com/facebook/presto/hive/TestHiveClientConfig.java‎

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -166,7 +166,9 @@ public void testDefaults()
166166
.setMaxConcurrentParquetQuickStatsCalls(500)
167167
.setCteVirtualBucketCount(128)
168168
.setSkipEmptyFilesEnabled(false)
169-
.setLegacyTimestampBucketing(false));
169+
.setLegacyTimestampBucketing(false)
170+
.setReadFormats("")
171+
.setWriteFormats(null));
170172
}
171173

172174
@Test
@@ -293,6 +295,8 @@ public void testExplicitPropertyMappings()
293295
.put("hive.cte-virtual-bucket-count", "256")
294296
.put("hive.skip-empty-files", "true")
295297
.put("hive.legacy-timestamp-bucketing", "true")
298+
.put("hive.read-formats", "DWRF,ORC,PARQUET")
299+
.put("hive.write-formats", "DWRF,PARQUET")
296300
.build();
297301

298302
HiveClientConfig expected = new HiveClientConfig()
@@ -414,7 +418,9 @@ public void testExplicitPropertyMappings()
414418
.setMaxConcurrentQuickStatsCalls(101)
415419
.setSkipEmptyFilesEnabled(true)
416420
.setCteVirtualBucketCount(256)
417-
.setLegacyTimestampBucketing(true);
421+
.setLegacyTimestampBucketing(true)
422+
.setReadFormats("DWRF,ORC,PARQUET")
423+
.setWriteFormats("DWRF,PARQUET");
418424

419425
ConfigAssertions.assertFullMapping(properties, expected);
420426
}

‎presto-hive/src/test/java/com/facebook/presto/hive/TestHiveSplitManager.java‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -552,7 +552,8 @@ private void assertRedundantColumnDomains(Range predicateRange, PartitionStatist
552552
false,
553553
new ConfigBasedCacheQuotaRequirementProvider(new CacheConfig()),
554554
new HiveEncryptionInformationProvider(ImmutableList.of()),
555-
new HivePartitionSkippabilityChecker());
555+
new HivePartitionSkippabilityChecker(),
556+
ImmutableList.of());
556557

557558
HiveColumnHandle partitionColumn = new HiveColumnHandle(
558559
"ds",
@@ -700,7 +701,8 @@ public void testEncryptionInformation()
700701
false,
701702
new ConfigBasedCacheQuotaRequirementProvider(new CacheConfig()),
702703
encryptionInformationProvider,
703-
new HivePartitionSkippabilityChecker());
704+
new HivePartitionSkippabilityChecker(),
705+
ImmutableList.of());
704706

705707
HiveColumnHandle partitionColumn = new HiveColumnHandle(
706708
"ds",

0 commit comments

Comments
 (0)