Skip to content

Commit c504528

Browse files
sopel39sopel39
committed
Add Parquet decryption support for Hive tables
1 parent 1a4bdbf commit c504528

File tree

86 files changed

+5451
-162
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

86 files changed

+5451
-162
lines changed

Diff for: ‎lib/trino-parquet/pom.xml

+25-1
Original file line numberDiff line numberDiff line change
@@ -13,12 +13,25 @@
1313
<description>Trino - Parquet file format support</description>
1414

1515
<dependencies>
16+
<dependency>
17+
<groupId>com.fasterxml.jackson.core</groupId>
18+
<artifactId>jackson-core</artifactId>
19+
<type>jar</type>
20+
<scope>compile</scope>
21+
</dependency>
22+
23+
<dependency>
24+
<groupId>com.fasterxml.jackson.core</groupId>
25+
<artifactId>jackson-databind</artifactId>
26+
<type>jar</type>
27+
<scope>compile</scope>
28+
</dependency>
29+
1630
<dependency>
1731
<groupId>com.google.errorprone</groupId>
1832
<artifactId>error_prone_annotations</artifactId>
1933
<optional>true</optional>
2034
</dependency>
21-
2235
<dependency>
2336
<groupId>com.google.guava</groupId>
2437
<artifactId>guava</artifactId>
@@ -29,6 +42,11 @@
2942
<artifactId>aircompressor-v3</artifactId>
3043
</dependency>
3144

45+
<dependency>
46+
<groupId>io.airlift</groupId>
47+
<artifactId>json</artifactId>
48+
</dependency>
49+
3250
<dependency>
3351
<groupId>io.airlift</groupId>
3452
<artifactId>log</artifactId>
@@ -95,6 +113,12 @@
95113
</exclusions>
96114
</dependency>
97115

116+
<dependency>
117+
<groupId>io.trino</groupId>
118+
<artifactId>trino-filesystem</artifactId>
119+
<scope>provided</scope>
120+
</dependency>
121+
98122
<dependency>
99123
<groupId>io.trino</groupId>
100124
<artifactId>trino-spi</artifactId>

Diff for: ‎lib/trino-parquet/src/main/java/io/trino/parquet/DataPage.java

+8-1
Original file line numberDiff line numberDiff line change
@@ -21,12 +21,14 @@ public abstract sealed class DataPage
2121
{
2222
protected final int valueCount;
2323
private final OptionalLong firstRowIndex;
24+
private final int pageIndex;
2425

25-
public DataPage(int uncompressedSize, int valueCount, OptionalLong firstRowIndex)
26+
public DataPage(int uncompressedSize, int valueCount, OptionalLong firstRowIndex, int pageIndex)
2627
{
2728
super(uncompressedSize);
2829
this.valueCount = valueCount;
2930
this.firstRowIndex = firstRowIndex;
31+
this.pageIndex = pageIndex;
3032
}
3133

3234
/**
@@ -41,4 +43,9 @@ public int getValueCount()
4143
{
4244
return valueCount;
4345
}
46+
47+
public int getPageIndex()
48+
{
49+
return pageIndex;
50+
}
4451
}

Diff for: ‎lib/trino-parquet/src/main/java/io/trino/parquet/DataPageV1.java

+4-2
Original file line numberDiff line numberDiff line change
@@ -35,15 +35,17 @@ public DataPageV1(
3535
OptionalLong firstRowIndex,
3636
ParquetEncoding repetitionLevelEncoding,
3737
ParquetEncoding definitionLevelEncoding,
38-
ParquetEncoding valuesEncoding)
38+
ParquetEncoding valuesEncoding,
39+
int pageIndex)
3940
{
40-
super(uncompressedSize, valueCount, firstRowIndex);
41+
super(uncompressedSize, valueCount, firstRowIndex, pageIndex);
4142
this.slice = requireNonNull(slice, "slice is null");
4243
this.repetitionLevelEncoding = repetitionLevelEncoding;
4344
this.definitionLevelEncoding = definitionLevelEncoding;
4445
this.valuesEncoding = valuesEncoding;
4546
}
4647

48+
@Override
4749
public Slice getSlice()
4850
{
4951
return slice;

Diff for: ‎lib/trino-parquet/src/main/java/io/trino/parquet/DataPageV2.java

+4-2
Original file line numberDiff line numberDiff line change
@@ -44,9 +44,10 @@ public DataPageV2(
4444
int uncompressedSize,
4545
OptionalLong firstRowIndex,
4646
Statistics<?> statistics,
47-
boolean isCompressed)
47+
boolean isCompressed,
48+
int pageIndex)
4849
{
49-
super(uncompressedSize, valueCount, firstRowIndex);
50+
super(uncompressedSize, valueCount, firstRowIndex, pageIndex);
5051
this.rowCount = rowCount;
5152
this.nullCount = nullCount;
5253
this.repetitionLevels = requireNonNull(repetitionLevels, "repetitionLevels slice is null");
@@ -82,6 +83,7 @@ public ParquetEncoding getDataEncoding()
8283
return dataEncoding;
8384
}
8485

86+
@Override
8587
public Slice getSlice()
8688
{
8789
return slice;

Diff for: ‎lib/trino-parquet/src/main/java/io/trino/parquet/DictionaryPage.java

+1
Original file line numberDiff line numberDiff line change
@@ -43,6 +43,7 @@ public DictionaryPage(Slice slice, int uncompressedSize, int dictionarySize, Par
4343
encoding);
4444
}
4545

46+
@Override
4647
public Slice getSlice()
4748
{
4849
return slice;

Diff for: ‎lib/trino-parquet/src/main/java/io/trino/parquet/Page.java

+4
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,8 @@
1313
*/
1414
package io.trino.parquet;
1515

16+
import io.airlift.slice.Slice;
17+
1618
public abstract class Page
1719
{
1820
protected final int uncompressedSize;
@@ -26,4 +28,6 @@ public int getUncompressedSize()
2628
{
2729
return uncompressedSize;
2830
}
31+
32+
public abstract Slice getSlice();
2933
}

Diff for: ‎lib/trino-parquet/src/main/java/io/trino/parquet/ParquetReaderEncryptionOptions.java

+221
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,221 @@
1+
/*
2+
* Licensed under the Apache License, Version 2.0 (the "License");
3+
* you may not use this file except in compliance with the License.
4+
* You may obtain a copy of the License at
5+
*
6+
* http://www.apache.org/licenses/LICENSE-2.0
7+
*
8+
* Unless required by applicable law or agreed to in writing, software
9+
* distributed under the License is distributed on an "AS IS" BASIS,
10+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11+
* See the License for the specific language governing permissions and
12+
* limitations under the License.
13+
*/
14+
package io.trino.parquet;
15+
16+
import io.trino.parquet.crypto.keytools.TrinoKeyToolkit;
17+
import io.trino.parquet.crypto.keytools.TrinoKmsClient;
18+
19+
public class ParquetReaderEncryptionOptions
20+
{
21+
final String cryptoFactoryClass;
22+
final String encryptionKmsClientClass;
23+
final String encryptionKmsInstanceId;
24+
final String encryptionKmsInstanceUrl;
25+
final String encryptionKeyAccessToken;
26+
final long encryptionCacheLifetimeSeconds;
27+
final boolean uniformEncryption;
28+
boolean encryptionParameterChecked;
29+
final String failsafeEncryptionKeyId;
30+
final String columnKeys;
31+
final String footerKeyId;
32+
final String[] versionedKeyList;
33+
final String keyFile;
34+
final String[] keyList;
35+
final boolean isEncryptionEnvironmentKeys;
36+
37+
public ParquetReaderEncryptionOptions()
38+
{
39+
this.cryptoFactoryClass = null;
40+
this.encryptionKmsClientClass = null;
41+
this.encryptionKmsInstanceId = null;
42+
this.encryptionKmsInstanceUrl = null;
43+
this.encryptionKeyAccessToken = TrinoKmsClient.KEY_ACCESS_TOKEN_DEFAULT;
44+
this.encryptionCacheLifetimeSeconds = TrinoKeyToolkit.CACHE_LIFETIME_DEFAULT_SECONDS;
45+
this.uniformEncryption = false;
46+
this.encryptionParameterChecked = false;
47+
this.failsafeEncryptionKeyId = null;
48+
this.footerKeyId = null;
49+
this.columnKeys = null;
50+
this.versionedKeyList = null;
51+
this.keyFile = null;
52+
this.keyList = null;
53+
this.isEncryptionEnvironmentKeys = false;
54+
}
55+
56+
public ParquetReaderEncryptionOptions(String cryptoFactoryClass,
57+
String encryptionKmsClientClass,
58+
String encryptionKmsInstanceId,
59+
String encryptionKmsInstanceUrl,
60+
String encryptionKeyAccessToken,
61+
long encryptionCacheLifetimeSeconds,
62+
boolean uniformEncryption,
63+
boolean encryptionParameterChecked,
64+
String failsafeEncryptionKeyId,
65+
String footerKeyId,
66+
String columnKeys,
67+
String[] versionedKeyList,
68+
String keyFile,
69+
String[] keyList,
70+
boolean isEncryptionEnvironmentKeys)
71+
{
72+
this.cryptoFactoryClass = cryptoFactoryClass;
73+
this.encryptionKmsClientClass = encryptionKmsClientClass;
74+
this.encryptionKmsInstanceId = encryptionKmsInstanceId;
75+
this.encryptionKmsInstanceUrl = encryptionKmsInstanceUrl;
76+
this.encryptionKeyAccessToken = encryptionKeyAccessToken;
77+
this.encryptionCacheLifetimeSeconds = encryptionCacheLifetimeSeconds;
78+
this.uniformEncryption = uniformEncryption;
79+
this.encryptionParameterChecked = encryptionParameterChecked;
80+
this.failsafeEncryptionKeyId = failsafeEncryptionKeyId;
81+
this.footerKeyId = footerKeyId;
82+
this.columnKeys = columnKeys;
83+
this.versionedKeyList = versionedKeyList;
84+
this.keyFile = keyFile;
85+
this.keyList = keyList;
86+
this.isEncryptionEnvironmentKeys = isEncryptionEnvironmentKeys;
87+
}
88+
89+
public ParquetReaderEncryptionOptions withEncryptionKmsClientClass(String encryptionKmsClientClass)
90+
{
91+
return new ParquetReaderEncryptionOptions(this.cryptoFactoryClass,
92+
encryptionKmsClientClass,
93+
this.encryptionKmsInstanceId,
94+
this.encryptionKmsInstanceUrl,
95+
this.encryptionKeyAccessToken,
96+
this.encryptionCacheLifetimeSeconds,
97+
this.uniformEncryption,
98+
this.encryptionParameterChecked,
99+
this.failsafeEncryptionKeyId,
100+
this.footerKeyId,
101+
this.columnKeys,
102+
this.versionedKeyList,
103+
this.keyFile,
104+
this.keyList,
105+
this.isEncryptionEnvironmentKeys);
106+
}
107+
108+
public ParquetReaderEncryptionOptions withCryptoFactoryClass(String cryptoFactoryClass)
109+
{
110+
return new ParquetReaderEncryptionOptions(cryptoFactoryClass,
111+
this.encryptionKmsClientClass,
112+
this.encryptionKmsInstanceId,
113+
this.encryptionKmsInstanceUrl,
114+
this.encryptionKeyAccessToken,
115+
this.encryptionCacheLifetimeSeconds,
116+
this.uniformEncryption,
117+
this.encryptionParameterChecked,
118+
this.failsafeEncryptionKeyId,
119+
this.footerKeyId,
120+
this.columnKeys,
121+
this.versionedKeyList,
122+
this.keyFile,
123+
this.keyList,
124+
this.isEncryptionEnvironmentKeys);
125+
}
126+
127+
public ParquetReaderEncryptionOptions withEncryptionKmsInstanceId(String encryptionKmsInstanceId)
128+
{
129+
return new ParquetReaderEncryptionOptions(this.cryptoFactoryClass,
130+
this.encryptionKmsClientClass,
131+
encryptionKmsInstanceId,
132+
this.encryptionKmsInstanceUrl,
133+
this.encryptionKeyAccessToken,
134+
this.encryptionCacheLifetimeSeconds,
135+
this.uniformEncryption,
136+
this.encryptionParameterChecked,
137+
this.failsafeEncryptionKeyId,
138+
this.footerKeyId,
139+
this.columnKeys,
140+
this.versionedKeyList,
141+
this.keyFile,
142+
this.keyList,
143+
this.isEncryptionEnvironmentKeys);
144+
}
145+
146+
public ParquetReaderEncryptionOptions withEncryptionKmsInstanceUrl(String encryptionKmsInstanceUrl)
147+
{
148+
return new ParquetReaderEncryptionOptions(this.cryptoFactoryClass,
149+
this.encryptionKmsClientClass,
150+
this.encryptionKmsInstanceId,
151+
encryptionKmsInstanceUrl,
152+
this.encryptionKeyAccessToken,
153+
this.encryptionCacheLifetimeSeconds,
154+
this.uniformEncryption,
155+
this.encryptionParameterChecked,
156+
this.failsafeEncryptionKeyId,
157+
this.footerKeyId,
158+
this.columnKeys,
159+
this.versionedKeyList,
160+
this.keyFile,
161+
this.keyList,
162+
this.isEncryptionEnvironmentKeys);
163+
}
164+
165+
public ParquetReaderEncryptionOptions withEncryptionKeyAccessToken(String encryptionKeyAccessToken)
166+
{
167+
return new ParquetReaderEncryptionOptions(this.cryptoFactoryClass,
168+
this.encryptionKmsClientClass,
169+
this.encryptionKmsInstanceId,
170+
this.encryptionKmsInstanceUrl,
171+
encryptionKeyAccessToken,
172+
this.encryptionCacheLifetimeSeconds,
173+
this.uniformEncryption,
174+
this.encryptionParameterChecked,
175+
this.failsafeEncryptionKeyId,
176+
this.footerKeyId,
177+
this.columnKeys,
178+
this.versionedKeyList,
179+
this.keyFile,
180+
this.keyList,
181+
this.isEncryptionEnvironmentKeys);
182+
}
183+
184+
public ParquetReaderEncryptionOptions withEncryptionCacheLifetimeSeconds(Long encryptionCacheLifetimeSeconds)
185+
{
186+
return new ParquetReaderEncryptionOptions(this.cryptoFactoryClass,
187+
this.encryptionKmsClientClass,
188+
this.encryptionKmsInstanceId,
189+
this.encryptionKmsInstanceUrl,
190+
this.encryptionKeyAccessToken,
191+
encryptionCacheLifetimeSeconds,
192+
this.uniformEncryption,
193+
this.encryptionParameterChecked,
194+
this.failsafeEncryptionKeyId,
195+
this.footerKeyId,
196+
this.columnKeys,
197+
this.versionedKeyList,
198+
this.keyFile,
199+
this.keyList,
200+
this.isEncryptionEnvironmentKeys);
201+
}
202+
203+
public ParquetReaderEncryptionOptions withEncryptionKeyFile(String keyFile)
204+
{
205+
return new ParquetReaderEncryptionOptions(this.cryptoFactoryClass,
206+
this.encryptionKmsClientClass,
207+
this.encryptionKmsInstanceId,
208+
this.encryptionKmsInstanceUrl,
209+
this.encryptionKeyAccessToken,
210+
this.encryptionCacheLifetimeSeconds,
211+
this.uniformEncryption,
212+
this.encryptionParameterChecked,
213+
this.failsafeEncryptionKeyId,
214+
this.footerKeyId,
215+
this.columnKeys,
216+
this.versionedKeyList,
217+
keyFile,
218+
this.keyList,
219+
this.isEncryptionEnvironmentKeys);
220+
}
221+
}

0 commit comments

Comments
 (0)