Skip to content

Commit 915276a

Browse files
JingsongLiJingsongLi
committed
[core] Introduce specific options for changelog files
1 parent e65a9e1 commit 915276a

18 files changed

+332
-179
lines changed

‎docs/layouts/shortcodes/generated/core_configuration.html‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -68,12 +68,30 @@
6868
<td>MemorySize</td>
6969
<td>Memory page size for caching.</td>
7070
</tr>
71+
<tr>
72+
<td><h5>changelog-file.compression</h5></td>
73+
<td style="word-wrap: break-word;">(none)</td>
74+
<td>String</td>
75+
<td>Changelog file compression.</td>
76+
</tr>
77+
<tr>
78+
<td><h5>changelog-file.format</h5></td>
79+
<td style="word-wrap: break-word;">(none)</td>
80+
<td>String</td>
81+
<td>Specify the message format of changelog files, currently parquet, avro and orc are supported.</td>
82+
</tr>
7183
<tr>
7284
<td><h5>changelog-file.prefix</h5></td>
7385
<td style="word-wrap: break-word;">"changelog-"</td>
7486
<td>String</td>
7587
<td>Specify the file name prefix of changelog files.</td>
7688
</tr>
89+
<tr>
90+
<td><h5>changelog-file.stats-mode</h5></td>
91+
<td style="word-wrap: break-word;">(none)</td>
92+
<td>String</td>
93+
<td>Changelog file metadata stats collection. none, counts, truncate(16), full is available.</td>
94+
</tr>
7795
<tr>
7896
<td><h5>changelog-producer</h5></td>
7997
<td style="word-wrap: break-word;">none</td>

‎paimon-common/src/main/java/org/apache/paimon/CoreOptions.java‎

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -226,6 +226,26 @@ public class CoreOptions implements Serializable {
226226
.defaultValue("changelog-")
227227
.withDescription("Specify the file name prefix of changelog files.");
228228

229+
public static final ConfigOption<String> CHANGELOG_FILE_FORMAT =
230+
key("changelog-file.format")
231+
.stringType()
232+
.noDefaultValue()
233+
.withDescription(
234+
"Specify the message format of changelog files, currently parquet, avro and orc are supported.");
235+
236+
public static final ConfigOption<String> CHANGELOG_FILE_COMPRESSION =
237+
key("changelog-file.compression")
238+
.stringType()
239+
.noDefaultValue()
240+
.withDescription("Changelog file compression.");
241+
242+
public static final ConfigOption<String> CHANGELOG_FILE_STATS_MODE =
243+
key("changelog-file.stats-mode")
244+
.stringType()
245+
.noDefaultValue()
246+
.withDescription(
247+
"Changelog file metadata stats collection. none, counts, truncate(16), full is available.");
248+
229249
public static final ConfigOption<Boolean> FILE_SUFFIX_INCLUDE_COMPRESSION =
230250
key("file.suffix.include.compression")
231251
.booleanType()
@@ -1848,6 +1868,21 @@ public String changelogFilePrefix() {
18481868
return options.get(CHANGELOG_FILE_PREFIX);
18491869
}
18501870

1871+
@Nullable
1872+
public String changelogFileFormat() {
1873+
return options.get(CHANGELOG_FILE_FORMAT);
1874+
}
1875+
1876+
@Nullable
1877+
public String changelogFileCompression() {
1878+
return options.get(CHANGELOG_FILE_COMPRESSION);
1879+
}
1880+
1881+
@Nullable
1882+
public String changelogFileStatsMode() {
1883+
return options.get(CHANGELOG_FILE_STATS_MODE);
1884+
}
1885+
18511886
public boolean fileSuffixIncludeCompression() {
18521887
return options.get(FILE_SUFFIX_INCLUDE_COMPRESSION);
18531888
}

‎paimon-core/src/main/java/org/apache/paimon/io/FileWriterContext.java‎

Lines changed: 48 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,48 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing, software
13+
* distributed under the License is distributed on an "AS IS" BASIS,
14+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15+
* See the License for the specific language governing permissions and
16+
* limitations under the License.
17+
*/
18+
19+
package org.apache.paimon.io;
20+
21+
import org.apache.paimon.format.FormatWriterFactory;
22+
23+
/** Context of file writer. */
24+
public class FileWriterContext {
25+
26+
private final FormatWriterFactory factory;
27+
private final SimpleStatsProducer statsProducer;
28+
private final String compression;
29+
30+
public FileWriterContext(
31+
FormatWriterFactory factory, SimpleStatsProducer statsProducer, String compression) {
32+
this.factory = factory;
33+
this.statsProducer = statsProducer;
34+
this.compression = compression;
35+
}
36+
37+
public FormatWriterFactory factory() {
38+
return factory;
39+
}
40+
41+
public SimpleStatsProducer statsProducer() {
42+
return statsProducer;
43+
}
44+
45+
public String compression() {
46+
return compression;
47+
}
48+
}

‎paimon-core/src/main/java/org/apache/paimon/io/KeyValueDataFileWriter.java‎

Lines changed: 2 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,6 @@
2424
import org.apache.paimon.data.InternalRow;
2525
import org.apache.paimon.data.serializer.InternalRowSerializer;
2626
import org.apache.paimon.fileindex.FileIndexOptions;
27-
import org.apache.paimon.format.FormatWriterFactory;
2827
import org.apache.paimon.format.SimpleColStats;
2928
import org.apache.paimon.fs.FileIO;
3029
import org.apache.paimon.fs.Path;
@@ -78,29 +77,19 @@ public abstract class KeyValueDataFileWriter
7877

7978
public KeyValueDataFileWriter(
8079
FileIO fileIO,
81-
FormatWriterFactory factory,
80+
FileWriterContext context,
8281
Path path,
8382
Function<KeyValue, InternalRow> converter,
8483
RowType keyType,
8584
RowType valueType,
8685
RowType writeRowType,
87-
SimpleStatsProducer statsProducer,
8886
long schemaId,
8987
int level,
90-
String compression,
9188
CoreOptions options,
9289
FileSource fileSource,
9390
FileIndexOptions fileIndexOptions,
9491
boolean isExternalPath) {
95-
super(
96-
fileIO,
97-
factory,
98-
path,
99-
converter,
100-
writeRowType,
101-
statsProducer,
102-
compression,
103-
options.asyncFileWrite());
92+
super(fileIO, context, path, converter, writeRowType, options.asyncFileWrite());
10493

10594
this.keyType = keyType;
10695
this.valueType = valueType;

‎paimon-core/src/main/java/org/apache/paimon/io/KeyValueDataFileWriterImpl.java‎

Lines changed: 2 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,6 @@
2222
import org.apache.paimon.KeyValue;
2323
import org.apache.paimon.data.InternalRow;
2424
import org.apache.paimon.fileindex.FileIndexOptions;
25-
import org.apache.paimon.format.FormatWriterFactory;
2625
import org.apache.paimon.format.SimpleColStats;
2726
import org.apache.paimon.fs.FileIO;
2827
import org.apache.paimon.fs.Path;
@@ -38,31 +37,27 @@ public class KeyValueDataFileWriterImpl extends KeyValueDataFileWriter {
3837

3938
public KeyValueDataFileWriterImpl(
4039
FileIO fileIO,
41-
FormatWriterFactory factory,
40+
FileWriterContext context,
4241
Path path,
4342
Function<KeyValue, InternalRow> converter,
4443
RowType keyType,
4544
RowType valueType,
46-
SimpleStatsProducer statsProducer,
4745
long schemaId,
4846
int level,
49-
String compression,
5047
CoreOptions options,
5148
FileSource fileSource,
5249
FileIndexOptions fileIndexOptions,
5350
boolean isExternalPath) {
5451
super(
5552
fileIO,
56-
factory,
53+
context,
5754
path,
5855
converter,
5956
keyType,
6057
valueType,
6158
KeyValue.schema(keyType, valueType),
62-
statsProducer,
6359
schemaId,
6460
level,
65-
compression,
6661
options,
6762
fileSource,
6863
fileIndexOptions,

0 commit comments

Comments
 (0)