Skip to content

Commit b964189

Browse files
lyne7-scLyneHisoka-X
authored
[improve] update clickhouse connector config option (#8755)
Co-authored-by: Lyne <[email protected]> Co-authored-by: Jia Fan <[email protected]>
1 parent b16aac2 commit b964189

File tree

15 files changed

+391
-351
lines changed

15 files changed

+391
-351
lines changed

Diff for: ‎seatunnel-ci-tools/src/test/java/org/apache/seatunnel/api/ConnectorOptionCheckTest.java

-3
Original file line numberDiff line numberDiff line change
@@ -197,18 +197,15 @@ private Set<String> buildWhiteList() {
197197
whiteList.add("QdrantSinkOptions");
198198
whiteList.add("MilvusSourceOptions");
199199
whiteList.add("RocketMqSinkOptions");
200-
whiteList.add("ClickhouseFileSinkOptions");
201200
whiteList.add("IcebergSinkOptions");
202201
whiteList.add("MaxcomputeSourceOptions");
203202
whiteList.add("InfluxDBSourceOptions");
204203
whiteList.add("InfluxDBSinkOptions");
205204
whiteList.add("KuduSourceOptions");
206205
whiteList.add("SocketSinkOptions");
207-
whiteList.add("ClickhouseSinkOptions");
208206
whiteList.add("SelectDBSinkOptions");
209207
whiteList.add("PrometheusSinkOptions");
210208
whiteList.add("FirestoreSinkOptions");
211-
whiteList.add("ClickhouseSourceOptions");
212209
whiteList.add("MilvusSinkOptions");
213210
whiteList.add("RocketMqSourceOptions");
214211
whiteList.add("TablestoreSinkOptions");

Diff for: ‎seatunnel-connectors-v2/connector-clickhouse/src/main/java/org/apache/seatunnel/connectors/seatunnel/clickhouse/catalog/ClickhouseCatalog.java

+7-7
Original file line numberDiff line numberDiff line change
@@ -34,7 +34,7 @@
3434
import org.apache.seatunnel.api.table.catalog.exception.DatabaseNotExistException;
3535
import org.apache.seatunnel.api.table.catalog.exception.TableAlreadyExistException;
3636
import org.apache.seatunnel.api.table.catalog.exception.TableNotExistException;
37-
import org.apache.seatunnel.connectors.seatunnel.clickhouse.config.ClickhouseConfig;
37+
import org.apache.seatunnel.connectors.seatunnel.clickhouse.config.ClickhouseSinkOptions;
3838
import org.apache.seatunnel.connectors.seatunnel.clickhouse.util.ClickhouseCatalogUtil;
3939
import org.apache.seatunnel.connectors.seatunnel.clickhouse.util.ClickhouseProxy;
4040
import org.apache.seatunnel.connectors.seatunnel.clickhouse.util.ClickhouseUtil;
@@ -54,11 +54,11 @@
5454
import java.util.Properties;
5555
import java.util.concurrent.ExecutionException;
5656

57-
import static org.apache.seatunnel.connectors.seatunnel.clickhouse.config.ClickhouseConfig.CLICKHOUSE_CONFIG;
58-
import static org.apache.seatunnel.connectors.seatunnel.clickhouse.config.ClickhouseConfig.HOST;
59-
import static org.apache.seatunnel.connectors.seatunnel.clickhouse.config.ClickhouseConfig.PASSWORD;
60-
import static org.apache.seatunnel.connectors.seatunnel.clickhouse.config.ClickhouseConfig.SAVE_MODE_CREATE_TEMPLATE;
61-
import static org.apache.seatunnel.connectors.seatunnel.clickhouse.config.ClickhouseConfig.USERNAME;
57+
import static org.apache.seatunnel.connectors.seatunnel.clickhouse.config.ClickhouseBaseOptions.CLICKHOUSE_CONFIG;
58+
import static org.apache.seatunnel.connectors.seatunnel.clickhouse.config.ClickhouseBaseOptions.HOST;
59+
import static org.apache.seatunnel.connectors.seatunnel.clickhouse.config.ClickhouseBaseOptions.PASSWORD;
60+
import static org.apache.seatunnel.connectors.seatunnel.clickhouse.config.ClickhouseBaseOptions.USERNAME;
61+
import static org.apache.seatunnel.connectors.seatunnel.clickhouse.config.ClickhouseSinkOptions.SAVE_MODE_CREATE_TEMPLATE;
6262
import static org.apache.seatunnel.shade.com.google.common.base.Preconditions.checkArgument;
6363

6464
@Slf4j
@@ -254,7 +254,7 @@ public PreviewResult previewAction(
254254
tablePath.getTableName(),
255255
catalogTable.get().getTableSchema(),
256256
catalogTable.get().getComment(),
257-
ClickhouseConfig.SAVE_MODE_CREATE_TEMPLATE.key()));
257+
ClickhouseSinkOptions.SAVE_MODE_CREATE_TEMPLATE.key()));
258258
} else if (actionType == ActionType.DROP_TABLE) {
259259
return new SQLPreviewResult(
260260
ClickhouseCatalogUtil.INSTANCE.getDropTableSql(tablePath, true));

Diff for: ‎seatunnel-connectors-v2/connector-clickhouse/src/main/java/org/apache/seatunnel/connectors/seatunnel/clickhouse/catalog/ClickhouseCatalogFactory.java

+5-5
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,7 @@
2222
import org.apache.seatunnel.api.table.catalog.Catalog;
2323
import org.apache.seatunnel.api.table.factory.CatalogFactory;
2424
import org.apache.seatunnel.api.table.factory.Factory;
25-
import org.apache.seatunnel.connectors.seatunnel.clickhouse.config.ClickhouseConfig;
25+
import org.apache.seatunnel.connectors.seatunnel.clickhouse.config.ClickhouseBaseOptions;
2626

2727
import com.google.auto.service.AutoService;
2828

@@ -44,10 +44,10 @@ public String factoryIdentifier() {
4444
@Override
4545
public OptionRule optionRule() {
4646
return OptionRule.builder()
47-
.required(ClickhouseConfig.HOST)
48-
.required(ClickhouseConfig.DATABASE)
49-
.required(ClickhouseConfig.USERNAME)
50-
.required(ClickhouseConfig.PASSWORD)
47+
.required(ClickhouseBaseOptions.HOST)
48+
.required(ClickhouseBaseOptions.DATABASE)
49+
.required(ClickhouseBaseOptions.USERNAME)
50+
.required(ClickhouseBaseOptions.PASSWORD)
5151
.build();
5252
}
5353
}

Diff for: ‎seatunnel-connectors-v2/connector-clickhouse/src/main/java/org/apache/seatunnel/connectors/seatunnel/clickhouse/config/ClickhouseBaseOptions.java

+71
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,71 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one or more
3+
* contributor license agreements. See the NOTICE file distributed with
4+
* this work for additional information regarding copyright ownership.
5+
* The ASF licenses this file to You under the Apache License, Version 2.0
6+
* (the "License"); you may not use this file except in compliance with
7+
* the License. You may obtain a copy of the License at
8+
*
9+
* http://www.apache.org/licenses/LICENSE-2.0
10+
*
11+
* Unless required by applicable law or agreed to in writing, software
12+
* distributed under the License is distributed on an "AS IS" BASIS,
13+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
* See the License for the specific language governing permissions and
15+
* limitations under the License.
16+
*/
17+
18+
package org.apache.seatunnel.connectors.seatunnel.clickhouse.config;
19+
20+
import org.apache.seatunnel.api.configuration.Option;
21+
import org.apache.seatunnel.api.configuration.Options;
22+
23+
import java.time.ZoneId;
24+
import java.util.Collections;
25+
import java.util.Map;
26+
27+
public class ClickhouseBaseOptions {
28+
29+
/** Clickhouse server host */
30+
public static final Option<String> HOST =
31+
Options.key("host")
32+
.stringType()
33+
.noDefaultValue()
34+
.withDescription("Clickhouse server host");
35+
36+
/** Clickhouse database name */
37+
public static final Option<String> DATABASE =
38+
Options.key("database")
39+
.stringType()
40+
.noDefaultValue()
41+
.withDescription("Clickhouse database name");
42+
43+
/** Clickhouse server username */
44+
public static final Option<String> USERNAME =
45+
Options.key("username")
46+
.stringType()
47+
.noDefaultValue()
48+
.withDescription("Clickhouse server username");
49+
50+
/** Clickhouse server password */
51+
public static final Option<String> PASSWORD =
52+
Options.key("password")
53+
.stringType()
54+
.noDefaultValue()
55+
.withDescription("Clickhouse server password");
56+
57+
/** Clickhouse server timezone */
58+
public static final Option<String> SERVER_TIME_ZONE =
59+
Options.key("server_time_zone")
60+
.stringType()
61+
.defaultValue(ZoneId.systemDefault().getId())
62+
.withDescription(
63+
"The session time zone in database server."
64+
+ "If not set, then ZoneId.systemDefault() is used to determine the server time zone");
65+
66+
public static final Option<Map<String, String>> CLICKHOUSE_CONFIG =
67+
Options.key("clickhouse.config")
68+
.mapType()
69+
.defaultValue(Collections.emptyMap())
70+
.withDescription("Clickhouse custom config");
71+
}

Diff for: ‎seatunnel-connectors-v2/connector-clickhouse/src/main/java/org/apache/seatunnel/connectors/seatunnel/clickhouse/config/ClickhouseFileSinkOptions.java

+87
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,87 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one or more
3+
* contributor license agreements. See the NOTICE file distributed with
4+
* this work for additional information regarding copyright ownership.
5+
* The ASF licenses this file to You under the Apache License, Version 2.0
6+
* (the "License"); you may not use this file except in compliance with
7+
* the License. You may obtain a copy of the License at
8+
*
9+
* http://www.apache.org/licenses/LICENSE-2.0
10+
*
11+
* Unless required by applicable law or agreed to in writing, software
12+
* distributed under the License is distributed on an "AS IS" BASIS,
13+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
* See the License for the specific language governing permissions and
15+
* limitations under the License.
16+
*/
17+
18+
package org.apache.seatunnel.connectors.seatunnel.clickhouse.config;
19+
20+
import org.apache.seatunnel.api.configuration.Option;
21+
import org.apache.seatunnel.api.configuration.Options;
22+
23+
import java.util.List;
24+
25+
public class ClickhouseFileSinkOptions {
26+
/** ClickhouseFile sink connector used clickhouse-local program's path */
27+
public static final Option<String> CLICKHOUSE_LOCAL_PATH =
28+
Options.key("clickhouse_local_path")
29+
.stringType()
30+
.noDefaultValue()
31+
.withDescription(
32+
"ClickhouseFile sink connector used clickhouse-local program's path");
33+
34+
/** The method of copy Clickhouse file */
35+
public static final Option<ClickhouseFileCopyMethod> COPY_METHOD =
36+
Options.key("copy_method")
37+
.enumType(ClickhouseFileCopyMethod.class)
38+
.defaultValue(ClickhouseFileCopyMethod.SCP)
39+
.withDescription("The method of copy Clickhouse file");
40+
41+
public static final Option<Boolean> COMPATIBLE_MODE =
42+
Options.key("compatible_mode")
43+
.booleanType()
44+
.defaultValue(false)
45+
.withDescription(
46+
"In the lower version of Clickhouse, the ClickhouseLocal program does not support the `--path` parameter, "
47+
+ "you need to use this mode to take other ways to realize the --path parameter function");
48+
49+
public static final String NODE_ADDRESS = "node_address";
50+
51+
public static final Option<Boolean> NODE_FREE_PASSWORD =
52+
Options.key("node_free_password")
53+
.booleanType()
54+
.defaultValue(false)
55+
.withDescription(
56+
"Because seatunnel need to use scp or rsync for file transfer, "
57+
+ "seatunnel need clickhouse server-side access. If each spark node and clickhouse server are configured with password-free login, "
58+
+ "you can configure this option to true, otherwise you need to configure the corresponding node password in the node_pass configuration");
59+
60+
/** The password of Clickhouse server node */
61+
public static final Option<List<NodePassConfig>> NODE_PASS =
62+
Options.key("node_pass")
63+
.listType(NodePassConfig.class)
64+
.noDefaultValue()
65+
.withDescription("The password of Clickhouse server node");
66+
67+
public static final Option<String> KEY_PATH =
68+
Options.key("key_path")
69+
.stringType()
70+
.noDefaultValue()
71+
.withDescription("The path of rsync/ssh key file");
72+
73+
public static final Option<String> FILE_FIELDS_DELIMITER =
74+
Options.key("file_fields_delimiter")
75+
.stringType()
76+
.defaultValue("\t")
77+
.withDescription(
78+
"ClickhouseFile uses csv format to temporarily save data. If the data in the row contains the delimiter value of csv,"
79+
+ " it may cause program exceptions. Avoid this with this configuration. Value string has to be an exactly one character long");
80+
81+
public static final Option<String> FILE_TEMP_PATH =
82+
Options.key("file_temp_path")
83+
.stringType()
84+
.defaultValue("/tmp/seatunnel/clickhouse-local/file")
85+
.withDescription(
86+
"The directory where ClickhouseFile stores temporary files locally.");
87+
}

Diff for: ‎seatunnel-connectors-v2/connector-clickhouse/src/main/java/org/apache/seatunnel/connectors/seatunnel/clickhouse/config/ClickhouseConfig.java renamed to ‎seatunnel-connectors-v2/connector-clickhouse/src/main/java/org/apache/seatunnel/connectors/seatunnel/clickhouse/config/ClickhouseSinkOptions.java

+1-115
Original file line numberDiff line numberDiff line change
@@ -23,12 +23,7 @@
2323
import org.apache.seatunnel.api.sink.SaveModePlaceHolder;
2424
import org.apache.seatunnel.api.sink.SchemaSaveMode;
2525

26-
import java.time.ZoneId;
27-
import java.util.Collections;
28-
import java.util.List;
29-
import java.util.Map;
30-
31-
public class ClickhouseConfig {
26+
public class ClickhouseSinkOptions {
3227

3328
/** Bulk size of clickhouse jdbc */
3429
public static final Option<Integer> BULK_SIZE =
@@ -37,56 +32,13 @@ public class ClickhouseConfig {
3732
.defaultValue(20000)
3833
.withDescription("Bulk size of clickhouse jdbc");
3934

40-
public static final Option<String> SQL =
41-
Options.key("sql")
42-
.stringType()
43-
.noDefaultValue()
44-
.withDescription("Clickhouse sql used to query data");
45-
46-
/** Clickhouse server host */
47-
public static final Option<String> HOST =
48-
Options.key("host")
49-
.stringType()
50-
.noDefaultValue()
51-
.withDescription("Clickhouse server host");
52-
5335
/** Clickhouse table name */
5436
public static final Option<String> TABLE =
5537
Options.key("table")
5638
.stringType()
5739
.noDefaultValue()
5840
.withDescription("Clickhouse table name");
5941

60-
/** Clickhouse database name */
61-
public static final Option<String> DATABASE =
62-
Options.key("database")
63-
.stringType()
64-
.noDefaultValue()
65-
.withDescription("Clickhouse database name");
66-
67-
/** Clickhouse server username */
68-
public static final Option<String> USERNAME =
69-
Options.key("username")
70-
.stringType()
71-
.noDefaultValue()
72-
.withDescription("Clickhouse server username");
73-
74-
/** Clickhouse server password */
75-
public static final Option<String> PASSWORD =
76-
Options.key("password")
77-
.stringType()
78-
.noDefaultValue()
79-
.withDescription("Clickhouse server password");
80-
81-
/** Clickhouse server timezone */
82-
public static final Option<String> SERVER_TIME_ZONE =
83-
Options.key("server_time_zone")
84-
.stringType()
85-
.defaultValue(ZoneId.systemDefault().getId())
86-
.withDescription(
87-
"The session time zone in database server."
88-
+ "If not set, then ZoneId.systemDefault() is used to determine the server time zone");
89-
9042
/** Split mode when table is distributed engine */
9143
public static final Option<Boolean> SPLIT_MODE =
9244
Options.key("split_mode")
@@ -121,72 +73,6 @@ public class ClickhouseConfig {
12173
.withDescription(
12274
"Allow experimental lightweight delete based on `*MergeTree` table engine");
12375

124-
/** ClickhouseFile sink connector used clickhouse-local program's path */
125-
public static final Option<String> CLICKHOUSE_LOCAL_PATH =
126-
Options.key("clickhouse_local_path")
127-
.stringType()
128-
.noDefaultValue()
129-
.withDescription(
130-
"ClickhouseFile sink connector used clickhouse-local program's path");
131-
132-
/** The method of copy Clickhouse file */
133-
public static final Option<ClickhouseFileCopyMethod> COPY_METHOD =
134-
Options.key("copy_method")
135-
.enumType(ClickhouseFileCopyMethod.class)
136-
.defaultValue(ClickhouseFileCopyMethod.SCP)
137-
.withDescription("The method of copy Clickhouse file");
138-
139-
public static final Option<Boolean> COMPATIBLE_MODE =
140-
Options.key("compatible_mode")
141-
.booleanType()
142-
.defaultValue(false)
143-
.withDescription(
144-
"In the lower version of Clickhouse, the ClickhouseLocal program does not support the `--path` parameter, "
145-
+ "you need to use this mode to take other ways to realize the --path parameter function");
146-
147-
public static final String NODE_ADDRESS = "node_address";
148-
149-
public static final Option<Boolean> NODE_FREE_PASSWORD =
150-
Options.key("node_free_password")
151-
.booleanType()
152-
.defaultValue(false)
153-
.withDescription(
154-
"Because seatunnel need to use scp or rsync for file transfer, "
155-
+ "seatunnel need clickhouse server-side access. If each spark node and clickhouse server are configured with password-free login, "
156-
+ "you can configure this option to true, otherwise you need to configure the corresponding node password in the node_pass configuration");
157-
/** The password of Clickhouse server node */
158-
public static final Option<List<NodePassConfig>> NODE_PASS =
159-
Options.key("node_pass")
160-
.listType(NodePassConfig.class)
161-
.noDefaultValue()
162-
.withDescription("The password of Clickhouse server node");
163-
164-
public static final Option<Map<String, String>> CLICKHOUSE_CONFIG =
165-
Options.key("clickhouse.config")
166-
.mapType()
167-
.defaultValue(Collections.emptyMap())
168-
.withDescription("Clickhouse custom config");
169-
170-
public static final Option<String> KEY_PATH =
171-
Options.key("key_path")
172-
.stringType()
173-
.noDefaultValue()
174-
.withDescription("The path of rsync/ssh key file");
175-
176-
public static final Option<String> FILE_FIELDS_DELIMITER =
177-
Options.key("file_fields_delimiter")
178-
.stringType()
179-
.defaultValue("\t")
180-
.withDescription(
181-
"ClickhouseFile uses csv format to temporarily save data. If the data in the row contains the delimiter value of csv,"
182-
+ " it may cause program exceptions. Avoid this with this configuration. Value string has to be an exactly one character long");
183-
184-
public static final Option<String> FILE_TEMP_PATH =
185-
Options.key("file_temp_path")
186-
.stringType()
187-
.defaultValue("/tmp/seatunnel/clickhouse-local/file")
188-
.withDescription(
189-
"The directory where ClickhouseFile stores temporary files locally.");
19076
public static final Option<SchemaSaveMode> SCHEMA_SAVE_MODE =
19177
Options.key("schema_save_mode")
19278
.enumType(SchemaSaveMode.class)

0 commit comments

Comments
 (0)