Skip to content

Commit da8d9cb

Browse files
fcb-xiaobofcb-xiaobo
authored
[improve] add StarRocks options (#8639)
1 parent f1c313e commit da8d9cb

File tree

12 files changed

+202
-218
lines changed

12 files changed

+202
-218
lines changed

Diff for: ‎seatunnel-ci-tools/src/test/java/org/apache/seatunnel/api/ConnectorOptionCheckTest.java

-1
Original file line numberDiff line numberDiff line change
@@ -191,7 +191,6 @@ private Set<String> buildWhiteList() {
191191
whiteList.add("IoTDBSinkOptions");
192192
whiteList.add("EasysearchSourceOptions");
193193
whiteList.add("RabbitmqSinkOptions");
194-
whiteList.add("StarRocksSourceOptions");
195194
whiteList.add("IcebergSourceOptions");
196195
whiteList.add("HbaseSourceOptions");
197196
whiteList.add("PaimonSourceOptions");

Diff for: ‎seatunnel-connectors-v2/connector-starrocks/src/main/java/org/apache/seatunnel/connectors/seatunnel/starrocks/catalog/StarRocksCatalogFactory.java

+8-9
Original file line numberDiff line numberDiff line change
@@ -22,23 +22,22 @@
2222
import org.apache.seatunnel.api.table.catalog.Catalog;
2323
import org.apache.seatunnel.api.table.factory.CatalogFactory;
2424
import org.apache.seatunnel.api.table.factory.Factory;
25-
import org.apache.seatunnel.connectors.seatunnel.starrocks.config.CommonConfig;
26-
import org.apache.seatunnel.connectors.seatunnel.starrocks.config.StarRocksOptions;
2725
import org.apache.seatunnel.connectors.seatunnel.starrocks.config.StarRocksSinkOptions;
26+
import org.apache.seatunnel.connectors.seatunnel.starrocks.config.StarRocksSourceOptions;
2827

2928
import com.google.auto.service.AutoService;
3029

3130
@AutoService(Factory.class)
3231
public class StarRocksCatalogFactory implements CatalogFactory {
33-
public static final String IDENTIFIER = CommonConfig.CONNECTOR_IDENTITY;
32+
public static final String IDENTIFIER = StarRocksSinkOptions.CONNECTOR_IDENTITY;
3433

3534
@Override
3635
public Catalog createCatalog(String catalogName, ReadonlyConfig options) {
3736
return new StarRocksCatalog(
3837
catalogName,
39-
options.get(StarRocksOptions.USERNAME),
40-
options.get(StarRocksOptions.PASSWORD),
41-
options.get(StarRocksOptions.BASE_URL),
38+
options.get(StarRocksSourceOptions.USERNAME),
39+
options.get(StarRocksSourceOptions.PASSWORD),
40+
options.get(StarRocksSinkOptions.BASE_URL),
4241
options.get(StarRocksSinkOptions.SAVE_MODE_CREATE_TEMPLATE));
4342
}
4443

@@ -50,9 +49,9 @@ public String factoryIdentifier() {
5049
@Override
5150
public OptionRule optionRule() {
5251
return OptionRule.builder()
53-
.required(StarRocksOptions.BASE_URL)
54-
.required(StarRocksOptions.USERNAME)
55-
.required(StarRocksOptions.PASSWORD)
52+
.required(StarRocksSinkOptions.BASE_URL)
53+
.required(StarRocksSourceOptions.USERNAME)
54+
.required(StarRocksSourceOptions.PASSWORD)
5655
.build();
5756
}
5857
}

Diff for: ‎seatunnel-connectors-v2/connector-starrocks/src/main/java/org/apache/seatunnel/connectors/seatunnel/starrocks/config/SinkConfig.java

+3-3
Original file line numberDiff line numberDiff line change
@@ -71,9 +71,9 @@ public static SinkConfig of(ReadonlyConfig config) {
7171
SinkConfig sinkConfig = new SinkConfig();
7272
sinkConfig.setNodeUrls(config.get(StarRocksSinkOptions.NODE_URLS));
7373
sinkConfig.setDatabase(config.get(StarRocksSinkOptions.DATABASE));
74-
sinkConfig.setJdbcUrl(config.get(StarRocksOptions.BASE_URL));
75-
config.getOptional(StarRocksOptions.USERNAME).ifPresent(sinkConfig::setUsername);
76-
config.getOptional(StarRocksOptions.PASSWORD).ifPresent(sinkConfig::setPassword);
74+
sinkConfig.setJdbcUrl(config.get(StarRocksSinkOptions.BASE_URL));
75+
config.getOptional(StarRocksSinkOptions.USERNAME).ifPresent(sinkConfig::setUsername);
76+
config.getOptional(StarRocksSinkOptions.PASSWORD).ifPresent(sinkConfig::setPassword);
7777
config.getOptional(StarRocksSinkOptions.TABLE).ifPresent(sinkConfig::setTable);
7878
config.getOptional(StarRocksSinkOptions.LABEL_PREFIX).ifPresent(sinkConfig::setLabelPrefix);
7979
sinkConfig.setBatchMaxSize(config.get(StarRocksSinkOptions.BATCH_MAX_SIZE));

Diff for: ‎seatunnel-connectors-v2/connector-starrocks/src/main/java/org/apache/seatunnel/connectors/seatunnel/starrocks/config/SourceConfig.java

+19-74
Original file line numberDiff line numberDiff line change
@@ -17,8 +17,6 @@
1717

1818
package org.apache.seatunnel.connectors.seatunnel.starrocks.config;
1919

20-
import org.apache.seatunnel.api.configuration.Option;
21-
import org.apache.seatunnel.api.configuration.Options;
2220
import org.apache.seatunnel.api.configuration.ReadonlyConfig;
2321

2422
import lombok.Getter;
@@ -29,22 +27,20 @@
2927

3028
@Setter
3129
@Getter
32-
public class SourceConfig extends CommonConfig {
33-
34-
private static final long DEFAULT_SCAN_MEM_LIMIT = 1024 * 1024 * 1024L;
30+
public class SourceConfig extends StarRocksConfig {
3531

3632
public SourceConfig(ReadonlyConfig config) {
3733
super(config);
38-
this.maxRetries = config.get(MAX_RETRIES);
39-
this.requestTabletSize = config.get(QUERY_TABLET_SIZE);
40-
this.scanFilter = config.get(SCAN_FILTER);
41-
this.connectTimeoutMs = config.get(SCAN_CONNECT_TIMEOUT);
42-
this.batchRows = config.get(SCAN_BATCH_ROWS);
43-
this.keepAliveMin = config.get(SCAN_KEEP_ALIVE_MIN);
44-
this.queryTimeoutSec = config.get(SCAN_QUERY_TIMEOUT_SEC);
45-
this.memLimit = config.get(SCAN_MEM_LIMIT);
46-
47-
String prefix = STARROCKS_SCAN_CONFIG_PREFIX.key();
34+
this.maxRetries = config.get(StarRocksSourceOptions.MAX_RETRIES);
35+
this.requestTabletSize = config.get(StarRocksSourceOptions.QUERY_TABLET_SIZE);
36+
this.scanFilter = config.get(StarRocksSourceOptions.SCAN_FILTER);
37+
this.connectTimeoutMs = config.get(StarRocksSourceOptions.SCAN_CONNECT_TIMEOUT);
38+
this.batchRows = config.get(StarRocksSourceOptions.SCAN_BATCH_ROWS);
39+
this.keepAliveMin = config.get(StarRocksSourceOptions.SCAN_KEEP_ALIVE_MIN);
40+
this.queryTimeoutSec = config.get(StarRocksSourceOptions.SCAN_QUERY_TIMEOUT_SEC);
41+
this.memLimit = config.get(StarRocksSourceOptions.SCAN_MEM_LIMIT);
42+
43+
String prefix = StarRocksSourceOptions.STARROCKS_SCAN_CONFIG_PREFIX.key();
4844
config.toMap()
4945
.forEach(
5046
(key, value) -> {
@@ -55,64 +51,13 @@ public SourceConfig(ReadonlyConfig config) {
5551
});
5652
}
5753

58-
public static final Option<Integer> MAX_RETRIES =
59-
Options.key("max_retries")
60-
.intType()
61-
.defaultValue(3)
62-
.withDescription("number of retry requests sent to StarRocks");
63-
64-
public static final Option<Integer> QUERY_TABLET_SIZE =
65-
Options.key("request_tablet_size")
66-
.intType()
67-
.defaultValue(Integer.MAX_VALUE)
68-
.withDescription("The number of Tablets corresponding to an Partition");
69-
70-
public static final Option<String> SCAN_FILTER =
71-
Options.key("scan_filter").stringType().defaultValue("").withDescription("SQL filter");
72-
73-
public static final Option<Integer> SCAN_CONNECT_TIMEOUT =
74-
Options.key("scan_connect_timeout_ms")
75-
.intType()
76-
.defaultValue(1000)
77-
.withDescription("scan connect timeout");
78-
79-
public static final Option<Integer> SCAN_BATCH_ROWS =
80-
Options.key("scan_batch_rows")
81-
.intType()
82-
.defaultValue(1024)
83-
.withDescription("scan batch rows");
84-
85-
public static final Option<Integer> SCAN_KEEP_ALIVE_MIN =
86-
Options.key("scan_keep_alive_min")
87-
.intType()
88-
.defaultValue(10)
89-
.withDescription("Max keep alive time min");
90-
91-
public static final Option<Integer> SCAN_QUERY_TIMEOUT_SEC =
92-
Options.key("scan_query_timeout_sec")
93-
.intType()
94-
.defaultValue(3600)
95-
.withDescription("Query timeout for a single query");
96-
97-
public static final Option<Long> SCAN_MEM_LIMIT =
98-
Options.key("scan_mem_limit")
99-
.longType()
100-
.defaultValue(DEFAULT_SCAN_MEM_LIMIT)
101-
.withDescription("Memory byte limit for a single query");
102-
103-
public static final Option<String> STARROCKS_SCAN_CONFIG_PREFIX =
104-
Options.key("scan.params.")
105-
.stringType()
106-
.noDefaultValue()
107-
.withDescription("The parameter of the scan data from be");
108-
109-
private int maxRetries = MAX_RETRIES.defaultValue();
110-
private int requestTabletSize = QUERY_TABLET_SIZE.defaultValue();
111-
private String scanFilter = SCAN_FILTER.defaultValue();
112-
private long memLimit = SCAN_MEM_LIMIT.defaultValue();
113-
private int queryTimeoutSec = SCAN_QUERY_TIMEOUT_SEC.defaultValue();
114-
private int keepAliveMin = SCAN_KEEP_ALIVE_MIN.defaultValue();
115-
private int batchRows = SCAN_BATCH_ROWS.defaultValue();
116-
private int connectTimeoutMs = SCAN_CONNECT_TIMEOUT.defaultValue();
54+
private int maxRetries = StarRocksSourceOptions.MAX_RETRIES.defaultValue();
55+
private int requestTabletSize = StarRocksSourceOptions.QUERY_TABLET_SIZE.defaultValue();
56+
private String scanFilter = StarRocksSourceOptions.SCAN_FILTER.defaultValue();
57+
private long memLimit = StarRocksSourceOptions.SCAN_MEM_LIMIT.defaultValue();
58+
private int queryTimeoutSec = StarRocksSourceOptions.SCAN_QUERY_TIMEOUT_SEC.defaultValue();
59+
private int keepAliveMin = StarRocksSourceOptions.SCAN_KEEP_ALIVE_MIN.defaultValue();
60+
private int batchRows = StarRocksSourceOptions.SCAN_BATCH_ROWS.defaultValue();
61+
private int connectTimeoutMs = StarRocksSourceOptions.SCAN_CONNECT_TIMEOUT.defaultValue();
11762
private Map<String, String> sourceOptionProps = new HashMap<>();
11863
}

Diff for: ‎seatunnel-connectors-v2/connector-starrocks/src/main/java/org/apache/seatunnel/connectors/seatunnel/starrocks/config/CommonConfig.java renamed to ‎seatunnel-connectors-v2/connector-starrocks/src/main/java/org/apache/seatunnel/connectors/seatunnel/starrocks/config/StarRocksBaseOptions.java

+1-26
Original file line numberDiff line numberDiff line change
@@ -14,27 +14,16 @@
1414
* See the License for the specific language governing permissions and
1515
* limitations under the License.
1616
*/
17-
1817
package org.apache.seatunnel.connectors.seatunnel.starrocks.config;
1918

2019
import org.apache.seatunnel.api.configuration.Option;
2120
import org.apache.seatunnel.api.configuration.Options;
22-
import org.apache.seatunnel.api.configuration.ReadonlyConfig;
23-
24-
import lombok.AllArgsConstructor;
25-
import lombok.Getter;
26-
import lombok.ToString;
2721

2822
import java.io.Serializable;
2923
import java.util.List;
3024

31-
@Getter
32-
@ToString
33-
@AllArgsConstructor
34-
public class CommonConfig implements Serializable {
35-
25+
public class StarRocksBaseOptions implements Serializable {
3626
public static final String CONNECTOR_IDENTITY = "StarRocks";
37-
3827
public static final Option<List<String>> NODE_URLS =
3928
Options.key("nodeUrls")
4029
.listType()
@@ -65,18 +54,4 @@ public class CommonConfig implements Serializable {
6554
.stringType()
6655
.noDefaultValue()
6756
.withDescription("StarRocks user password");
68-
69-
private List<String> nodeUrls;
70-
private String username;
71-
private String password;
72-
private String database;
73-
private String table;
74-
75-
public CommonConfig(ReadonlyConfig config) {
76-
this.nodeUrls = config.get(NODE_URLS);
77-
this.username = config.get(USERNAME);
78-
this.password = config.get(PASSWORD);
79-
this.database = config.get(DATABASE);
80-
this.table = config.get(TABLE);
81-
}
8257
}

Diff for: ‎seatunnel-connectors-v2/connector-starrocks/src/main/java/org/apache/seatunnel/connectors/seatunnel/starrocks/config/StarRocksConfig.java

+47
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,47 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one or more
3+
* contributor license agreements. See the NOTICE file distributed with
4+
* this work for additional information regarding copyright ownership.
5+
* The ASF licenses this file to You under the Apache License, Version 2.0
6+
* (the "License"); you may not use this file except in compliance with
7+
* the License. You may obtain a copy of the License at
8+
*
9+
* http://www.apache.org/licenses/LICENSE-2.0
10+
*
11+
* Unless required by applicable law or agreed to in writing, software
12+
* distributed under the License is distributed on an "AS IS" BASIS,
13+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
* See the License for the specific language governing permissions and
15+
* limitations under the License.
16+
*/
17+
18+
package org.apache.seatunnel.connectors.seatunnel.starrocks.config;
19+
20+
import org.apache.seatunnel.api.configuration.ReadonlyConfig;
21+
22+
import lombok.AllArgsConstructor;
23+
import lombok.Getter;
24+
import lombok.ToString;
25+
26+
import java.io.Serializable;
27+
import java.util.List;
28+
29+
@Getter
30+
@ToString
31+
@AllArgsConstructor
32+
public class StarRocksConfig implements Serializable {
33+
34+
private List<String> nodeUrls;
35+
private String username;
36+
private String password;
37+
private String database;
38+
private String table;
39+
40+
public StarRocksConfig(ReadonlyConfig config) {
41+
this.nodeUrls = config.get(StarRocksBaseOptions.NODE_URLS);
42+
this.username = config.get(StarRocksBaseOptions.USERNAME);
43+
this.password = config.get(StarRocksBaseOptions.PASSWORD);
44+
this.database = config.get(StarRocksBaseOptions.DATABASE);
45+
this.table = config.get(StarRocksBaseOptions.TABLE);
46+
}
47+
}

Diff for: ‎seatunnel-connectors-v2/connector-starrocks/src/main/java/org/apache/seatunnel/connectors/seatunnel/starrocks/config/StarRocksOptions.java

-43
This file was deleted.

0 commit comments

Comments
 (0)