Skip to content

Commit fef6bb7

Browse files
PaultagorasPaultagoras
authored
Merge pull request #474 from ClickHouse/remove-redis-keystore-support
Removed RedisStateProvider + optimze imports for project
2 parents bce144d + 5c2af87 commit fef6bb7

File tree

56 files changed

+217
-250
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

56 files changed

+217
-250
lines changed

Diff for: ‎CHANGELOG.md

+1
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
# 1.2.5
2+
* Remove redis state provide since we are using KeeperMap for state storage
23
* Remove unused avro property from `build.gradle.kts`
34
* Trim schemaless data to only pass the fields that are in the table
45
* Allow bypassing the schema validation

Diff for: ‎build.gradle.kts

+5-2
Original file line numberDiff line numberDiff line change
@@ -7,9 +7,9 @@
77
*/
88

99
import com.github.jengelman.gradle.plugins.shadow.tasks.ShadowJar
10+
import org.gradle.api.tasks.testing.logging.TestExceptionFormat
1011
import java.time.LocalDateTime
1112
import java.time.format.DateTimeFormatter
12-
import org.gradle.api.tasks.testing.logging.TestExceptionFormat
1313

1414
val defaultJdkVersion = 17
1515
java {
@@ -72,7 +72,7 @@ dependencies {
7272
implementation("com.clickhouse:clickhouse-http-client:${project.extra["clickHouseDriverVersion"]}")
7373
implementation("com.clickhouse:clickhouse-data:${project.extra["clickHouseDriverVersion"]}")
7474
implementation("com.clickhouse:client-v2:${project.extra["clickHouseDriverVersion"]}")
75-
implementation("io.lettuce:lettuce-core:6.5.0.RELEASE")
75+
implementation("io.projectreactor:reactor-core:3.7.0")
7676
implementation("com.google.code.gson:gson:2.11.0")
7777
// https://mvnrepository.com/artifact/org.apache.httpcomponents.client5/httpclient5
7878
implementation("org.apache.httpcomponents.client5:httpclient5:5.3.1")
@@ -148,6 +148,9 @@ tasks.create("integrationTest", Test::class.java) {
148148
systemProperties = System.getProperties() as Map<String, Any>
149149
}
150150

151+
tasks.withType<JavaCompile> {
152+
options.compilerArgs.addAll(listOf("-Xlint:unchecked", "-Xlint:deprecation"))
153+
}
151154

152155
tasks.withType<Test> {
153156
maxParallelForks = (Runtime.getRuntime().availableProcessors() / 2).takeIf { it > 0 } ?: 1

Diff for: ‎src/integrationTest/java/com/clickhouse/kafka/connect/sink/ClickHouseCloudTest.java

+6-1
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,12 @@
99
import org.junit.jupiter.api.Test;
1010
import org.slf4j.Logger;
1111
import org.slf4j.LoggerFactory;
12-
import java.util.*;
12+
13+
import java.util.ArrayList;
14+
import java.util.Collection;
15+
import java.util.HashMap;
16+
import java.util.Map;
17+
import java.util.Properties;
1318

1419
import static org.junit.Assert.assertTrue;
1520

Diff for: ‎src/integrationTest/java/com/clickhouse/kafka/connect/sink/ClickHouseSinkConnectorIntegrationTest.java

+11-5
Original file line numberDiff line numberDiff line change
@@ -1,14 +1,17 @@
11
package com.clickhouse.kafka.connect.sink;
22

3-
import com.clickhouse.client.*;
3+
import com.clickhouse.client.ClickHouseProtocol;
44
import com.clickhouse.client.config.ClickHouseProxyType;
55
import com.clickhouse.kafka.connect.ClickHouseSinkConnector;
66
import com.clickhouse.kafka.connect.sink.db.helper.ClickHouseHelperClient;
77
import com.clickhouse.kafka.connect.sink.helper.ClickHouseTestHelpers;
8+
import com.clickhouse.kafka.connect.sink.helper.ConfluentPlatform;
89
import eu.rekawek.toxiproxy.Proxy;
910
import eu.rekawek.toxiproxy.ToxiproxyClient;
10-
import org.junit.jupiter.api.*;
11-
import com.clickhouse.kafka.connect.sink.helper.ConfluentPlatform;
11+
import org.junit.jupiter.api.AfterAll;
12+
import org.junit.jupiter.api.BeforeAll;
13+
import org.junit.jupiter.api.BeforeEach;
14+
import org.junit.jupiter.api.Test;
1215
import org.slf4j.Logger;
1316
import org.slf4j.LoggerFactory;
1417
import org.testcontainers.clickhouse.ClickHouseContainer;
@@ -19,10 +22,13 @@
1922
import java.io.IOException;
2023
import java.nio.file.Files;
2124
import java.nio.file.Paths;
22-
import java.util.*;
25+
import java.util.HashMap;
26+
import java.util.LinkedList;
27+
import java.util.List;
28+
import java.util.Map;
2329

2430
import static com.clickhouse.kafka.connect.sink.helper.ClickHouseAPI.*;
25-
import static org.junit.jupiter.api.Assertions.*;
31+
import static org.junit.jupiter.api.Assertions.assertTrue;
2632

2733
public class ClickHouseSinkConnectorIntegrationTest {
2834
private static final Logger LOGGER = LoggerFactory.getLogger(ClickHouseSinkConnectorIntegrationTest.class);

Diff for: ‎src/integrationTest/java/com/clickhouse/kafka/connect/sink/ExactlyOnceTest.java

+11-5
Original file line numberDiff line numberDiff line change
@@ -3,22 +3,28 @@
33
import com.clickhouse.client.ClickHouseProtocol;
44
import com.clickhouse.client.api.query.Records;
55
import com.clickhouse.client.config.ClickHouseProxyType;
6-
import com.clickhouse.data.ClickHouseRecord;
76
import com.clickhouse.kafka.connect.sink.db.helper.ClickHouseHelperClient;
87
import com.clickhouse.kafka.connect.sink.helper.ClickHouseAPI;
98
import com.clickhouse.kafka.connect.sink.helper.ConfluentPlatform;
10-
import org.junit.jupiter.api.*;
9+
import org.junit.jupiter.api.AfterAll;
10+
import org.junit.jupiter.api.BeforeAll;
11+
import org.junit.jupiter.api.BeforeEach;
12+
import org.junit.jupiter.api.Test;
1113
import org.slf4j.Logger;
1214
import org.slf4j.LoggerFactory;
1315
import org.testcontainers.containers.Network;
1416

15-
import java.io.*;
17+
import java.io.File;
18+
import java.io.IOException;
1619
import java.net.URISyntaxException;
1720
import java.nio.file.Files;
1821
import java.nio.file.Paths;
19-
import java.util.*;
22+
import java.util.LinkedList;
23+
import java.util.List;
24+
import java.util.Properties;
2025

21-
import static com.clickhouse.kafka.connect.sink.helper.ClickHouseAPI.*;
26+
import static com.clickhouse.kafka.connect.sink.helper.ClickHouseAPI.createReplicatedMergeTreeTable;
27+
import static com.clickhouse.kafka.connect.sink.helper.ClickHouseAPI.dropTable;
2228
import static org.junit.jupiter.api.Assertions.assertTrue;
2329

2430

Diff for: ‎src/integrationTest/java/com/clickhouse/kafka/connect/sink/helper/ClickHouseAPI.java

+10-4
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,11 @@
11
package com.clickhouse.kafka.connect.sink.helper;
22

3-
import com.clickhouse.client.*;
4-
import com.clickhouse.client.api.query.GenericRecord;
3+
import com.clickhouse.client.ClickHouseClient;
4+
import com.clickhouse.client.ClickHouseException;
5+
import com.clickhouse.client.ClickHouseNodeSelector;
6+
import com.clickhouse.client.ClickHouseProtocol;
7+
import com.clickhouse.client.ClickHouseResponse;
58
import com.clickhouse.client.api.query.Records;
6-
import com.clickhouse.data.ClickHouseRecord;
79
import com.clickhouse.kafka.connect.sink.db.helper.ClickHouseHelperClient;
810
import org.slf4j.Logger;
911
import org.slf4j.LoggerFactory;
@@ -16,7 +18,11 @@
1618
import java.net.http.HttpClient;
1719
import java.net.http.HttpRequest;
1820
import java.net.http.HttpResponse;
19-
import java.util.*;
21+
import java.util.Arrays;
22+
import java.util.Base64;
23+
import java.util.HashMap;
24+
import java.util.Map;
25+
import java.util.Properties;
2026
import java.util.concurrent.ExecutionException;
2127
import java.util.concurrent.TimeUnit;
2228
import java.util.concurrent.TimeoutException;

Diff for: ‎src/integrationTest/java/com/clickhouse/kafka/connect/sink/helper/ClickHouseTestHelpers.java

+5-1
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,10 @@
11
package com.clickhouse.kafka.connect.sink.helper;
22

3-
import com.clickhouse.client.*;
3+
import com.clickhouse.client.ClickHouseClient;
4+
import com.clickhouse.client.ClickHouseException;
5+
import com.clickhouse.client.ClickHouseNodeSelector;
6+
import com.clickhouse.client.ClickHouseProtocol;
7+
import com.clickhouse.client.ClickHouseResponse;
48
import com.clickhouse.data.ClickHouseRecord;
59
import com.clickhouse.kafka.connect.sink.db.helper.ClickHouseHelperClient;
610
import org.slf4j.Logger;

Diff for: ‎src/integrationTest/java/com/clickhouse/kafka/connect/sink/helper/ConfluentPlatform.java

+4-4
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,6 @@
11
package com.clickhouse.kafka.connect.sink.helper;
22

33

4-
5-
64
import okhttp3.MediaType;
75
import okhttp3.OkHttpClient;
86
import okhttp3.Request;
@@ -21,9 +19,11 @@
2119
import java.io.IOException;
2220
import java.nio.file.Files;
2321
import java.nio.file.Paths;
24-
import java.util.*;
22+
import java.util.HashMap;
23+
import java.util.List;
24+
import java.util.Map;
25+
import java.util.UUID;
2526
import java.util.regex.Pattern;
26-
import java.util.stream.Stream;
2727

2828

2929
public class ConfluentPlatform {

Diff for: ‎src/integrationTest/java/com/clickhouse/kafka/connect/sink/helper/SchemaTestData.java

+12-2
Original file line numberDiff line numberDiff line change
@@ -1,15 +1,25 @@
11
package com.clickhouse.kafka.connect.sink.helper;
22

33
import org.apache.kafka.common.record.TimestampType;
4-
import org.apache.kafka.connect.data.*;
4+
import org.apache.kafka.connect.data.Decimal;
5+
import org.apache.kafka.connect.data.Schema;
6+
import org.apache.kafka.connect.data.SchemaBuilder;
7+
import org.apache.kafka.connect.data.Struct;
8+
import org.apache.kafka.connect.data.Time;
9+
import org.apache.kafka.connect.data.Timestamp;
510
import org.apache.kafka.connect.sink.SinkRecord;
611

712
import java.math.BigDecimal;
813
import java.time.LocalDate;
914
import java.time.LocalDateTime;
1015
import java.time.ZoneOffset;
16+
import java.util.ArrayList;
17+
import java.util.Arrays;
18+
import java.util.Collection;
1119
import java.util.Date;
12-
import java.util.*;
20+
import java.util.List;
21+
import java.util.Map;
22+
import java.util.UUID;
1323
import java.util.stream.LongStream;
1424

1525
public class SchemaTestData {

Diff for: ‎src/integrationTest/java/com/clickhouse/kafka/connect/sink/helper/SchemalessTestData.java

+6-1
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,12 @@
44
import org.apache.kafka.connect.sink.SinkRecord;
55

66
import java.math.BigDecimal;
7-
import java.util.*;
7+
import java.util.ArrayList;
8+
import java.util.Arrays;
9+
import java.util.Collection;
10+
import java.util.HashMap;
11+
import java.util.List;
12+
import java.util.Map;
813
import java.util.stream.LongStream;
914

1015
public class SchemalessTestData {

Diff for: ‎src/main/java/com/clickhouse/kafka/connect/ClickHouseSinkConnector.java

-2
Original file line numberDiff line numberDiff line change
@@ -7,13 +7,11 @@
77
import org.apache.kafka.common.config.ConfigDef;
88
import org.apache.kafka.connect.connector.Task;
99
import org.apache.kafka.connect.sink.SinkConnector;
10-
1110
import org.apache.kafka.connect.sink.SinkConnectorContext;
1211
import org.slf4j.Logger;
1312
import org.slf4j.LoggerFactory;
1413

1514
import java.util.ArrayList;
16-
import java.util.HashMap;
1715
import java.util.List;
1816
import java.util.Map;
1917
import java.util.stream.Collectors;

Diff for: ‎src/main/java/com/clickhouse/kafka/connect/sink/ClickHouseSinkConfig.java

-6
Original file line numberDiff line numberDiff line change
@@ -62,12 +62,6 @@ public class ClickHouseSinkConfig {
6262
public static final Integer tableRefreshIntervalDefault = 0;
6363
public static final Boolean exactlyOnceDefault = Boolean.FALSE;
6464
public static final Boolean customInsertFormatDefault = Boolean.FALSE;
65-
public enum StateStores {
66-
NONE,
67-
IN_MEMORY,
68-
REDIS,
69-
KEEPER_MAP
70-
}
7165

7266
private final String hostname;
7367
private final int port;

Diff for: ‎src/main/java/com/clickhouse/kafka/connect/sink/ClickHouseSinkTask.java

+2-19
Original file line numberDiff line numberDiff line change
@@ -1,35 +1,18 @@
11
package com.clickhouse.kafka.connect.sink;
22

3-
import com.clickhouse.client.*;
4-
import com.clickhouse.kafka.connect.ClickHouseSinkConnector;
5-
import com.clickhouse.kafka.connect.sink.data.Record;
6-
import com.clickhouse.kafka.connect.sink.db.ClickHouseWriter;
7-
import com.clickhouse.kafka.connect.sink.db.DBWriter;
8-
import com.clickhouse.kafka.connect.sink.db.InMemoryDBWriter;
93
import com.clickhouse.kafka.connect.sink.dlq.ErrorReporter;
10-
import com.clickhouse.kafka.connect.sink.kafka.RangeContainer;
11-
import com.clickhouse.kafka.connect.sink.processing.Processing;
12-
import com.clickhouse.kafka.connect.sink.state.State;
13-
import com.clickhouse.kafka.connect.sink.state.StateProvider;
14-
import com.clickhouse.kafka.connect.sink.state.StateRecord;
15-
import com.clickhouse.kafka.connect.sink.state.provider.InMemoryState;
16-
import com.clickhouse.kafka.connect.sink.state.provider.RedisStateProvider;
174
import com.clickhouse.kafka.connect.util.Utils;
18-
import com.clickhouse.kafka.connect.util.jmx.SinkTaskStatistics;
195
import org.apache.kafka.clients.consumer.OffsetAndMetadata;
206
import org.apache.kafka.common.TopicPartition;
21-
import org.apache.kafka.connect.data.Field;
22-
import org.apache.kafka.connect.data.Struct;
237
import org.apache.kafka.connect.errors.ConnectException;
248
import org.apache.kafka.connect.sink.ErrantRecordReporter;
259
import org.apache.kafka.connect.sink.SinkRecord;
2610
import org.apache.kafka.connect.sink.SinkTask;
27-
2811
import org.slf4j.Logger;
2912
import org.slf4j.LoggerFactory;
3013

31-
import java.util.*;
32-
import java.util.stream.Collectors;
14+
import java.util.Collection;
15+
import java.util.Map;
3316

3417
public class ClickHouseSinkTask extends SinkTask {
3518

Diff for: ‎src/main/java/com/clickhouse/kafka/connect/sink/ProxySinkTask.java

+2-2
Original file line numberDiff line numberDiff line change
@@ -9,9 +9,9 @@
99
import com.clickhouse.kafka.connect.sink.state.StateProvider;
1010
import com.clickhouse.kafka.connect.sink.state.provider.InMemoryState;
1111
import com.clickhouse.kafka.connect.sink.state.provider.KeeperStateProvider;
12+
import com.clickhouse.kafka.connect.util.jmx.ExecutionTimer;
1213
import com.clickhouse.kafka.connect.util.jmx.MBeanServerUtils;
1314
import com.clickhouse.kafka.connect.util.jmx.SinkTaskStatistics;
14-
import com.clickhouse.kafka.connect.util.jmx.ExecutionTimer;
1515
import org.apache.kafka.connect.sink.SinkRecord;
1616
import org.slf4j.Logger;
1717
import org.slf4j.LoggerFactory;
@@ -20,10 +20,10 @@
2020
import java.util.Collection;
2121
import java.util.List;
2222
import java.util.Map;
23+
import java.util.Timer;
2324
import java.util.concurrent.ExecutionException;
2425
import java.util.concurrent.atomic.AtomicInteger;
2526
import java.util.stream.Collectors;
26-
import java.util.Timer;
2727

2828
public class ProxySinkTask {
2929

Diff for: ‎src/main/java/com/clickhouse/kafka/connect/sink/data/Record.java

+1-3
Original file line numberDiff line numberDiff line change
@@ -2,8 +2,8 @@
22

33
import com.clickhouse.kafka.connect.sink.data.convert.EmptyRecordConvertor;
44
import com.clickhouse.kafka.connect.sink.data.convert.RecordConvertor;
5-
import com.clickhouse.kafka.connect.sink.data.convert.SchemalessRecordConvertor;
65
import com.clickhouse.kafka.connect.sink.data.convert.SchemaRecordConvertor;
6+
import com.clickhouse.kafka.connect.sink.data.convert.SchemalessRecordConvertor;
77
import com.clickhouse.kafka.connect.sink.data.convert.StringRecordConvertor;
88
import com.clickhouse.kafka.connect.sink.kafka.OffsetContainer;
99
import lombok.Getter;
@@ -12,8 +12,6 @@
1212
import org.apache.kafka.connect.data.Struct;
1313
import org.apache.kafka.connect.errors.DataException;
1414
import org.apache.kafka.connect.sink.SinkRecord;
15-
import org.slf4j.Logger;
16-
import org.slf4j.LoggerFactory;
1715

1816
import java.util.List;
1917
import java.util.Map;

Diff for: ‎src/main/java/com/clickhouse/kafka/connect/sink/data/StructToJsonMap.java

+7-2
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,12 @@
11
package com.clickhouse.kafka.connect.sink.data;
22

3-
import org.apache.kafka.connect.data.*;
3+
import org.apache.kafka.connect.data.Date;
4+
import org.apache.kafka.connect.data.Decimal;
5+
import org.apache.kafka.connect.data.Field;
6+
import org.apache.kafka.connect.data.Schema;
7+
import org.apache.kafka.connect.data.Struct;
8+
import org.apache.kafka.connect.data.Time;
9+
import org.apache.kafka.connect.data.Timestamp;
410
import org.slf4j.Logger;
511
import org.slf4j.LoggerFactory;
612

@@ -9,7 +15,6 @@
915
import java.util.HashMap;
1016
import java.util.List;
1117
import java.util.Map;
12-
import java.util.stream.Collectors;
1318

1419
public class StructToJsonMap {
1520

Diff for: ‎src/main/java/com/clickhouse/kafka/connect/sink/data/convert/EmptyRecordConvertor.java

-4
Original file line numberDiff line numberDiff line change
@@ -1,17 +1,13 @@
11
package com.clickhouse.kafka.connect.sink.data.convert;
22

3-
import com.clickhouse.kafka.connect.sink.data.Data;
43
import com.clickhouse.kafka.connect.sink.data.Record;
54
import com.clickhouse.kafka.connect.sink.data.SchemaType;
65
import com.clickhouse.kafka.connect.sink.kafka.OffsetContainer;
76
import org.apache.kafka.connect.data.Field;
8-
import org.apache.kafka.connect.data.Schema;
97
import org.apache.kafka.connect.sink.SinkRecord;
108

119
import java.util.ArrayList;
12-
import java.util.HashMap;
1310
import java.util.List;
14-
import java.util.Map;
1511

1612
public class EmptyRecordConvertor extends RecordConvertor {
1713
@Override

Diff for: ‎src/main/java/com/clickhouse/kafka/connect/sink/data/convert/RecordConvertor.java

+2-2
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,10 @@
11
package com.clickhouse.kafka.connect.sink.data.convert;
22

3-
import java.util.regex.Pattern;
4-
import org.apache.kafka.connect.data.Schema;
53
import com.clickhouse.kafka.connect.sink.data.Record;
64
import org.apache.kafka.connect.sink.SinkRecord;
75

6+
import java.util.regex.Pattern;
7+
88
public abstract class RecordConvertor {
99
public Record convert(SinkRecord sinkRecord, boolean splitDBTopic, String dbTopicSeparatorChar, String configurationDatabase) {
1010
String database = configurationDatabase;

Diff for: ‎src/main/java/com/clickhouse/kafka/connect/sink/data/convert/SchemaRecordConvertor.java

-2
Original file line numberDiff line numberDiff line change
@@ -7,8 +7,6 @@
77
import com.clickhouse.kafka.connect.sink.kafka.OffsetContainer;
88
import org.apache.kafka.connect.data.Struct;
99
import org.apache.kafka.connect.sink.SinkRecord;
10-
import org.slf4j.Logger;
11-
import org.slf4j.LoggerFactory;
1210

1311
import java.util.Map;
1412

0 commit comments

Comments
 (0)