Skip to content

Commit 1f93b62

Browse files
BillFarberBillFarber
committed
Merge branch 'master' of https://github.com/marklogic-community/kafka-marklogic-connector into origin-master
2 parents 4a8d769 + d9338af commit 1f93b62

File tree

9 files changed

+239
-29
lines changed

9 files changed

+239
-29
lines changed

‎MarkLogic_Kafka_Connector_v1.2.2.pdf

870 KB
Binary file not shown.

‎README.md

+6
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,9 @@
1+
# v1.2.2 Changes
2+
1. Support of additional authentication options
3+
2. Documentation of how to update the connector for security options
4+
5+
Refer MarkLogic_Kafka_Connector_v1.2.2.pdf for details
6+
17
# kafka-connect-marklogic
28

39
This is a connector for subscribing to Kafka queues and pushing messages to MarkLogic

‎build.gradle

+5-3
Original file line numberDiff line numberDiff line change
@@ -17,9 +17,10 @@ configurations {
1717
}
1818

1919
dependencies {
20-
compileOnly "org.apache.kafka:connect-api:2.3.0"
20+
compileOnly "org.apache.kafka:connect-api:2.5.0"
21+
compileOnly "org.apache.kafka:connect-json:2.5.0"
2122

22-
compile ("com.marklogic:marklogic-data-hub:5.2.0") {
23+
compile ("com.marklogic:marklogic-data-hub:5.2.2") {
2324
// Excluding these because there's no need for them
2425
exclude module: "spring-boot-autoconfigure"
2526
exclude module: "spring-integration-http"
@@ -31,7 +32,8 @@ dependencies {
3132
}
3233

3334
testCompile "org.junit.jupiter:junit-jupiter-api:5.3.0"
34-
testCompile "org.apache.kafka:connect-api:2.3.0"
35+
testCompile "org.apache.kafka:connect-api:2.5.0"
36+
testCompile "org.apache.kafka:connect-json:2.5.0"
3537

3638
// Needed by Gradle 4.6+ - see https://www.petrikainulainen.net/programming/testing/junit-5-tutorial-running-unit-tests-with-gradle/
3739
testRuntime "org.junit.jupiter:junit-jupiter-engine:5.3.0"

‎gradle.properties

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
group=com.marklogic
2-
version=1.2.1
2+
version=1.2.2
33

44
# For the Confluent Connector Archive
55
componentOwner=marklogic

‎src/main/java/com/marklogic/kafka/connect/DefaultDatabaseClientConfigBuilder.java

+136-19
Original file line numberDiff line numberDiff line change
@@ -1,14 +1,33 @@
11
package com.marklogic.kafka.connect;
22

3+
import javax.net.ssl.SSLContext;
4+
import java.security.NoSuchAlgorithmException;
5+
import java.util.Map;
6+
7+
import java.io.FileInputStream;
8+
import java.io.FileNotFoundException;
9+
import java.io.IOException;
10+
import java.io.InputStream;
11+
import java.security.KeyManagementException;
12+
import java.security.KeyStore;
13+
import java.security.KeyStoreException;
14+
import java.security.NoSuchAlgorithmException;
15+
import java.security.UnrecoverableKeyException;
16+
import java.security.cert.CertificateException;
17+
18+
import javax.net.ssl.KeyManager;
19+
import javax.net.ssl.KeyManagerFactory;
20+
import javax.net.ssl.SSLContext;
21+
import javax.net.ssl.TrustManager;
22+
import javax.net.ssl.TrustManagerFactory;
23+
324
import com.marklogic.client.DatabaseClient;
425
import com.marklogic.client.DatabaseClientFactory;
526
import com.marklogic.client.ext.DatabaseClientConfig;
627
import com.marklogic.client.ext.SecurityContextType;
728
import com.marklogic.kafka.connect.sink.MarkLogicSinkConfig;
829

9-
import javax.net.ssl.SSLContext;
10-
import java.security.NoSuchAlgorithmException;
11-
import java.util.Map;
30+
import com.marklogic.client.ext.modulesloader.ssl.SimpleX509TrustManager;
1231

1332
public class DefaultDatabaseClientConfigBuilder implements DatabaseClientConfigBuilder {
1433

@@ -17,32 +36,26 @@ public DatabaseClientConfig buildDatabaseClientConfig(Map<String, String> kafkaC
1736
DatabaseClientConfig clientConfig = new DatabaseClientConfig();
1837
clientConfig.setCertFile(kafkaConfig.get(MarkLogicSinkConfig.CONNECTION_CERT_FILE));
1938
clientConfig.setCertPassword(kafkaConfig.get(MarkLogicSinkConfig.CONNECTION_CERT_PASSWORD));
20-
21-
String type = kafkaConfig.get(MarkLogicSinkConfig.CONNECTION_TYPE);
22-
if (type != null && type.trim().length() > 0) {
23-
clientConfig.setConnectionType(DatabaseClient.ConnectionType.valueOf(type.toUpperCase()));
24-
}
25-
39+
clientConfig.setTrustManager(new SimpleX509TrustManager());
40+
clientConfig = configureHostNameVerifier(clientConfig,kafkaConfig);
2641
String database = kafkaConfig.get(MarkLogicSinkConfig.CONNECTION_DATABASE);
2742
if (database != null && database.trim().length() > 0) {
2843
clientConfig.setDatabase(database);
2944
}
30-
45+
String connType = kafkaConfig.get(MarkLogicSinkConfig.CONNECTION_TYPE);
46+
if (connType != null && connType.trim().length() > 0) {
47+
clientConfig.setConnectionType(DatabaseClient.ConnectionType.valueOf(connType.toUpperCase()));
48+
}
3149
clientConfig.setExternalName(kafkaConfig.get(MarkLogicSinkConfig.CONNECTION_EXTERNAL_NAME));
3250
clientConfig.setHost(kafkaConfig.get(MarkLogicSinkConfig.CONNECTION_HOST));
3351
clientConfig.setPassword(kafkaConfig.get(MarkLogicSinkConfig.CONNECTION_PASSWORD));
3452
clientConfig.setPort(Integer.parseInt(kafkaConfig.get(MarkLogicSinkConfig.CONNECTION_PORT)));
35-
36-
String securityContextType = kafkaConfig.get(MarkLogicSinkConfig.CONNECTION_SECURITY_CONTEXT_TYPE).toUpperCase();
37-
clientConfig.setSecurityContextType(SecurityContextType.valueOf(securityContextType));
38-
53+
clientConfig = configureCustomSslConnection(clientConfig, kafkaConfig);
3954
String simpleSsl = kafkaConfig.get(MarkLogicSinkConfig.CONNECTION_SIMPLE_SSL);
4055
if (simpleSsl != null && Boolean.parseBoolean(simpleSsl)) {
41-
configureSimpleSsl(clientConfig);
56+
clientConfig = configureSimpleSsl(clientConfig);
4257
}
43-
4458
clientConfig.setUsername(kafkaConfig.get(MarkLogicSinkConfig.CONNECTION_USERNAME));
45-
4659
return clientConfig;
4760
}
4861

@@ -53,13 +66,117 @@ public DatabaseClientConfig buildDatabaseClientConfig(Map<String, String> kafkaC
5366
*
5467
* @param clientConfig
5568
*/
56-
protected void configureSimpleSsl(DatabaseClientConfig clientConfig) {
69+
protected DatabaseClientConfig configureSimpleSsl(DatabaseClientConfig clientConfig) {
5770
try {
5871
clientConfig.setSslContext(SSLContext.getDefault());
72+
clientConfig.setTrustManager(new SimpleX509TrustManager());
5973
} catch (NoSuchAlgorithmException e) {
6074
throw new RuntimeException("Unable to get default SSLContext: " + e.getMessage(), e);
6175
}
62-
6376
clientConfig.setSslHostnameVerifier(DatabaseClientFactory.SSLHostnameVerifier.ANY);
77+
return clientConfig;
78+
}
79+
80+
/**
81+
* This function configures the Host Name verifier based on the configuration.
82+
* ANY, STRICT and COMMON are the possible values, ANY being default.
83+
*
84+
* @param clientConfig
85+
*/
86+
protected DatabaseClientConfig configureHostNameVerifier(DatabaseClientConfig clientConfig, Map<String, String> kafkaConfig) {
87+
String sslHostNameVerifier = kafkaConfig.get(MarkLogicSinkConfig.SSL_HOST_VERIFIER);
88+
if ("ANY".equals(sslHostNameVerifier))
89+
clientConfig.setSslHostnameVerifier(DatabaseClientFactory.SSLHostnameVerifier.ANY);
90+
else if ("COMMON".equals(sslHostNameVerifier))
91+
clientConfig.setSslHostnameVerifier(DatabaseClientFactory.SSLHostnameVerifier.COMMON);
92+
else if ("STRICT".equals(sslHostNameVerifier))
93+
clientConfig.setSslHostnameVerifier(DatabaseClientFactory.SSLHostnameVerifier.STRICT);
94+
else
95+
clientConfig.setSslHostnameVerifier(DatabaseClientFactory.SSLHostnameVerifier.ANY);
96+
return clientConfig;
97+
}
98+
99+
protected DatabaseClientConfig configureCustomSslConnection(DatabaseClientConfig clientConfig, Map<String, String> kafkaConfig) {
100+
String ssl = kafkaConfig.get(MarkLogicSinkConfig.SSL);
101+
String tlsVersion = kafkaConfig.get(MarkLogicSinkConfig.TLS_VERSION);
102+
String sslMutualAuth = kafkaConfig.get(MarkLogicSinkConfig.SSL_MUTUAL_AUTH);
103+
SSLContext sslContext = null;
104+
String securityContextType = kafkaConfig.get(MarkLogicSinkConfig.CONNECTION_SECURITY_CONTEXT_TYPE).toUpperCase();
105+
clientConfig.setSecurityContextType(SecurityContextType.valueOf(securityContextType));
106+
107+
if ("BASIC".equals(securityContextType) ||
108+
"DIGEST".equals(securityContextType)
109+
) {
110+
if (ssl != null && Boolean.parseBoolean(ssl)) {
111+
if (sslMutualAuth != null && Boolean.parseBoolean(sslMutualAuth)) {
112+
/*2 way ssl changes*/
113+
KeyStore clientKeyStore = null;
114+
try {
115+
clientKeyStore = KeyStore.getInstance("PKCS12");
116+
} catch (KeyStoreException e) {
117+
118+
throw new RuntimeException("Unable to get default SSLContext: " + e.getMessage(), e);
119+
}
120+
TrustManager[] trust = new TrustManager[] { new SimpleX509TrustManager()};
121+
122+
try (InputStream keystoreInputStream = new FileInputStream(clientConfig.getCertFile())) {
123+
clientKeyStore.load(keystoreInputStream, clientConfig.getCertPassword().toCharArray());
124+
} catch (Exception e) {
125+
throw new RuntimeException("Unable to configure custom SSL connection: " + e.getMessage(), e);
126+
}
127+
KeyManagerFactory keyManagerFactory = null;
128+
try {
129+
keyManagerFactory = KeyManagerFactory.getInstance(TrustManagerFactory.getDefaultAlgorithm());
130+
} catch (Exception e) {
131+
132+
throw new RuntimeException("Unable to configure custom SSL connection: " + e.getMessage(), e);
133+
}
134+
try {
135+
keyManagerFactory.init(clientKeyStore, clientConfig.getCertPassword().toCharArray());
136+
} catch (Exception e) {
137+
138+
throw new RuntimeException("Unable to configure custom SSL connection: " + e.getMessage(), e);
139+
}
140+
KeyManager[] key = keyManagerFactory.getKeyManagers();
141+
try {
142+
if (tlsVersion != null && tlsVersion.trim().length() > 0 ) {
143+
sslContext = SSLContext.getInstance(tlsVersion);
144+
}
145+
else {
146+
sslContext = SSLContext.getInstance("TLSv1.2");
147+
}
148+
} catch (NoSuchAlgorithmException e) {
149+
150+
throw new RuntimeException("Unable to configure custom SSL connection:" + e.getMessage(), e);
151+
}
152+
try {
153+
sslContext.init(key, trust, null);
154+
} catch (KeyManagementException e) {
155+
throw new RuntimeException("Unable to configure custom SSL connection:" + e.getMessage(), e);
156+
}
157+
clientConfig.setSslContext(sslContext);
158+
}
159+
else {/*1wayssl*/
160+
TrustManager[] trust = new TrustManager[] { new SimpleX509TrustManager()};
161+
try {
162+
if (tlsVersion != null && tlsVersion.trim().length() > 0 ) {
163+
sslContext = SSLContext.getInstance(tlsVersion);
164+
}
165+
else {
166+
sslContext = SSLContext.getInstance("TLSv1.2");
167+
}
168+
} catch (NoSuchAlgorithmException e) {
169+
throw new RuntimeException("Unable to configure custom SSL connection: " + e.getMessage(), e);
170+
}
171+
try {
172+
sslContext.init(null, trust, null);
173+
}catch (KeyManagementException e) {
174+
throw new RuntimeException("Unable to configure custom SSL connection:" + e.getMessage(), e);
175+
}
176+
clientConfig.setSslContext(sslContext);
177+
}
178+
} /* End of if ssl */
179+
}
180+
return clientConfig;
64181
}
65182
}

‎src/main/java/com/marklogic/kafka/connect/sink/MarkLogicSinkConfig.java

+11-1
Original file line numberDiff line numberDiff line change
@@ -42,6 +42,11 @@ public class MarkLogicSinkConfig extends AbstractConfig {
4242
public static final String DOCUMENT_URI_PREFIX = "ml.document.uriPrefix";
4343
public static final String DOCUMENT_URI_SUFFIX = "ml.document.uriSuffix";
4444

45+
public static final String SSL = "ml.connection.enableCustomSsl";
46+
public static final String TLS_VERSION = "ml.connection.customSsl.tlsVersion";
47+
public static final String SSL_HOST_VERIFIER = "ml.connection.customSsl.hostNameVerifier";
48+
public static final String SSL_MUTUAL_AUTH = "ml.connection.customSsl.mutualAuth";
49+
4550
public static ConfigDef CONFIG_DEF = new ConfigDef()
4651
.define(CONNECTION_HOST, Type.STRING, Importance.HIGH, "MarkLogic server hostname")
4752
.define(CONNECTION_PORT, Type.INT, Importance.HIGH, "The REST app server port to connect to")
@@ -68,7 +73,12 @@ public class MarkLogicSinkConfig extends AbstractConfig {
6873
.define(DOCUMENT_MIMETYPE, Type.STRING, Importance.LOW, "Defines the mime type of each document; optional, and typically the format is set instead of the mime type")
6974
.define(DOCUMENT_PERMISSIONS, Type.STRING, Importance.MEDIUM, "String-delimited permissions to add to each document; role1,capability1,role2,capability2,etc")
7075
.define(DOCUMENT_URI_PREFIX, Type.STRING, Importance.MEDIUM, "Prefix to prepend to each generated URI")
71-
.define(DOCUMENT_URI_SUFFIX, Type.STRING, Importance.MEDIUM, "Suffix to append to each generated URI");
76+
.define(DOCUMENT_URI_SUFFIX, Type.STRING, Importance.MEDIUM, "Suffix to append to each generated URI")
77+
.define(SSL, Type.BOOLEAN, Importance.LOW, "Whether SSL connection to the App server - true or false.")
78+
.define(TLS_VERSION, Type.STRING, Importance.LOW, "Version of TLS to connect to MarkLogic SSL enabled App server. Ex. TLSv1.2")
79+
.define(SSL_HOST_VERIFIER, Type.STRING, Importance.LOW, "The strictness of Host Verifier - ANY, COMMON, STRICT")
80+
.define(SSL_MUTUAL_AUTH, Type.BOOLEAN, Importance.LOW, "Mutual Authentication for Basic or Digest : true or false")
81+
;
7282

7383
public MarkLogicSinkConfig(final Map<?, ?> originals) {
7484
super(CONFIG_DEF, originals, false);

‎src/test/java/com/marklogic/kafka/connect/BuildDatabaseClientConfigTest.java

+74-2
Original file line numberDiff line numberDiff line change
@@ -6,9 +6,11 @@
66
import com.marklogic.kafka.connect.sink.MarkLogicSinkConfig;
77
import org.junit.jupiter.api.BeforeEach;
88
import org.junit.jupiter.api.Test;
9+
import com.marklogic.client.DatabaseClientFactory;
910

1011
import java.util.HashMap;
1112
import java.util.Map;
13+
import java.io.File;
1214

1315
import static org.junit.jupiter.api.Assertions.*;
1416

@@ -70,8 +72,78 @@ public void digestAuthenticationAndSimpleSsl() {
7072
assertEquals(SecurityContextType.DIGEST, clientConfig.getSecurityContextType());
7173
assertNotNull(clientConfig.getSslContext());
7274
assertNotNull(clientConfig.getSslHostnameVerifier());
73-
assertNull(clientConfig.getTrustManager(), "If DatabaseClientFactory is given a null TrustManager, it will " +
74-
"default to the JVM's cacerts file, which is a reasonable default approach");
75+
assertNotNull(clientConfig.getTrustManager());
76+
}
77+
78+
@Test
79+
public void basictAuthenticationAndSimpleSsl() {
80+
config.put(MarkLogicSinkConfig.CONNECTION_SECURITY_CONTEXT_TYPE, "basic");
81+
config.put(MarkLogicSinkConfig.CONNECTION_SIMPLE_SSL, "true");
82+
83+
DatabaseClientConfig clientConfig = builder.buildDatabaseClientConfig(config);
84+
assertEquals(SecurityContextType.BASIC, clientConfig.getSecurityContextType());
85+
assertNotNull(clientConfig.getSslContext());
86+
assertNotNull(clientConfig.getSslHostnameVerifier());
87+
assertNotNull(clientConfig.getTrustManager());
88+
}
89+
90+
@Test
91+
public void basicAuthenticationAndMutualSSL() {
92+
File file = new File("src/test/resources/srportal.p12");
93+
String absolutePath = file.getAbsolutePath();
94+
config.put(MarkLogicSinkConfig.CONNECTION_SECURITY_CONTEXT_TYPE, "basic");
95+
config.put(MarkLogicSinkConfig.CONNECTION_SIMPLE_SSL, "false");
96+
config.put(MarkLogicSinkConfig.SSL, "true");
97+
config.put(MarkLogicSinkConfig.TLS_VERSION, "TLSv1.2");
98+
config.put(MarkLogicSinkConfig.SSL_HOST_VERIFIER, "STRICT");
99+
config.put(MarkLogicSinkConfig.SSL_MUTUAL_AUTH, "true");
100+
config.put(MarkLogicSinkConfig.CONNECTION_CERT_FILE, absolutePath);
101+
config.put(MarkLogicSinkConfig.CONNECTION_CERT_PASSWORD, "abc");
102+
103+
DatabaseClientConfig clientConfig = builder.buildDatabaseClientConfig(config);
104+
assertEquals(SecurityContextType.BASIC, clientConfig.getSecurityContextType());
105+
assertNotNull(clientConfig.getSslContext());
106+
assertEquals(DatabaseClientFactory.SSLHostnameVerifier.STRICT, clientConfig.getSslHostnameVerifier());
107+
assertNotNull(clientConfig.getTrustManager());
108+
}
109+
110+
@Test
111+
public void basicAuthenticationAndMutualSSLWithInvalidHost() {
112+
File file = new File("src/test/resources/srportal.p12");
113+
String absolutePath = file.getAbsolutePath();
114+
config.put(MarkLogicSinkConfig.CONNECTION_SECURITY_CONTEXT_TYPE, "basic");
115+
config.put(MarkLogicSinkConfig.CONNECTION_SIMPLE_SSL, "false");
116+
config.put(MarkLogicSinkConfig.SSL, "true");
117+
config.put(MarkLogicSinkConfig.TLS_VERSION, "TLSv1.2");
118+
config.put(MarkLogicSinkConfig.SSL_HOST_VERIFIER, "SOMETHING");
119+
config.put(MarkLogicSinkConfig.SSL_MUTUAL_AUTH, "true");
120+
config.put(MarkLogicSinkConfig.CONNECTION_CERT_FILE, absolutePath);
121+
config.put(MarkLogicSinkConfig.CONNECTION_CERT_PASSWORD, "abc");
122+
123+
DatabaseClientConfig clientConfig = builder.buildDatabaseClientConfig(config);
124+
assertEquals(SecurityContextType.BASIC, clientConfig.getSecurityContextType());
125+
assertNotNull(clientConfig.getSslContext());
126+
assertEquals(DatabaseClientFactory.SSLHostnameVerifier.ANY, clientConfig.getSslHostnameVerifier());
127+
System.out.println(clientConfig.getSslHostnameVerifier());
128+
assertNotNull(clientConfig.getTrustManager());
129+
}
130+
131+
132+
@Test
133+
public void digestAuthenticationAnd1WaySSL() {
134+
135+
config.put(MarkLogicSinkConfig.CONNECTION_SECURITY_CONTEXT_TYPE, "digest");
136+
config.put(MarkLogicSinkConfig.CONNECTION_SIMPLE_SSL, "false");
137+
config.put(MarkLogicSinkConfig.SSL, "true");
138+
config.put(MarkLogicSinkConfig.TLS_VERSION, "TLSv1.2");
139+
config.put(MarkLogicSinkConfig.SSL_HOST_VERIFIER, "STRICT");
140+
config.put(MarkLogicSinkConfig.SSL_MUTUAL_AUTH, "false");
141+
142+
DatabaseClientConfig clientConfig = builder.buildDatabaseClientConfig(config);
143+
assertEquals(SecurityContextType.DIGEST, clientConfig.getSecurityContextType());
144+
assertNotNull(clientConfig.getSslContext());
145+
assertNotNull(clientConfig.getSslHostnameVerifier());
146+
assertNotNull(clientConfig.getTrustManager());
75147
}
76148

77149
@Test

‎src/test/java/com/marklogic/kafka/connect/sink/ConvertSinkRecordTest.java

+6-3
Original file line numberDiff line numberDiff line change
@@ -6,9 +6,12 @@
66
import com.marklogic.client.io.Format;
77
import com.marklogic.client.io.StringHandle;
88
import org.apache.kafka.connect.sink.SinkRecord;
9+
910
import org.junit.jupiter.api.Test;
1011

12+
1113
import java.util.*;
14+
import java.io.IOException;
1215

1316
import static org.junit.jupiter.api.Assertions.assertEquals;
1417
import static org.junit.jupiter.api.Assertions.assertTrue;
@@ -22,7 +25,7 @@ public class ConvertSinkRecordTest {
2225
MarkLogicSinkTask markLogicSinkTask = new MarkLogicSinkTask();
2326

2427
@Test
25-
public void allPropertiesSet() {
28+
public void allPropertiesSet() throws IOException {
2629
Map<String, String> config = new HashMap<>();
2730
config.put("ml.document.collections", "one,two");
2831
config.put("ml.document.format", "json");
@@ -54,7 +57,7 @@ public void allPropertiesSet() {
5457
}
5558

5659
@Test
57-
public void noPropertiesSet() {
60+
public void noPropertiesSet() throws IOException {
5861
converter = new DefaultSinkRecordConverter(new HashMap<>());
5962
converter.getDocumentWriteOperationBuilder().withContentIdExtractor(content -> "12345");
6063

@@ -68,7 +71,7 @@ public void noPropertiesSet() {
6871
}
6972

7073
@Test
71-
public void binaryContent() {
74+
public void binaryContent() throws IOException{
7275
converter = new DefaultSinkRecordConverter(new HashMap<>());
7376

7477
DocumentWriteOperation op = converter.convert(newSinkRecord("hello world".getBytes()));

‎src/test/resources/srportal.p12

2.34 KB
Binary file not shown.

0 commit comments

Comments
 (0)