Skip to content

Commit a847660

Browse files
agappleagapple
committed
fixed issue #1826 , remove kafka transaction send
1 parent 4957083 commit a847660

File tree

7 files changed

+173
-208
lines changed

7 files changed

+173
-208
lines changed

‎deployer/src/main/java/com/alibaba/otter/canal/deployer/CanalConstants.java‎

Lines changed: 43 additions & 44 deletions
Original file line numberDiff line numberDiff line change
@@ -10,56 +10,55 @@
1010
*/
1111
public class CanalConstants {
1212

13-
public static final String MDC_DESTINATION = "destination";
14-
public static final String ROOT = "canal";
15-
public static final String CANAL_ID = ROOT + "." + "id";
16-
public static final String CANAL_IP = ROOT + "." + "ip";
17-
public static final String CANAL_PORT = ROOT + "." + "port";
18-
public static final String CANAL_METRICS_PULL_PORT = ROOT + "." + "metrics.pull.port";
19-
public static final String CANAL_ZKSERVERS = ROOT + "." + "zkServers";
20-
public static final String CANAL_WITHOUT_NETTY = ROOT + "." + "withoutNetty";
13+
public static final String MDC_DESTINATION = "destination";
14+
public static final String ROOT = "canal";
15+
public static final String CANAL_ID = ROOT + "." + "id";
16+
public static final String CANAL_IP = ROOT + "." + "ip";
17+
public static final String CANAL_PORT = ROOT + "." + "port";
18+
public static final String CANAL_METRICS_PULL_PORT = ROOT + "." + "metrics.pull.port";
19+
public static final String CANAL_ZKSERVERS = ROOT + "." + "zkServers";
20+
public static final String CANAL_WITHOUT_NETTY = ROOT + "." + "withoutNetty";
2121

22-
public static final String CANAL_DESTINATIONS = ROOT + "." + "destinations";
23-
public static final String CANAL_AUTO_SCAN = ROOT + "." + "auto.scan";
24-
public static final String CANAL_AUTO_SCAN_INTERVAL = ROOT + "." + "auto.scan.interval";
25-
public static final String CANAL_CONF_DIR = ROOT + "." + "conf.dir";
26-
public static final String CANAL_SERVER_MODE = ROOT + "." + "serverMode";
22+
public static final String CANAL_DESTINATIONS = ROOT + "." + "destinations";
23+
public static final String CANAL_AUTO_SCAN = ROOT + "." + "auto.scan";
24+
public static final String CANAL_AUTO_SCAN_INTERVAL = ROOT + "." + "auto.scan.interval";
25+
public static final String CANAL_CONF_DIR = ROOT + "." + "conf.dir";
26+
public static final String CANAL_SERVER_MODE = ROOT + "." + "serverMode";
2727

28-
public static final String CANAL_DESTINATION_SPLIT = ",";
29-
public static final String GLOBAL_NAME = "global";
28+
public static final String CANAL_DESTINATION_SPLIT = ",";
29+
public static final String GLOBAL_NAME = "global";
3030

31-
public static final String INSTANCE_MODE_TEMPLATE = ROOT + "." + "instance.{0}.mode";
32-
public static final String INSTANCE_LAZY_TEMPLATE = ROOT + "." + "instance.{0}.lazy";
33-
public static final String INSTANCE_MANAGER_ADDRESS_TEMPLATE = ROOT + "." + "instance.{0}.manager.address";
34-
public static final String INSTANCE_SPRING_XML_TEMPLATE = ROOT + "." + "instance.{0}.spring.xml";
31+
public static final String INSTANCE_MODE_TEMPLATE = ROOT + "." + "instance.{0}.mode";
32+
public static final String INSTANCE_LAZY_TEMPLATE = ROOT + "." + "instance.{0}.lazy";
33+
public static final String INSTANCE_MANAGER_ADDRESS_TEMPLATE = ROOT + "." + "instance.{0}.manager.address";
34+
public static final String INSTANCE_SPRING_XML_TEMPLATE = ROOT + "." + "instance.{0}.spring.xml";
3535

36-
public static final String CANAL_DESTINATION_PROPERTY = ROOT + ".instance.destination";
36+
public static final String CANAL_DESTINATION_PROPERTY = ROOT + ".instance.destination";
3737

38-
public static final String CANAL_SOCKETCHANNEL = ROOT + "." + "socketChannel";
38+
public static final String CANAL_SOCKETCHANNEL = ROOT + "." + "socketChannel";
3939

40-
public static final String CANAL_MQ_SERVERS = ROOT + "." + "mq.servers";
41-
public static final String CANAL_MQ_RETRIES = ROOT + "." + "mq.retries";
42-
public static final String CANAL_MQ_BATCHSIZE = ROOT + "." + "mq.batchSize";
43-
public static final String CANAL_MQ_LINGERMS = ROOT + "." + "mq.lingerMs";
44-
public static final String CANAL_MQ_MAXREQUESTSIZE = ROOT + "." + "mq.maxRequestSize";
45-
public static final String CANAL_MQ_BUFFERMEMORY = ROOT + "." + "mq.bufferMemory";
46-
public static final String CANAL_MQ_CANALBATCHSIZE = ROOT + "." + "mq.canalBatchSize";
47-
public static final String CANAL_MQ_CANALGETTIMEOUT = ROOT + "." + "mq.canalGetTimeout";
48-
public static final String CANAL_MQ_FLATMESSAGE = ROOT + "." + "mq.flatMessage";
49-
public static final String CANAL_MQ_COMPRESSION_TYPE = ROOT + "." + "mq.compressionType";
50-
public static final String CANAL_MQ_ACKS = ROOT + "." + "mq.acks";
51-
public static final String CANAL_MQ_TRANSACTION = ROOT + "." + "mq.transaction";
52-
public static final String CANAL_MQ_PRODUCERGROUP = ROOT + "." + "mq.producerGroup";
53-
public static final String CANAL_ALIYUN_ACCESSKEY = ROOT + "." + "aliyun.accessKey";
54-
public static final String CANAL_ALIYUN_SECRETKEY = ROOT + "." + "aliyun.secretKey";
55-
public static final String CANAL_MQ_PROPERTIES = ROOT + "." + "mq.properties";
56-
public static final String CANAL_MQ_ENABLE_MESSAGE_TRACE = ROOT + "." + "mq.enableMessageTrace";
57-
public static final String CANAL_MQ_ACCESS_CHANNEL = ROOT + "." + "mq.accessChannel";
58-
public static final String CANAL_MQ_CUSTOMIZED_TRACE_TOPIC = ROOT + "." + "mq.customizedTraceTopic";
59-
public static final String CANAL_MQ_NAMESPACE = ROOT + "." + "mq.namespace";
60-
public static final String CANAL_MQ_KAFKA_KERBEROS_ENABLE = ROOT + "." + "mq.kafka.kerberos.enable";
61-
public static final String CANAL_MQ_KAFKA_KERBEROS_KRB5FILEPATH = ROOT + "." + "mq.kafka.kerberos.krb5FilePath";
62-
public static final String CANAL_MQ_KAFKA_KERBEROS_JAASFILEPATH = ROOT + "." + "mq.kafka.kerberos.jaasFilePath";
40+
public static final String CANAL_MQ_SERVERS = ROOT + "." + "mq.servers";
41+
public static final String CANAL_MQ_RETRIES = ROOT + "." + "mq.retries";
42+
public static final String CANAL_MQ_BATCHSIZE = ROOT + "." + "mq.batchSize";
43+
public static final String CANAL_MQ_LINGERMS = ROOT + "." + "mq.lingerMs";
44+
public static final String CANAL_MQ_MAXREQUESTSIZE = ROOT + "." + "mq.maxRequestSize";
45+
public static final String CANAL_MQ_BUFFERMEMORY = ROOT + "." + "mq.bufferMemory";
46+
public static final String CANAL_MQ_CANALBATCHSIZE = ROOT + "." + "mq.canalBatchSize";
47+
public static final String CANAL_MQ_CANALGETTIMEOUT = ROOT + "." + "mq.canalGetTimeout";
48+
public static final String CANAL_MQ_FLATMESSAGE = ROOT + "." + "mq.flatMessage";
49+
public static final String CANAL_MQ_COMPRESSION_TYPE = ROOT + "." + "mq.compressionType";
50+
public static final String CANAL_MQ_ACKS = ROOT + "." + "mq.acks";
51+
public static final String CANAL_MQ_PRODUCERGROUP = ROOT + "." + "mq.producerGroup";
52+
public static final String CANAL_ALIYUN_ACCESSKEY = ROOT + "." + "aliyun.accessKey";
53+
public static final String CANAL_ALIYUN_SECRETKEY = ROOT + "." + "aliyun.secretKey";
54+
public static final String CANAL_MQ_PROPERTIES = ROOT + "." + "mq.properties";
55+
public static final String CANAL_MQ_ENABLE_MESSAGE_TRACE = ROOT + "." + "mq.enableMessageTrace";
56+
public static final String CANAL_MQ_ACCESS_CHANNEL = ROOT + "." + "mq.accessChannel";
57+
public static final String CANAL_MQ_CUSTOMIZED_TRACE_TOPIC = ROOT + "." + "mq.customizedTraceTopic";
58+
public static final String CANAL_MQ_NAMESPACE = ROOT + "." + "mq.namespace";
59+
public static final String CANAL_MQ_KAFKA_KERBEROS_ENABLE = ROOT + "." + "mq.kafka.kerberos.enable";
60+
public static final String CANAL_MQ_KAFKA_KERBEROS_KRB5FILEPATH = ROOT + "." + "mq.kafka.kerberos.krb5FilePath";
61+
public static final String CANAL_MQ_KAFKA_KERBEROS_JAASFILEPATH = ROOT + "." + "mq.kafka.kerberos.jaasFilePath";
6362

6463
public static String getInstanceModeKey(String destination) {
6564
return MessageFormat.format(INSTANCE_MODE_TEMPLATE, destination);

‎deployer/src/main/java/com/alibaba/otter/canal/deployer/CanalStater.java‎

Lines changed: 22 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,15 @@
11
package com.alibaba.otter.canal.deployer;
22

3+
import java.io.File;
4+
import java.io.FileFilter;
5+
import java.util.Arrays;
6+
import java.util.List;
7+
import java.util.Properties;
8+
9+
import org.apache.commons.lang.StringUtils;
10+
import org.slf4j.Logger;
11+
import org.slf4j.LoggerFactory;
12+
313
import com.alibaba.otter.canal.common.MQProperties;
414
import com.alibaba.otter.canal.kafka.CanalKafkaProducer;
515
import com.alibaba.otter.canal.rocketmq.CanalRocketMQProducer;
@@ -8,15 +18,6 @@
818
import com.google.common.base.Function;
919
import com.google.common.base.Joiner;
1020
import com.google.common.collect.Lists;
11-
import org.apache.commons.lang.StringUtils;
12-
import org.slf4j.Logger;
13-
import org.slf4j.LoggerFactory;
14-
15-
import java.io.File;
16-
import java.io.FileFilter;
17-
import java.util.Arrays;
18-
import java.util.List;
19-
import java.util.Properties;
2021

2122
/**
2223
* Canal server 启动类
@@ -64,8 +65,8 @@ synchronized void start(Properties properties) throws Throwable {
6465

6566
public boolean accept(File pathname) {
6667
String filename = pathname.getName();
67-
return pathname.isDirectory() && !"spring".equalsIgnoreCase(filename) &&
68-
!"metrics".equalsIgnoreCase(filename);
68+
return pathname.isDirectory() && !"spring".equalsIgnoreCase(filename)
69+
&& !"metrics".equalsIgnoreCase(filename);
6970
}
7071
});
7172
if (instanceDirs != null && instanceDirs.length > 0) {
@@ -196,17 +197,14 @@ private static MQProperties buildMQProperties(Properties properties) {
196197
if (!StringUtils.isEmpty(aliyunSecretKey)) {
197198
mqProperties.setAliyunSecretKey(aliyunSecretKey);
198199
}
199-
String transaction = CanalController.getProperty(properties, CanalConstants.CANAL_MQ_TRANSACTION);
200-
if (!StringUtils.isEmpty(transaction)) {
201-
mqProperties.setTransaction(Boolean.valueOf(transaction));
202-
}
203200

204201
String producerGroup = CanalController.getProperty(properties, CanalConstants.CANAL_MQ_PRODUCERGROUP);
205202
if (!StringUtils.isEmpty(producerGroup)) {
206203
mqProperties.setProducerGroup(producerGroup);
207204
}
208205

209-
String enableMessageTrace = CanalController.getProperty(properties, CanalConstants.CANAL_MQ_ENABLE_MESSAGE_TRACE);
206+
String enableMessageTrace = CanalController.getProperty(properties,
207+
CanalConstants.CANAL_MQ_ENABLE_MESSAGE_TRACE);
210208
if (!StringUtils.isEmpty(enableMessageTrace)) {
211209
mqProperties.setEnableMessageTrace(Boolean.valueOf(enableMessageTrace));
212210
}
@@ -216,7 +214,8 @@ private static MQProperties buildMQProperties(Properties properties) {
216214
mqProperties.setAccessChannel(accessChannel);
217215
}
218216

219-
String customizedTraceTopic = CanalController.getProperty(properties, CanalConstants.CANAL_MQ_CUSTOMIZED_TRACE_TOPIC);
217+
String customizedTraceTopic = CanalController.getProperty(properties,
218+
CanalConstants.CANAL_MQ_CUSTOMIZED_TRACE_TOPIC);
220219
if (!StringUtils.isEmpty(customizedTraceTopic)) {
221220
mqProperties.setCustomizedTraceTopic(customizedTraceTopic);
222221
}
@@ -226,17 +225,20 @@ private static MQProperties buildMQProperties(Properties properties) {
226225
mqProperties.setNamespace(namespace);
227226
}
228227

229-
String kafkaKerberosEnable = CanalController.getProperty(properties, CanalConstants.CANAL_MQ_KAFKA_KERBEROS_ENABLE);
228+
String kafkaKerberosEnable = CanalController.getProperty(properties,
229+
CanalConstants.CANAL_MQ_KAFKA_KERBEROS_ENABLE);
230230
if (!StringUtils.isEmpty(kafkaKerberosEnable)) {
231231
mqProperties.setKerberosEnable(Boolean.valueOf(kafkaKerberosEnable));
232232
}
233233

234-
String kafkaKerberosKrb5Filepath = CanalController.getProperty(properties, CanalConstants.CANAL_MQ_KAFKA_KERBEROS_KRB5FILEPATH);
234+
String kafkaKerberosKrb5Filepath = CanalController.getProperty(properties,
235+
CanalConstants.CANAL_MQ_KAFKA_KERBEROS_KRB5FILEPATH);
235236
if (!StringUtils.isEmpty(kafkaKerberosKrb5Filepath)) {
236237
mqProperties.setKerberosKrb5FilePath(kafkaKerberosKrb5Filepath);
237238
}
238239

239-
String kafkaKerberosJaasFilepath = CanalController.getProperty(properties, CanalConstants.CANAL_MQ_KAFKA_KERBEROS_JAASFILEPATH);
240+
String kafkaKerberosJaasFilepath = CanalController.getProperty(properties,
241+
CanalConstants.CANAL_MQ_KAFKA_KERBEROS_JAASFILEPATH);
240242
if (!StringUtils.isEmpty(kafkaKerberosJaasFilepath)) {
241243
mqProperties.setKerberosJaasFilePath(kafkaKerberosJaasFilepath);
242244
}

‎deployer/src/main/resources/canal.properties‎

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -115,8 +115,6 @@ canal.mq.canalGetTimeout = 100
115115
canal.mq.flatMessage = true
116116
canal.mq.compressionType = none
117117
canal.mq.acks = all
118-
# use transaction for kafka flatMessage batch produce
119-
canal.mq.transaction = true
120118
#canal.mq.properties. =
121119
canal.mq.producerGroup = test
122120
# Set this value to "cloud", if you want open message trace feature in aliyun.

‎server/src/main/java/com/alibaba/otter/canal/common/MQProperties.java‎

Lines changed: 15 additions & 39 deletions
Original file line numberDiff line numberDiff line change
@@ -25,15 +25,14 @@ public class MQProperties {
2525
private String acks = "all";
2626
private String aliyunAccessKey = "";
2727
private String aliyunSecretKey = "";
28-
private boolean transaction = false; // 是否开启事务
2928
private Properties properties = new Properties();
3029
private boolean enableMessageTrace = false;
3130
private String accessChannel = null;
3231
private String customizedTraceTopic = null;
3332
private String namespace = "";
34-
private boolean kerberosEnable = false; //kafka集群是否启动Kerberos认证
35-
private String kerberosKrb5FilePath = ""; //启动Kerberos认证时配置为krb5.conf文件的路径
36-
private String kerberosJaasFilePath = ""; //启动Kerberos认证时配置为jaas.conf文件的路径
33+
private boolean kerberosEnable = false; // kafka集群是否启动Kerberos认证
34+
private String kerberosKrb5FilePath = ""; // 启动Kerberos认证时配置为krb5.conf文件的路径
35+
private String kerberosJaasFilePath = ""; // 启动Kerberos认证时配置为jaas.conf文件的路径
3736

3837
public static class CanalDestination {
3938

@@ -213,14 +212,6 @@ public void setMaxRequestSize(int maxRequestSize) {
213212
this.maxRequestSize = maxRequestSize;
214213
}
215214

216-
public boolean getTransaction() {
217-
return transaction;
218-
}
219-
220-
public void setTransaction(boolean transaction) {
221-
this.transaction = transaction;
222-
}
223-
224215
public Properties getProperties() {
225216
return properties;
226217
}
@@ -285,32 +276,17 @@ public void setKerberosJaasFilePath(String kerberosJaasFilePath) {
285276
this.kerberosJaasFilePath = kerberosJaasFilePath;
286277
}
287278

288-
@Override public String toString() {
289-
return "MQProperties{" +
290-
"servers='" + servers + '\'' +
291-
", retries=" + retries +
292-
", batchSize=" + batchSize +
293-
", lingerMs=" + lingerMs +
294-
", maxRequestSize=" + maxRequestSize +
295-
", bufferMemory=" + bufferMemory +
296-
", filterTransactionEntry=" + filterTransactionEntry +
297-
", producerGroup='" + producerGroup + '\'' +
298-
", canalBatchSize=" + canalBatchSize +
299-
", canalGetTimeout=" + canalGetTimeout +
300-
", flatMessage=" + flatMessage +
301-
", compressionType='" + compressionType + '\'' +
302-
", acks='" + acks + '\'' +
303-
", aliyunAccessKey='" + aliyunAccessKey + '\'' +
304-
", aliyunSecretKey='" + aliyunSecretKey + '\'' +
305-
", transaction=" + transaction +
306-
", properties=" + properties +
307-
", enableMessageTrace=" + enableMessageTrace +
308-
", accessChannel='" + accessChannel + '\'' +
309-
", customizedTraceTopic='" + customizedTraceTopic + '\'' +
310-
", namespace='" + namespace + '\'' +
311-
", kerberosEnable='" + kerberosEnable + '\'' +
312-
", kerberosKrb5FilePath='" + kerberosKrb5FilePath + '\'' +
313-
", kerberosJaasFilePath='" + kerberosJaasFilePath + '\'' +
314-
'}';
279+
@Override
280+
public String toString() {
281+
return "MQProperties{" + "servers='" + servers + '\'' + ", retries=" + retries + ", batchSize=" + batchSize
282+
+ ", lingerMs=" + lingerMs + ", maxRequestSize=" + maxRequestSize + ", bufferMemory=" + bufferMemory
283+
+ ", filterTransactionEntry=" + filterTransactionEntry + ", producerGroup='" + producerGroup + '\''
284+
+ ", canalBatchSize=" + canalBatchSize + ", canalGetTimeout=" + canalGetTimeout + ", flatMessage="
285+
+ flatMessage + ", compressionType='" + compressionType + '\'' + ", acks='" + acks + '\''
286+
+ ", aliyunAccessKey='" + aliyunAccessKey + '\'' + ", aliyunSecretKey='" + aliyunSecretKey + '\''
287+
+ ", properties=" + properties + ", enableMessageTrace=" + enableMessageTrace + ", accessChannel='"
288+
+ accessChannel + '\'' + ", customizedTraceTopic='" + customizedTraceTopic + '\'' + ", namespace='"
289+
+ namespace + '\'' + ", kerberosEnable='" + kerberosEnable + '\'' + ", kerberosKrb5FilePath='"
290+
+ kerberosKrb5FilePath + '\'' + ", kerberosJaasFilePath='" + kerberosJaasFilePath + '\'' + '}';
315291
}
316292
}

0 commit comments

Comments
 (0)