Skip to content

Commit 9c1971d

Browse files
authored
[ISSUE apache#5040] Support gtid mode for sync data with mysql (apache#5041)
* [ISSUE apache#5040] Support gtid mode for sync data with mysql * fix conflicts with master * fix checkstyle error
1 parent 9d93400 commit 9c1971d

File tree

19 files changed

+511
-170
lines changed

19 files changed

+511
-170
lines changed

eventmesh-admin-server/conf/eventmesh.sql

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -71,8 +71,11 @@ CREATE TABLE IF NOT EXISTS `event_mesh_job_info` (
7171
CREATE TABLE IF NOT EXISTS `event_mesh_mysql_position` (
7272
`id` int unsigned NOT NULL AUTO_INCREMENT,
7373
`jobID` int unsigned NOT NULL,
74+
`serverUUID` varchar(50) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci DEFAULT NULL,
7475
`address` varchar(50) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NOT NULL,
7576
`position` bigint DEFAULT NULL,
77+
`gtid` varchar(50) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci DEFAULT NULL,
78+
`currentGtid` varchar(50) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci DEFAULT NULL,
7679
`timestamp` bigint DEFAULT NULL,
7780
`journalName` varchar(50) COLLATE utf8mb4_general_ci DEFAULT NULL,
7881
`createTime` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP,

eventmesh-admin-server/conf/mapper/EventMeshMysqlPositionMapper.xml

Lines changed: 16 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -16,24 +16,28 @@
1616
limitations under the License.
1717
-->
1818
<!DOCTYPE mapper
19-
PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN"
20-
"http://mybatis.org/dtd/mybatis-3-mapper.dtd">
19+
PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN"
20+
"http://mybatis.org/dtd/mybatis-3-mapper.dtd">
2121
<mapper namespace="org.apache.eventmesh.admin.server.web.db.mapper.EventMeshMysqlPositionMapper">
2222

2323
<resultMap id="BaseResultMap" type="org.apache.eventmesh.admin.server.web.db.entity.EventMeshMysqlPosition">
24-
<id property="id" column="id" jdbcType="INTEGER"/>
25-
<result property="jobID" column="jobID" jdbcType="INTEGER"/>
26-
<result property="address" column="address" jdbcType="VARCHAR"/>
27-
<result property="position" column="position" jdbcType="BIGINT"/>
28-
<result property="timestamp" column="timestamp" jdbcType="BIGINT"/>
29-
<result property="journalName" column="journalName" jdbcType="VARCHAR"/>
30-
<result property="createTime" column="createTime" jdbcType="TIMESTAMP"/>
31-
<result property="updateTime" column="updateTime" jdbcType="TIMESTAMP"/>
24+
<id property="id" column="id" jdbcType="INTEGER"/>
25+
<result property="jobID" column="jobID" jdbcType="INTEGER"/>
26+
<result property="serverUUID" column="serverUUID" jdbcType="VARCHAR"/>
27+
<result property="address" column="address" jdbcType="VARCHAR"/>
28+
<result property="position" column="position" jdbcType="BIGINT"/>
29+
<result property="gtid" column="gtid" jdbcType="VARCHAR"/>
30+
<result property="currentGtid" column="currentGtid" jdbcType="VARCHAR"/>
31+
<result property="timestamp" column="timestamp" jdbcType="BIGINT"/>
32+
<result property="journalName" column="journalName" jdbcType="VARCHAR"/>
33+
<result property="createTime" column="createTime" jdbcType="TIMESTAMP"/>
34+
<result property="updateTime" column="updateTime" jdbcType="TIMESTAMP"/>
3235
</resultMap>
3336

3437
<sql id="Base_Column_List">
35-
id,jobID,address,
36-
position,timestamp,journalName,
38+
id
39+
,jobID,serverUUID,address,
40+
position,gtid,currentGtid,timestamp,journalName,
3741
createTime,updateTime
3842
</sql>
3943
</mapper>

eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/db/entity/EventMeshMysqlPosition.java

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -38,10 +38,16 @@ public class EventMeshMysqlPosition implements Serializable {
3838

3939
private Integer jobID;
4040

41+
private String serverUUID;
42+
4143
private String address;
4244

4345
private Long position;
4446

47+
private String gtid;
48+
49+
private String currentGtid;
50+
4551
private Long timestamp;
4652

4753
private String journalName;

eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/service/position/impl/MysqlPositionHandler.java

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -115,9 +115,12 @@ public boolean handler(ReportPositionRequest request, Metadata metadata) {
115115
CanalRecordOffset offset = (CanalRecordOffset) recordPosition.getRecordOffset();
116116
if (offset != null) {
117117
position.setPosition(offset.getOffset());
118+
position.setGtid(offset.getGtid());
119+
position.setCurrentGtid(offset.getCurrentGtid());
118120
}
119121
CanalRecordPartition partition = (CanalRecordPartition) recordPosition.getRecordPartition();
120122
if (partition != null) {
123+
position.setServerUUID(partition.getServerUUID());
121124
position.setTimestamp(partition.getTimeStamp());
122125
position.setJournalName(partition.getJournalName());
123126
}
@@ -148,13 +151,16 @@ public List<RecordPosition> handler(FetchPositionRequest request, Metadata metad
148151
request.getJobID()));
149152
List<RecordPosition> recordPositionList = new ArrayList<>();
150153
for (EventMeshMysqlPosition position : positionList) {
151-
RecordPosition recordPosition = new RecordPosition();
152154
CanalRecordPartition partition = new CanalRecordPartition();
153155
partition.setTimeStamp(position.getTimestamp());
154156
partition.setJournalName(position.getJournalName());
157+
partition.setServerUUID(position.getServerUUID());
158+
RecordPosition recordPosition = new RecordPosition();
155159
recordPosition.setRecordPartition(partition);
156160
CanalRecordOffset offset = new CanalRecordOffset();
157161
offset.setOffset(position.getPosition());
162+
offset.setGtid(position.getGtid());
163+
offset.setCurrentGtid(position.getCurrentGtid());
158164
recordPosition.setRecordOffset(offset);
159165
recordPositionList.add(recordPosition);
160166
}

eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/rdb/canal/CanalSinkConfig.java

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,8 @@ public class CanalSinkConfig extends SinkConfig {
3939
// sync mode: field/row
4040
private SyncMode syncMode;
4141

42+
private boolean isGTIDMode = true;
43+
4244
// skip sink process exception
4345
private Boolean skipException = false;
4446

eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/rdb/canal/CanalSourceConfig.java

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -45,6 +45,10 @@ public class CanalSourceConfig extends SourceConfig {
4545

4646
private Short clientId;
4747

48+
private String serverUUID;
49+
50+
private boolean isGTIDMode = true;
51+
4852
private Integer batchSize = 10000;
4953

5054
private Long batchTimeout = -1L;

eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/offset/canal/CanalRecordOffset.java

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,11 @@ public class CanalRecordOffset extends RecordOffset {
3030

3131
private Long offset;
3232

33+
// mysql instance gtid range
34+
private String gtid;
35+
36+
private String currentGtid;
37+
3338
@Override
3439
public Class<? extends RecordOffset> getRecordOffsetClass() {
3540
return CanalRecordOffset.class;

eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/offset/canal/CanalRecordPartition.java

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,8 @@
2929
@ToString
3030
public class CanalRecordPartition extends RecordPartition {
3131

32+
private String serverUUID;
33+
3234
private String journalName;
3335

3436
private Long timeStamp;

eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/CanalConnectRecord.java

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -31,8 +31,14 @@
3131
public class CanalConnectRecord {
3232

3333
private String schemaName;
34+
3435
private String tableName;
3536

37+
// mysql instance gtid range
38+
private String gtid;
39+
40+
private String currentGtid;
41+
3642
/**
3743
* The business type of the changed data (I/U/D/C/A/E), consistent with the EventType defined in EntryProtocol in canal.
3844
*/

eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/dialect/AbstractDbDialect.java

Lines changed: 0 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -97,10 +97,6 @@ public SqlTemplate getSqlTemplate() {
9797
return sqlTemplate;
9898
}
9999

100-
public boolean isDRDS() {
101-
return false;
102-
}
103-
104100
public String getShardColumns(String schema, String table) {
105101
return null;
106102
}

0 commit comments

Comments
 (0)