From 0374ca32cfe9ab1394816afe007fbde310368f07 Mon Sep 17 00:00:00 2001 From: sodaRyCN <757083350@qq.com> Date: Wed, 19 Jun 2024 19:22:42 +0800 Subject: [PATCH 01/15] use conf instead of resource --- build.gradle | 44 ++++++- eventmesh-admin-server/build.gradle | 23 ++-- .../main/resources => conf}/application.yaml | 0 .../eventmesh-admin.properties | 0 .../main/resources => conf}/eventmesh.sql | 0 eventmesh-admin-server/conf/log4j2.xml | 108 ++++++++++++++++++ .../mapper/EventMeshDataSourceMapper.xml | 0 .../mapper/EventMeshJobInfoMapper.xml | 0 .../mapper/EventMeshMysqlPositionMapper.xml | 0 ...EventMeshPositionReporterHistoryMapper.xml | 0 .../EventMeshRuntimeHeartbeatMapper.xml | 0 .../mapper/EventMeshRuntimeHistoryMapper.xml | 0 .../admin/server/ExampleAdminServer.java | 2 +- 13 files changed, 167 insertions(+), 10 deletions(-) rename eventmesh-admin-server/{src/main/resources => conf}/application.yaml (100%) rename eventmesh-admin-server/{src/main/resources => conf}/eventmesh-admin.properties (100%) rename eventmesh-admin-server/{src/main/resources => conf}/eventmesh.sql (100%) create mode 100644 eventmesh-admin-server/conf/log4j2.xml rename eventmesh-admin-server/{src/main/resources => conf}/mapper/EventMeshDataSourceMapper.xml (100%) rename eventmesh-admin-server/{src/main/resources => conf}/mapper/EventMeshJobInfoMapper.xml (100%) rename eventmesh-admin-server/{src/main/resources => conf}/mapper/EventMeshMysqlPositionMapper.xml (100%) rename eventmesh-admin-server/{src/main/resources => conf}/mapper/EventMeshPositionReporterHistoryMapper.xml (100%) rename eventmesh-admin-server/{src/main/resources => conf}/mapper/EventMeshRuntimeHeartbeatMapper.xml (100%) rename eventmesh-admin-server/{src/main/resources => conf}/mapper/EventMeshRuntimeHistoryMapper.xml (100%) diff --git a/build.gradle b/build.gradle index 63e9301a43..0408231fd7 100644 --- a/build.gradle +++ b/build.gradle @@ -210,6 +210,48 @@ tasks.register('dist') { } } +tasks.register('dist-admin') { + subprojects.forEach { subProject -> + dependsOn("${subProject.path}:jar") + } + def includedProjects = + [ + "eventmesh-admin-server", + "eventmesh-common", + "eventmesh-spi", + "eventmesh-registry:eventmesh-registry-api", + "eventmesh-registry:eventmesh-registry-nacos", + "eventmesh-openconnect:eventmesh-openconnect-offsetmgmt-plugin:eventmesh-openconnect-offsetmgmt-api" + ] + doLast { + includedProjects.each { + def subProject = findProject(it) + copy { + from subProject.jar.archivePath + into rootProject.file('dist/apps') + } + copy { + from subProject.configurations.runtimeClasspath + into rootProject.file('dist/lib') + exclude 'eventmesh-*' + } + copy { + from subProject.file('bin') + into rootProject.file('dist/bin') + } + copy { + from subProject.file('conf') + from subProject.sourceSets.main.resources.srcDirs + into rootProject.file('dist/conf') + duplicatesStrategy = DuplicatesStrategy.EXCLUDE + exclude 'META-INF' + } + + } + } + +} + tasks.register('installPlugin') { var pluginProjects = subprojects.findAll { it.file('gradle.properties').exists() @@ -754,8 +796,8 @@ subprojects { dependency "software.amazon.awssdk:s3:2.26.3" dependency "com.github.rholder:guava-retrying:2.0.0" - dependency "org.mybatis.spring.boot:mybatis-spring-boot-starter:2.3.2" dependency "com.alibaba:druid-spring-boot-starter:1.2.23" + dependency "com.baomidou:mybatis-plus-boot-starter:3.5.5" dependency "org.springframework.boot:spring-boot-starter-jetty:2.7.18" dependency "com.baomidou:mybatis-plus:3.5.7" dependency "com.mysql:mysql-connector-j:8.4.0" diff --git a/eventmesh-admin-server/build.gradle b/eventmesh-admin-server/build.gradle index 6f91f48001..6de881725a 100644 --- a/eventmesh-admin-server/build.gradle +++ b/eventmesh-admin-server/build.gradle @@ -31,18 +31,25 @@ dependencies { implementation "io.grpc:grpc-stub" implementation "io.grpc:grpc-netty-shaded" - // https://mvnrepository.com/artifact/com.baomidou/mybatis-plus-boot-starter - implementation 'com.baomidou:mybatis-plus-boot-starter:3.5.7' - implementation "org.reflections:reflections:0.10.2" + // https://mvnrepository.com/artifact/com.baomidou/mybatis-plus-boot-starter + implementation "com.baomidou:mybatis-plus-boot-starter" - // https://mvnrepository.com/artifact/com.alibaba/druid-spring-boot-starter - implementation "com.alibaba:druid-spring-boot-starter" - compileOnly 'com.mysql:mysql-connector-j' - compileOnly 'org.projectlombok:lombok' - annotationProcessor 'org.projectlombok:lombok' + // https://mvnrepository.com/artifact/com.alibaba/druid-spring-boot-starter + implementation "com.alibaba:druid-spring-boot-starter" + implementation 'com.mysql:mysql-connector-j' + compileOnly 'org.projectlombok:lombok' + annotationProcessor 'org.projectlombok:lombok' } configurations.implementation { exclude group: "org.springframework.boot", module: "spring-boot-starter-logging" } +sourceSets { + main { + resources { + srcDirs = ['src/main/resources', 'conf'] + } + } +} + diff --git a/eventmesh-admin-server/src/main/resources/application.yaml b/eventmesh-admin-server/conf/application.yaml similarity index 100% rename from eventmesh-admin-server/src/main/resources/application.yaml rename to eventmesh-admin-server/conf/application.yaml diff --git a/eventmesh-admin-server/src/main/resources/eventmesh-admin.properties b/eventmesh-admin-server/conf/eventmesh-admin.properties similarity index 100% rename from eventmesh-admin-server/src/main/resources/eventmesh-admin.properties rename to eventmesh-admin-server/conf/eventmesh-admin.properties diff --git a/eventmesh-admin-server/src/main/resources/eventmesh.sql b/eventmesh-admin-server/conf/eventmesh.sql similarity index 100% rename from eventmesh-admin-server/src/main/resources/eventmesh.sql rename to eventmesh-admin-server/conf/eventmesh.sql diff --git a/eventmesh-admin-server/conf/log4j2.xml b/eventmesh-admin-server/conf/log4j2.xml new file mode 100644 index 0000000000..6341a0e629 --- /dev/null +++ b/eventmesh-admin-server/conf/log4j2.xml @@ -0,0 +1,108 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/eventmesh-admin-server/src/main/resources/mapper/EventMeshDataSourceMapper.xml b/eventmesh-admin-server/conf/mapper/EventMeshDataSourceMapper.xml similarity index 100% rename from eventmesh-admin-server/src/main/resources/mapper/EventMeshDataSourceMapper.xml rename to eventmesh-admin-server/conf/mapper/EventMeshDataSourceMapper.xml diff --git a/eventmesh-admin-server/src/main/resources/mapper/EventMeshJobInfoMapper.xml b/eventmesh-admin-server/conf/mapper/EventMeshJobInfoMapper.xml similarity index 100% rename from eventmesh-admin-server/src/main/resources/mapper/EventMeshJobInfoMapper.xml rename to eventmesh-admin-server/conf/mapper/EventMeshJobInfoMapper.xml diff --git a/eventmesh-admin-server/src/main/resources/mapper/EventMeshMysqlPositionMapper.xml b/eventmesh-admin-server/conf/mapper/EventMeshMysqlPositionMapper.xml similarity index 100% rename from eventmesh-admin-server/src/main/resources/mapper/EventMeshMysqlPositionMapper.xml rename to eventmesh-admin-server/conf/mapper/EventMeshMysqlPositionMapper.xml diff --git a/eventmesh-admin-server/src/main/resources/mapper/EventMeshPositionReporterHistoryMapper.xml b/eventmesh-admin-server/conf/mapper/EventMeshPositionReporterHistoryMapper.xml similarity index 100% rename from eventmesh-admin-server/src/main/resources/mapper/EventMeshPositionReporterHistoryMapper.xml rename to eventmesh-admin-server/conf/mapper/EventMeshPositionReporterHistoryMapper.xml diff --git a/eventmesh-admin-server/src/main/resources/mapper/EventMeshRuntimeHeartbeatMapper.xml b/eventmesh-admin-server/conf/mapper/EventMeshRuntimeHeartbeatMapper.xml similarity index 100% rename from eventmesh-admin-server/src/main/resources/mapper/EventMeshRuntimeHeartbeatMapper.xml rename to eventmesh-admin-server/conf/mapper/EventMeshRuntimeHeartbeatMapper.xml diff --git a/eventmesh-admin-server/src/main/resources/mapper/EventMeshRuntimeHistoryMapper.xml b/eventmesh-admin-server/conf/mapper/EventMeshRuntimeHistoryMapper.xml similarity index 100% rename from eventmesh-admin-server/src/main/resources/mapper/EventMeshRuntimeHistoryMapper.xml rename to eventmesh-admin-server/conf/mapper/EventMeshRuntimeHistoryMapper.xml diff --git a/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/ExampleAdminServer.java b/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/ExampleAdminServer.java index c6a6e16504..7f5fa22dda 100644 --- a/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/ExampleAdminServer.java +++ b/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/ExampleAdminServer.java @@ -23,7 +23,7 @@ import org.springframework.boot.SpringApplication; import org.springframework.boot.autoconfigure.SpringBootApplication; -@SpringBootApplication +@SpringBootApplication() public class ExampleAdminServer { public static void main(String[] args) throws Exception { From f040cf7e6385109589af40ee0f084ed8b217154a Mon Sep 17 00:00:00 2001 From: sodaRyCN <757083350@qq.com> Date: Sun, 30 Jun 2024 21:00:58 +0800 Subject: [PATCH 02/15] prepare for full and check --- .../apache/eventmesh/admin/server/Admin.java | 1 + .../eventmesh/admin/server/AdminServer.java | 11 +++---- .../admin/server/web/BaseServer.java | 6 ++-- .../admin/server/web/GrpcServer.java | 13 ++++---- .../eventmesh/common/AbstractComponent.java | 26 ++++++++++++++++ .../eventmesh/common}/ComponentLifeCycle.java | 6 ++-- .../connector/rdb/canal/RdbDBDefinition.java | 14 +++++++++ .../rdb/canal/RdbTableDefinition.java | 11 +++++++ .../connector/rdb/canal/RdbTableMgr.java | 30 +++++++++++++++++++ .../rdb/canal/SourceConnectorConfig.java | 6 ++-- .../rdb/canal/mysql/MysqlTableDef.java | 17 +++++++++++ .../openconnect/api/connector/Connector.java | 17 ++--------- 12 files changed, 119 insertions(+), 39 deletions(-) create mode 100644 eventmesh-common/src/main/java/org/apache/eventmesh/common/AbstractComponent.java rename {eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server => eventmesh-common/src/main/java/org/apache/eventmesh/common}/ComponentLifeCycle.java (89%) create mode 100644 eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/rdb/canal/RdbDBDefinition.java create mode 100644 eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/rdb/canal/RdbTableDefinition.java create mode 100644 eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/rdb/canal/RdbTableMgr.java create mode 100644 eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/rdb/canal/mysql/MysqlTableDef.java diff --git a/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/Admin.java b/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/Admin.java index 71c6d67be2..9ee25fadb2 100644 --- a/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/Admin.java +++ b/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/Admin.java @@ -17,6 +17,7 @@ package org.apache.eventmesh.admin.server; +import org.apache.eventmesh.common.ComponentLifeCycle; import org.apache.eventmesh.common.remote.Task; import org.apache.eventmesh.common.remote.request.ReportHeartBeatRequest; import org.apache.eventmesh.common.utils.PagedList; diff --git a/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/AdminServer.java b/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/AdminServer.java index 98247d19b6..ae2736aaaa 100644 --- a/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/AdminServer.java +++ b/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/AdminServer.java @@ -17,6 +17,8 @@ package org.apache.eventmesh.admin.server; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; import org.apache.eventmesh.common.Constants; import org.apache.eventmesh.common.config.CommonConfiguration; import org.apache.eventmesh.common.config.ConfigService; @@ -28,16 +30,11 @@ import org.apache.eventmesh.registry.RegisterServerInfo; import org.apache.eventmesh.registry.RegistryFactory; import org.apache.eventmesh.registry.RegistryService; - -import org.apache.commons.lang3.StringUtils; - -import javax.annotation.PostConstruct; - import org.springframework.boot.context.event.ApplicationReadyEvent; import org.springframework.context.ApplicationListener; import org.springframework.stereotype.Service; -import lombok.extern.slf4j.Slf4j; +import javax.annotation.PostConstruct; @Service @Slf4j @@ -102,7 +99,7 @@ public void start() { } @Override - public void destroy() { + public void stop() { if (configuration.isEventMeshRegistryPluginEnabled()) { registryService.unRegister(adminServeInfo); try { diff --git a/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/BaseServer.java b/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/BaseServer.java index 24085dd89e..9bbe4ce305 100644 --- a/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/BaseServer.java +++ b/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/BaseServer.java @@ -17,7 +17,7 @@ package org.apache.eventmesh.admin.server.web; -import org.apache.eventmesh.admin.server.ComponentLifeCycle; +import org.apache.eventmesh.common.ComponentLifeCycle; import org.apache.eventmesh.common.remote.payload.PayloadFactory; import javax.annotation.PostConstruct; @@ -40,9 +40,9 @@ public void init() throws Exception { } @PreDestroy - public void shutdown() { + public void shutdown() throws Exception { log.info("[{}] server will destroy", this.getClass().getSimpleName()); - destroy(); + stop(); log.info("[{}] server has be destroy", this.getClass().getSimpleName()); } diff --git a/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/GrpcServer.java b/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/GrpcServer.java index 5fbb34f489..3a4bbe3d71 100644 --- a/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/GrpcServer.java +++ b/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/GrpcServer.java @@ -17,17 +17,14 @@ package org.apache.eventmesh.admin.server.web; +import io.grpc.Server; +import io.grpc.netty.shaded.io.grpc.netty.NettyServerBuilder; +import lombok.extern.slf4j.Slf4j; import org.apache.eventmesh.admin.server.AdminServerProperties; - -import java.util.concurrent.TimeUnit; - import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Controller; -import io.grpc.Server; -import io.grpc.netty.shaded.io.grpc.netty.NettyServerBuilder; - -import lombok.extern.slf4j.Slf4j; +import java.util.concurrent.TimeUnit; @Controller @Slf4j @@ -52,7 +49,7 @@ public void start() throws Exception { } @Override - public void destroy() { + public void stop() { try { if (server != null) { server.shutdown(); diff --git a/eventmesh-common/src/main/java/org/apache/eventmesh/common/AbstractComponent.java b/eventmesh-common/src/main/java/org/apache/eventmesh/common/AbstractComponent.java new file mode 100644 index 0000000000..9a1c0a404f --- /dev/null +++ b/eventmesh-common/src/main/java/org/apache/eventmesh/common/AbstractComponent.java @@ -0,0 +1,26 @@ +package org.apache.eventmesh.common; + +import lombok.extern.slf4j.Slf4j; + +/** + * @Description: + */ +@Slf4j +public abstract class AbstractComponent implements ComponentLifeCycle { + @Override + public void start() throws Exception { + log.info("component [{}] will start", this.getClass()); + startup(); + log.info("component [{}] has started successfully", this.getClass()); + } + + @Override + public void stop() throws Exception { + log.info("component [{}] will stop", this.getClass()); + shutdown(); + log.info("component [{}] has stopped successfully", this.getClass()); + } + + protected abstract void startup() throws Exception; + protected abstract void shutdown() throws Exception; +} diff --git a/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/ComponentLifeCycle.java b/eventmesh-common/src/main/java/org/apache/eventmesh/common/ComponentLifeCycle.java similarity index 89% rename from eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/ComponentLifeCycle.java rename to eventmesh-common/src/main/java/org/apache/eventmesh/common/ComponentLifeCycle.java index 392eebfbba..76fdd548d0 100644 --- a/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/ComponentLifeCycle.java +++ b/eventmesh-common/src/main/java/org/apache/eventmesh/common/ComponentLifeCycle.java @@ -15,14 +15,14 @@ * limitations under the License. */ -package org.apache.eventmesh.admin.server; +package org.apache.eventmesh.common; /** - * adminServer ComponentLifeCycle + * LifeCycle of EventMesh Component */ public interface ComponentLifeCycle { void start() throws Exception; - void destroy(); + void stop() throws Exception; } diff --git a/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/rdb/canal/RdbDBDefinition.java b/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/rdb/canal/RdbDBDefinition.java new file mode 100644 index 0000000000..450d0afce0 --- /dev/null +++ b/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/rdb/canal/RdbDBDefinition.java @@ -0,0 +1,14 @@ +package org.apache.eventmesh.common.config.connector.rdb.canal; + +import lombok.Data; + +import java.util.Set; + +/** + * @Description: as class name + */ +@Data +public class RdbDBDefinition { + private String schemaName; + private Set tables; +} diff --git a/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/rdb/canal/RdbTableDefinition.java b/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/rdb/canal/RdbTableDefinition.java new file mode 100644 index 0000000000..cae88fdaa9 --- /dev/null +++ b/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/rdb/canal/RdbTableDefinition.java @@ -0,0 +1,11 @@ +package org.apache.eventmesh.common.config.connector.rdb.canal; + +import lombok.Data; + +/** + * @Description: + */ +@Data +public class RdbTableDefinition { + protected String tableName; +} diff --git a/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/rdb/canal/RdbTableMgr.java b/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/rdb/canal/RdbTableMgr.java new file mode 100644 index 0000000000..31607a1e4d --- /dev/null +++ b/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/rdb/canal/RdbTableMgr.java @@ -0,0 +1,30 @@ +package org.apache.eventmesh.common.config.connector.rdb.canal; + +import lombok.AllArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.apache.eventmesh.common.AbstractComponent; + +import java.util.HashMap; +import java.util.Map; + +/** + * @Description: + */ +@Slf4j +@AllArgsConstructor +public class RdbTableMgr extends AbstractComponent { + private final SourceConnectorConfig config; + private final Map tables = new HashMap<>(); + + @Override + protected void startup() throws Exception { + if (config != null) { + + } + } + + @Override + protected void shutdown() throws Exception { + + } +} diff --git a/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/rdb/canal/SourceConnectorConfig.java b/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/rdb/canal/SourceConnectorConfig.java index e9ae466079..07501625a5 100644 --- a/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/rdb/canal/SourceConnectorConfig.java +++ b/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/rdb/canal/SourceConnectorConfig.java @@ -19,6 +19,8 @@ import lombok.Data; +import java.util.Set; + /** * Represents the configuration for a database connector. */ @@ -37,8 +39,6 @@ public class SourceConnectorConfig { private String passWord; - private String schemaName; - - private String tableName; + private Set schemas; } diff --git a/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/rdb/canal/mysql/MysqlTableDef.java b/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/rdb/canal/mysql/MysqlTableDef.java new file mode 100644 index 0000000000..4b012f8245 --- /dev/null +++ b/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/rdb/canal/mysql/MysqlTableDef.java @@ -0,0 +1,17 @@ +package org.apache.eventmesh.common.config.connector.rdb.canal.mysql; + +import lombok.Data; +import lombok.EqualsAndHashCode; +import org.apache.eventmesh.common.config.connector.rdb.canal.RdbTableDefinition; + +import java.util.Set; + +/** + * @Description: + */ +@Data +@EqualsAndHashCode(callSuper = true) +public class MysqlTableDef extends RdbTableDefinition { + private Set colNames; + private Set primaryKeys; +} diff --git a/eventmesh-openconnect/eventmesh-openconnect-java/src/main/java/org/apache/eventmesh/openconnect/api/connector/Connector.java b/eventmesh-openconnect/eventmesh-openconnect-java/src/main/java/org/apache/eventmesh/openconnect/api/connector/Connector.java index 11c2b77454..8ac09eac38 100644 --- a/eventmesh-openconnect/eventmesh-openconnect-java/src/main/java/org/apache/eventmesh/openconnect/api/connector/Connector.java +++ b/eventmesh-openconnect/eventmesh-openconnect-java/src/main/java/org/apache/eventmesh/openconnect/api/connector/Connector.java @@ -17,13 +17,14 @@ package org.apache.eventmesh.openconnect.api.connector; +import org.apache.eventmesh.common.ComponentLifeCycle; import org.apache.eventmesh.common.config.connector.Config; import org.apache.eventmesh.openconnect.offsetmgmt.api.data.ConnectRecord; /** * Connector */ -public interface Connector { +public interface Connector extends ComponentLifeCycle { /** * Returns the class type of the configuration for this Connector. @@ -52,13 +53,6 @@ public interface Connector { */ void init(ConnectorContext connectorContext) throws Exception; - /** - * Starts the Connector. - * - * @throws Exception if the start operation fails - */ - void start() throws Exception; - /** * Commits the specified ConnectRecord object. * @@ -73,11 +67,4 @@ public interface Connector { */ String name(); - /** - * Stops the Connector. - * - * @throws Exception if stopping fails - */ - void stop() throws Exception; - } From 8096aa34fe8e2e38785590735f6db621591e0265 Mon Sep 17 00:00:00 2001 From: sodaRyCN <757083350@qq.com> Date: Mon, 1 Jul 2024 20:23:24 +0800 Subject: [PATCH 03/15] more and more --- .../eventmesh/common/AbstractComponent.java | 16 +++- .../rdb/canal/CanalSourceFullConfig.java | 15 ++++ .../connector/rdb/canal/RdbDBDefinition.java | 4 +- .../connector/rdb/canal/RdbFullPosition.java | 16 ++++ .../rdb/canal/RdbTableDefinition.java | 2 +- .../connector/rdb/canal/RdbTableMgr.java | 30 -------- .../rdb/canal/SourceConnectorConfig.java | 2 +- .../rdb/canal/mysql/MysqlTableDef.java | 2 +- .../connector/canal/source/EntryParser.java | 15 ++-- .../connector/CanalSourceConnector.java | 10 ++- .../connector/CanalSourceFullConnector.java | 75 +++++++++++++++++++ .../source/position/CanalFullPositionMgr.java | 65 ++++++++++++++++ .../canal/source/table/RdbSimpleTable.java | 31 ++++++++ .../canal/source/table/RdbTableMgr.java | 61 +++++++++++++++ 14 files changed, 297 insertions(+), 47 deletions(-) create mode 100644 eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/rdb/canal/CanalSourceFullConfig.java create mode 100644 eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/rdb/canal/RdbFullPosition.java delete mode 100644 eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/rdb/canal/RdbTableMgr.java create mode 100644 eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/source/connector/CanalSourceFullConnector.java create mode 100644 eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/source/position/CanalFullPositionMgr.java create mode 100644 eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/source/table/RdbSimpleTable.java create mode 100644 eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/source/table/RdbTableMgr.java diff --git a/eventmesh-common/src/main/java/org/apache/eventmesh/common/AbstractComponent.java b/eventmesh-common/src/main/java/org/apache/eventmesh/common/AbstractComponent.java index 9a1c0a404f..bca798602f 100644 --- a/eventmesh-common/src/main/java/org/apache/eventmesh/common/AbstractComponent.java +++ b/eventmesh-common/src/main/java/org/apache/eventmesh/common/AbstractComponent.java @@ -2,23 +2,35 @@ import lombok.extern.slf4j.Slf4j; +import java.util.concurrent.atomic.AtomicBoolean; + /** * @Description: */ @Slf4j public abstract class AbstractComponent implements ComponentLifeCycle { + private final AtomicBoolean init = new AtomicBoolean(false); + private final AtomicBoolean stopped = new AtomicBoolean(false); @Override public void start() throws Exception { + if (!init.compareAndSet(false, true)){ + log.info("component [{}] has started", this.getClass()); + return; + } log.info("component [{}] will start", this.getClass()); startup(); - log.info("component [{}] has started successfully", this.getClass()); + log.info("component [{}] started successfully", this.getClass()); } @Override public void stop() throws Exception { + if (!stopped.compareAndSet(false, true)){ + log.info("component [{}] has stopped", this.getClass()); + return; + } log.info("component [{}] will stop", this.getClass()); shutdown(); - log.info("component [{}] has stopped successfully", this.getClass()); + log.info("component [{}] stopped successfully", this.getClass()); } protected abstract void startup() throws Exception; diff --git a/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/rdb/canal/CanalSourceFullConfig.java b/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/rdb/canal/CanalSourceFullConfig.java new file mode 100644 index 0000000000..8c5f94bd14 --- /dev/null +++ b/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/rdb/canal/CanalSourceFullConfig.java @@ -0,0 +1,15 @@ +package org.apache.eventmesh.common.config.connector.rdb.canal; + +import lombok.Data; +import lombok.EqualsAndHashCode; +import org.apache.eventmesh.common.config.connector.SourceConfig; +import org.apache.eventmesh.common.remote.offset.RecordPosition; + +import java.util.List; + +@Data +@EqualsAndHashCode(callSuper = true) +public class CanalSourceFullConfig extends SourceConfig { + private SourceConnectorConfig connectorConfig; + private List startPosition; +} diff --git a/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/rdb/canal/RdbDBDefinition.java b/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/rdb/canal/RdbDBDefinition.java index 450d0afce0..8e8c075d54 100644 --- a/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/rdb/canal/RdbDBDefinition.java +++ b/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/rdb/canal/RdbDBDefinition.java @@ -5,10 +5,10 @@ import java.util.Set; /** - * @Description: as class name + * Description: as class name */ @Data public class RdbDBDefinition { - private String schemaName; + private String schema; private Set tables; } diff --git a/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/rdb/canal/RdbFullPosition.java b/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/rdb/canal/RdbFullPosition.java new file mode 100644 index 0000000000..41267946c9 --- /dev/null +++ b/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/rdb/canal/RdbFullPosition.java @@ -0,0 +1,16 @@ +package org.apache.eventmesh.common.config.connector.rdb.canal; + +import lombok.Data; + +import java.math.BigDecimal; + +@Data +public class RdbFullPosition { + private String jobId; + private String schema; + private String tableName; + private String curPrimaryKey; + private BigDecimal minPrimaryKeyNum; + private BigDecimal maxPrimaryKeyNum; + private boolean finished; +} diff --git a/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/rdb/canal/RdbTableDefinition.java b/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/rdb/canal/RdbTableDefinition.java index cae88fdaa9..b2bfbc6b84 100644 --- a/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/rdb/canal/RdbTableDefinition.java +++ b/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/rdb/canal/RdbTableDefinition.java @@ -3,7 +3,7 @@ import lombok.Data; /** - * @Description: + * Description: as class name */ @Data public class RdbTableDefinition { diff --git a/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/rdb/canal/RdbTableMgr.java b/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/rdb/canal/RdbTableMgr.java deleted file mode 100644 index 31607a1e4d..0000000000 --- a/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/rdb/canal/RdbTableMgr.java +++ /dev/null @@ -1,30 +0,0 @@ -package org.apache.eventmesh.common.config.connector.rdb.canal; - -import lombok.AllArgsConstructor; -import lombok.extern.slf4j.Slf4j; -import org.apache.eventmesh.common.AbstractComponent; - -import java.util.HashMap; -import java.util.Map; - -/** - * @Description: - */ -@Slf4j -@AllArgsConstructor -public class RdbTableMgr extends AbstractComponent { - private final SourceConnectorConfig config; - private final Map tables = new HashMap<>(); - - @Override - protected void startup() throws Exception { - if (config != null) { - - } - } - - @Override - protected void shutdown() throws Exception { - - } -} diff --git a/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/rdb/canal/SourceConnectorConfig.java b/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/rdb/canal/SourceConnectorConfig.java index 07501625a5..01aa01fe09 100644 --- a/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/rdb/canal/SourceConnectorConfig.java +++ b/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/rdb/canal/SourceConnectorConfig.java @@ -39,6 +39,6 @@ public class SourceConnectorConfig { private String passWord; - private Set schemas; + private Set databases; } diff --git a/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/rdb/canal/mysql/MysqlTableDef.java b/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/rdb/canal/mysql/MysqlTableDef.java index 4b012f8245..8125c7e5d9 100644 --- a/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/rdb/canal/mysql/MysqlTableDef.java +++ b/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/rdb/canal/mysql/MysqlTableDef.java @@ -7,7 +7,7 @@ import java.util.Set; /** - * @Description: + * Description: */ @Data @EqualsAndHashCode(callSuper = true) diff --git a/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/source/EntryParser.java b/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/source/EntryParser.java index 32c55ec42c..3ca2f7ec2f 100644 --- a/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/source/EntryParser.java +++ b/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/source/EntryParser.java @@ -31,6 +31,7 @@ import java.util.List; import java.util.Map; +import org.apache.eventmesh.connector.canal.source.table.RdbTableMgr; import org.springframework.util.CollectionUtils; import com.alibaba.otter.canal.protocol.CanalEntry; @@ -47,7 +48,7 @@ @Slf4j public class EntryParser { - public Map> parse(CanalSourceConfig sourceConfig, List datas) { + public static Map> parse(CanalSourceConfig sourceConfig, List datas) { List recordList = new ArrayList<>(); List transactionDataBuffer = new ArrayList<>(); // need check weather the entry is loopback @@ -116,11 +117,10 @@ private Column getColumnIgnoreCase(List columns, String columName) { return null; } - private List internParse(CanalSourceConfig sourceConfig, Entry entry) { + private static List internParse(CanalSourceConfig sourceConfig, Entry entry, RdbTableMgr tableMgr) { String schemaName = entry.getHeader().getSchemaName(); String tableName = entry.getHeader().getTableName(); - if (!schemaName.equalsIgnoreCase(sourceConfig.getSourceConnectorConfig().getSchemaName()) - || !tableName.equalsIgnoreCase(sourceConfig.getSourceConnectorConfig().getTableName())) { + if (tableMgr.getTable(schemaName, tableName) == null) { return null; } @@ -155,7 +155,7 @@ private List internParse(CanalSourceConfig sourceConfig, Ent return recordList; } - private CanalConnectRecord internParse(CanalSourceConfig canalSourceConfig, Entry entry, RowChange rowChange, RowData rowData) { + private static CanalConnectRecord internParse(CanalSourceConfig canalSourceConfig, Entry entry, RowChange rowChange, RowData rowData) { CanalConnectRecord canalConnectRecord = new CanalConnectRecord(); canalConnectRecord.setTableName(entry.getHeader().getTableName()); canalConnectRecord.setSchemaName(entry.getHeader().getSchemaName()); @@ -242,7 +242,8 @@ private CanalConnectRecord internParse(CanalSourceConfig canalSourceConfig, Entr return canalConnectRecord; } - private void checkUpdateKeyColumns(Map oldKeyColumns, Map keyColumns) { + private static void checkUpdateKeyColumns(Map oldKeyColumns, + Map keyColumns) { if (oldKeyColumns.isEmpty()) { return; } @@ -264,7 +265,7 @@ private void checkUpdateKeyColumns(Map oldKeyColumns, Map configClass() { return CanalSourceConfig.class; @@ -146,6 +149,8 @@ protected void startEventParserInternal(CanalEventParser parser, boolean isGroup return instance; } }); + tableMgr = RdbTableMgr.getInstance(); + tableMgr.init(sourceConfig.getSourceConnectorConfig()); } private Canal buildCanal(CanalSourceConfig sourceConfig) { @@ -218,6 +223,7 @@ public void start() throws Exception { if (running) { return; } + tableMgr.start(); canalServer.start(); canalServer.start(sourceConfig.getDestination()); @@ -288,11 +294,9 @@ public List poll() { entries = message.getEntries(); } - EntryParser entryParser = new EntryParser(); - List result = new ArrayList<>(); // key: Xid offset - Map> connectorRecordMap = entryParser.parse(sourceConfig, entries); + Map> connectorRecordMap = EntryParser.parse(sourceConfig, entries); if (!connectorRecordMap.isEmpty()) { Set>> entrySet = connectorRecordMap.entrySet(); diff --git a/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/source/connector/CanalSourceFullConnector.java b/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/source/connector/CanalSourceFullConnector.java new file mode 100644 index 0000000000..434c1464d4 --- /dev/null +++ b/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/source/connector/CanalSourceFullConnector.java @@ -0,0 +1,75 @@ +package org.apache.eventmesh.connector.canal.source.connector; + +import lombok.extern.slf4j.Slf4j; +import org.apache.eventmesh.common.AbstractComponent; +import org.apache.eventmesh.common.config.connector.Config; +import org.apache.eventmesh.common.config.connector.rdb.canal.CanalSourceConfig; +import org.apache.eventmesh.common.config.connector.rdb.canal.CanalSourceFullConfig; +import org.apache.eventmesh.common.config.connector.rdb.canal.RdbDBDefinition; +import org.apache.eventmesh.common.config.connector.rdb.canal.RdbTableDefinition; +import org.apache.eventmesh.common.config.connector.rdb.canal.SourceConnectorConfig; +import org.apache.eventmesh.openconnect.api.ConnectorCreateService; +import org.apache.eventmesh.openconnect.api.connector.ConnectorContext; +import org.apache.eventmesh.openconnect.api.connector.SourceConnectorContext; +import org.apache.eventmesh.openconnect.api.source.Source; +import org.apache.eventmesh.openconnect.offsetmgmt.api.data.ConnectRecord; + +import java.util.List; + +@Slf4j +public class CanalSourceFullConnector extends AbstractComponent implements Source, ConnectorCreateService { + private CanalSourceFullConfig config; + + @Override + protected void startup() throws Exception { + if (config.getConnectorConfig().getDatabases() != null) { + for (RdbDBDefinition db : config.getConnectorConfig().getDatabases()) { + for (RdbTableDefinition table : db.getTables()) { + log.info("[]"); + } + } + } + } + + @Override + protected void shutdown() throws Exception { + + } + + @Override + public Source create() { + return new CanalSourceFullConnector(); + } + + @Override + public Class configClass() { + return CanalSourceFullConfig.class; + } + + @Override + public void init(Config config) throws Exception { + this.config = (CanalSourceFullConfig)config; + } + + @Override + public void init(ConnectorContext connectorContext) throws Exception { + SourceConnectorContext sourceConnectorContext = (SourceConnectorContext) connectorContext; + this.config = (CanalSourceFullConfig) sourceConnectorContext.getSourceConfig(); + + } + + @Override + public void commit(ConnectRecord record) { + + } + + @Override + public String name() { + return this.config.getConnectorConfig().getConnectorName(); + } + + @Override + public List poll() { + return null; + } +} diff --git a/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/source/position/CanalFullPositionMgr.java b/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/source/position/CanalFullPositionMgr.java new file mode 100644 index 0000000000..70611da632 --- /dev/null +++ b/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/source/position/CanalFullPositionMgr.java @@ -0,0 +1,65 @@ +package org.apache.eventmesh.connector.canal.source.position; + +import lombok.AllArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.apache.eventmesh.common.AbstractComponent; +import org.apache.eventmesh.common.config.connector.rdb.canal.CanalSourceFullConfig; +import org.apache.eventmesh.common.config.connector.rdb.canal.RdbDBDefinition; +import org.apache.eventmesh.common.config.connector.rdb.canal.RdbFullPosition; +import org.apache.eventmesh.common.config.connector.rdb.canal.RdbTableDefinition; +import org.apache.eventmesh.common.remote.offset.RecordPosition; +import org.apache.eventmesh.connector.canal.source.table.RdbSimpleTable; + +import java.util.Map; +import java.util.concurrent.ScheduledThreadPoolExecutor; + +@AllArgsConstructor +@Slf4j +public class CanalFullPositionMgr extends AbstractComponent { + private CanalSourceFullConfig config; + private ScheduledThreadPoolExecutor executor; + private Map positions; + + @Override + protected void startup() throws Exception { + if (config == null || config.getConnectorConfig() == null || config.getConnectorConfig().getDatabases() == null) { + log.info("config or database is null"); + return; + } + executor = new ScheduledThreadPoolExecutor(1, r -> { + Thread thread = new Thread(r); + thread.setDaemon(true); + thread.setName("task-full-position-timer"); + return thread; + }, new ScheduledThreadPoolExecutor.DiscardOldestPolicy()); + if (config.getStartPosition() != null) { + for (RecordPosition recordPosition : config.getStartPosition()) { + + } + } + + } + + private void processPositions(CanalSourceFullConfig config) { + for (RdbDBDefinition database : config.getConnectorConfig().getDatabases()) { + for (RdbTableDefinition table : database.getTables()) { + log.info("init position of data [{}] table [{}]", database.getSchema(), table.getTableName()); + RdbSimpleTable simpleTable = new RdbSimpleTable(database.getSchema(), table.getTableName()); + RdbFullPosition recordPosition = positions.get(simpleTable); + if (recordPosition == null || !recordPosition.isFinished()) { + positions.put(simpleTable,initPosition(config, database.getSchema(), table.getTableName(), recordPosition)); + } + } + } + } + + private RdbFullPosition initPosition(CanalSourceFullConfig config, String db, String table, + RdbFullPosition recordPosition) { + return null; + } + + @Override + protected void shutdown() throws Exception { + + } +} diff --git a/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/source/table/RdbSimpleTable.java b/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/source/table/RdbSimpleTable.java new file mode 100644 index 0000000000..cd8fa4c93b --- /dev/null +++ b/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/source/table/RdbSimpleTable.java @@ -0,0 +1,31 @@ +package org.apache.eventmesh.connector.canal.source.table; + +import lombok.AllArgsConstructor; +import lombok.Data; +import org.apache.eventmesh.common.config.connector.rdb.canal.RdbTableDefinition; + +import java.util.Objects; + +@Data +public class RdbSimpleTable extends RdbTableDefinition { + public RdbSimpleTable(String schema, String tableName) { + super(); + this.schema = schema; + this.tableName = tableName; + } + private String schema; + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + if (!super.equals(o)) return false; + RdbSimpleTable that = (RdbSimpleTable) o; + return Objects.equals(schema, that.schema); + } + + @Override + public int hashCode() { + return Objects.hash(super.hashCode(), schema); + } +} diff --git a/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/source/table/RdbTableMgr.java b/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/source/table/RdbTableMgr.java new file mode 100644 index 0000000000..c7673880e7 --- /dev/null +++ b/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/source/table/RdbTableMgr.java @@ -0,0 +1,61 @@ +package org.apache.eventmesh.connector.canal.source.table; + +import lombok.AllArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.apache.eventmesh.common.AbstractComponent; +import org.apache.eventmesh.common.config.connector.rdb.canal.RdbDBDefinition; +import org.apache.eventmesh.common.config.connector.rdb.canal.RdbTableDefinition; +import org.apache.eventmesh.common.config.connector.rdb.canal.SourceConnectorConfig; + +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.atomic.AtomicBoolean; + +/** + * Description: + */ +@Slf4j +@AllArgsConstructor +public class RdbTableMgr extends AbstractComponent { + private SourceConnectorConfig config; + private final Map tables = new HashMap<>(); + + public static String generate(String ...params) { + return String.join("@", params); + } + + private RdbTableMgr(){ + } + + private static class RdbTableMgrHolder { + private static final RdbTableMgr INSTANCE = new RdbTableMgr(); + } + + public static RdbTableMgr getInstance() { + return RdbTableMgrHolder.INSTANCE; + } + + public void init(SourceConnectorConfig config) { + this.config = config; + } + + public RdbTableDefinition getTable(String dbName, String tableName) { + return tables.get(generate(dbName, tableName)); + } + + @Override + protected void startup() throws Exception { + if (config != null && config.getDatabases() != null) { + for (RdbDBDefinition db : config.getDatabases()) { + for (RdbTableDefinition table : db.getTables()) { + tables.put(generate(db.getSchema(), table.getTableName()), table); + } + } + } + } + + @Override + protected void shutdown() throws Exception { + + } +} From 43c1b6669ced29c917d6766d60be2c86b85b5f40 Mon Sep 17 00:00:00 2001 From: sodaRyCN <757083350@qq.com> Date: Fri, 5 Jul 2024 18:20:50 +0800 Subject: [PATCH 04/15] mysql type for read --- .../canal/source/connector/CanalSourceFullConnector.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/source/connector/CanalSourceFullConnector.java b/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/source/connector/CanalSourceFullConnector.java index 434c1464d4..79481a69af 100644 --- a/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/source/connector/CanalSourceFullConnector.java +++ b/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/source/connector/CanalSourceFullConnector.java @@ -3,11 +3,9 @@ import lombok.extern.slf4j.Slf4j; import org.apache.eventmesh.common.AbstractComponent; import org.apache.eventmesh.common.config.connector.Config; -import org.apache.eventmesh.common.config.connector.rdb.canal.CanalSourceConfig; import org.apache.eventmesh.common.config.connector.rdb.canal.CanalSourceFullConfig; import org.apache.eventmesh.common.config.connector.rdb.canal.RdbDBDefinition; import org.apache.eventmesh.common.config.connector.rdb.canal.RdbTableDefinition; -import org.apache.eventmesh.common.config.connector.rdb.canal.SourceConnectorConfig; import org.apache.eventmesh.openconnect.api.ConnectorCreateService; import org.apache.eventmesh.openconnect.api.connector.ConnectorContext; import org.apache.eventmesh.openconnect.api.connector.SourceConnectorContext; From 6f093d34081c946f00e16352e035f7f1c02dc1df Mon Sep 17 00:00:00 2001 From: sodaRyCN <757083350@qq.com> Date: Fri, 5 Jul 2024 18:20:50 +0800 Subject: [PATCH 05/15] mysql type for read --- build.gradle | 3 +- eventmesh-common/build.gradle | 1 + .../eventmesh/common/AbstractComponent.java | 4 +- .../connector/rdb/canal/CanalMySQLType.java | 142 ++++++++ .../rdb/canal/CanalSourceFullConfig.java | 1 + ...lPosition.java => JobRdbFullPosition.java} | 5 +- .../rdb/canal/RdbColumnDefinition.java | 11 + .../connector/rdb/canal/RdbDBDefinition.java | 2 +- .../rdb/canal/RdbTableDefinition.java | 1 + .../connector/rdb/canal/mysql/Constants.java | 5 + .../rdb/canal/mysql/MySQLColumnDef.java | 12 + ...{MysqlTableDef.java => MySQLTableDef.java} | 6 +- .../eventmesh-connector-canal/build.gradle | 1 + .../connector/canal/DatabaseConnection.java | 58 ++-- .../eventmesh/connector/canal/SqlUtils.java | 28 ++ .../canal/source/CanalFullProducer.java | 325 ++++++++++++++++++ .../connector/CanalSourceConnector.java | 4 +- .../connector/CanalSourceFullConnector.java | 48 ++- .../source/position/CanalFullPositionMgr.java | 181 +++++++++- .../source/position/TableFullPosition.java | 13 + .../canal/source/table/RdbTableMgr.java | 149 ++++++-- 21 files changed, 920 insertions(+), 80 deletions(-) create mode 100644 eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/rdb/canal/CanalMySQLType.java rename eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/rdb/canal/{RdbFullPosition.java => JobRdbFullPosition.java} (70%) create mode 100644 eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/rdb/canal/RdbColumnDefinition.java create mode 100644 eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/rdb/canal/mysql/Constants.java create mode 100644 eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/rdb/canal/mysql/MySQLColumnDef.java rename eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/rdb/canal/mysql/{MysqlTableDef.java => MySQLTableDef.java} (59%) create mode 100644 eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/source/CanalFullProducer.java create mode 100644 eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/source/position/TableFullPosition.java diff --git a/build.gradle b/build.gradle index 0408231fd7..3d2f8604a3 100644 --- a/build.gradle +++ b/build.gradle @@ -799,8 +799,9 @@ subprojects { dependency "com.alibaba:druid-spring-boot-starter:1.2.23" dependency "com.baomidou:mybatis-plus-boot-starter:3.5.5" dependency "org.springframework.boot:spring-boot-starter-jetty:2.7.18" - dependency "com.baomidou:mybatis-plus:3.5.7" dependency "com.mysql:mysql-connector-j:8.4.0" + dependency "org.springframework.boot:spring-boot-starter-jetty:2.7.10" + dependency "org.locationtech.jts:jts-core:1.19.0" } } } diff --git a/eventmesh-common/build.gradle b/eventmesh-common/build.gradle index 70244d2299..c95e9f6c29 100644 --- a/eventmesh-common/build.gradle +++ b/eventmesh-common/build.gradle @@ -48,6 +48,7 @@ dependencies { implementation "org.apache.httpcomponents:httpclient" implementation "io.netty:netty-all" + compileOnly 'com.mysql:mysql-connector-j' implementation "io.grpc:grpc-protobuf:${grpcVersion}" implementation "io.grpc:grpc-stub:${grpcVersion}" diff --git a/eventmesh-common/src/main/java/org/apache/eventmesh/common/AbstractComponent.java b/eventmesh-common/src/main/java/org/apache/eventmesh/common/AbstractComponent.java index bca798602f..bc4e9ad404 100644 --- a/eventmesh-common/src/main/java/org/apache/eventmesh/common/AbstractComponent.java +++ b/eventmesh-common/src/main/java/org/apache/eventmesh/common/AbstractComponent.java @@ -9,11 +9,11 @@ */ @Slf4j public abstract class AbstractComponent implements ComponentLifeCycle { - private final AtomicBoolean init = new AtomicBoolean(false); + private final AtomicBoolean started = new AtomicBoolean(false); private final AtomicBoolean stopped = new AtomicBoolean(false); @Override public void start() throws Exception { - if (!init.compareAndSet(false, true)){ + if (!started.compareAndSet(false, true)){ log.info("component [{}] has started", this.getClass()); return; } diff --git a/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/rdb/canal/CanalMySQLType.java b/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/rdb/canal/CanalMySQLType.java new file mode 100644 index 0000000000..969e631e93 --- /dev/null +++ b/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/rdb/canal/CanalMySQLType.java @@ -0,0 +1,142 @@ +package org.apache.eventmesh.common.config.connector.rdb.canal; + +import com.mysql.cj.MysqlType; + +import java.sql.JDBCType; + +public enum CanalMySQLType { + BIT("BIT"), + TINYINT("TINYINT"), + SMALLINT("SMALLINT"), + MEDIUMINT("MEDIUMINT"), + INT("INT"), + BIGINT("BIGINT"), + DECIMAL("DECIMAL"), + FLOAT("FLOAT", JDBCType.REAL), + DOUBLE("DOUBLE", JDBCType.DOUBLE), + DATE("DATE", JDBCType.DATE), + DATETIME("DATETIME", JDBCType.TIMESTAMP), + TIMESTAMP("TIMESTAMP", JDBCType.TIMESTAMP), + TIME("TIME", JDBCType.TIME), + YEAR("YEAR", JDBCType.DATE), + CHAR("CHAR", JDBCType.CHAR), + VARCHAR("VARCHAR", JDBCType.VARCHAR), + BINARY("BINARY", JDBCType.BINARY), + VARBINARY("VARBINARY", JDBCType.VARBINARY), + TINYBLOB("TINYBLOB", JDBCType.VARBINARY), + BLOB("BLOB", JDBCType.LONGVARBINARY), + MEDIUMBLOB("MEDIUMBLOB", JDBCType.LONGVARBINARY), + LONGBLOB("LONGBLOB", JDBCType.LONGVARBINARY), + TINYTEXT("TINYTEXT", JDBCType.VARCHAR), + TEXT("TEXT", JDBCType.LONGVARCHAR), + MEDIUMTEXT("MEDIUMTEXT", JDBCType.LONGVARCHAR), + LONGTEXT("LONGTEXT", JDBCType.LONGVARCHAR), + ENUM("ENUM", JDBCType.CHAR), + SET("SET", JDBCType.CHAR), + JSON("JSON", JDBCType.LONGVARCHAR), + GEOMETRY("GEOMETRY", JDBCType.BINARY), + POINT("POINT", JDBCType.BINARY), + LINESTRING("LINESTRING", JDBCType.BINARY), + POLYGON("POLYGON", JDBCType.BINARY), + MULTIPOINT("MULTIPOINT", JDBCType.BINARY), + GEOMETRY_COLLECTION("GEOMETRYCOLLECTION", JDBCType.BINARY), + GEOM_COLLECTION("GEOMCOLLECTION", JDBCType.BINARY), + MULTILINESTRING("MULTILINESTRING", JDBCType.BINARY), + MULTIPOLYGON("MULTIPOLYGON", JDBCType.BINARY); + + private final String codeKey; + private final MysqlType mysqlType; + + CanalMySQLType(String codeKey) { + this.codeKey = codeKey; + this.mysqlType = MysqlType.getByName(codeKey); + } + + public static CanalMySQLType valueOfCode(String code) { + CanalMySQLType[] values = values(); + for (CanalMySQLType tableType : values) { + if (tableType.codeKey.equalsIgnoreCase(code)) { + return tableType; + } + } + switch (MysqlType.getByName(code)) { + case BOOLEAN: + case TINYINT: + case TINYINT_UNSIGNED: + return TINYINT; + case SMALLINT: + case SMALLINT_UNSIGNED: + return SMALLINT; + case INT: + case INT_UNSIGNED: + return INT; + case BIGINT: + case BIGINT_UNSIGNED: + return BIGINT; + case MEDIUMINT: + case MEDIUMINT_UNSIGNED: + return MEDIUMINT; + case DECIMAL: + case DECIMAL_UNSIGNED: + return DECIMAL; + case FLOAT: + case FLOAT_UNSIGNED: + return FLOAT; + case DOUBLE: + case DOUBLE_UNSIGNED: + return DOUBLE; + case BIT: + return BIT; + case BINARY: + return BINARY; + case VARBINARY: + return VARBINARY; + case TINYBLOB: + return TINYBLOB; + case MEDIUMBLOB: + return MEDIUMBLOB; + case LONGBLOB: + return LONGBLOB; + case BLOB: + return BLOB; + case CHAR: + return CHAR; + case VARCHAR: + return VARCHAR; + case TINYTEXT: + return TINYTEXT; + case MEDIUMTEXT: + return MEDIUMTEXT; + case LONGTEXT: + return LONGTEXT; + case TEXT: + return TEXT; + case DATE: + return DATE; + case TIME: + return TIME; + case TIMESTAMP: + return TIMESTAMP; + case DATETIME: + return DATETIME; + case YEAR: + return YEAR; + case JSON: + return JSON; + case ENUM: + return ENUM; + case SET: + return SET; + case GEOMETRY: + return GEOMETRY; + case NULL: + case UNKNOWN: + default: + throw new UnsupportedOperationException("Unsupported mysql columnType " + code); + } + } + + public MysqlType getMysqlType() { + return mysqlType; + } +} diff --git a/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/rdb/canal/CanalSourceFullConfig.java b/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/rdb/canal/CanalSourceFullConfig.java index 8c5f94bd14..fcfa6a0e92 100644 --- a/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/rdb/canal/CanalSourceFullConfig.java +++ b/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/rdb/canal/CanalSourceFullConfig.java @@ -12,4 +12,5 @@ public class CanalSourceFullConfig extends SourceConfig { private SourceConnectorConfig connectorConfig; private List startPosition; + private int parallel; } diff --git a/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/rdb/canal/RdbFullPosition.java b/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/rdb/canal/JobRdbFullPosition.java similarity index 70% rename from eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/rdb/canal/RdbFullPosition.java rename to eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/rdb/canal/JobRdbFullPosition.java index 41267946c9..b4aeffd721 100644 --- a/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/rdb/canal/RdbFullPosition.java +++ b/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/rdb/canal/JobRdbFullPosition.java @@ -5,12 +5,11 @@ import java.math.BigDecimal; @Data -public class RdbFullPosition { +public class JobRdbFullPosition { private String jobId; private String schema; private String tableName; private String curPrimaryKey; - private BigDecimal minPrimaryKeyNum; - private BigDecimal maxPrimaryKeyNum; + private long maxCount; private boolean finished; } diff --git a/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/rdb/canal/RdbColumnDefinition.java b/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/rdb/canal/RdbColumnDefinition.java new file mode 100644 index 0000000000..2fe4d7a2f9 --- /dev/null +++ b/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/rdb/canal/RdbColumnDefinition.java @@ -0,0 +1,11 @@ +package org.apache.eventmesh.common.config.connector.rdb.canal; + +import lombok.Data; + +import java.sql.JDBCType; + +@Data +public class RdbColumnDefinition { + protected String name; + protected JDBCType jdbcType; +} diff --git a/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/rdb/canal/RdbDBDefinition.java b/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/rdb/canal/RdbDBDefinition.java index 8e8c075d54..d6b08503dd 100644 --- a/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/rdb/canal/RdbDBDefinition.java +++ b/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/rdb/canal/RdbDBDefinition.java @@ -9,6 +9,6 @@ */ @Data public class RdbDBDefinition { - private String schema; + private String schemaName; private Set tables; } diff --git a/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/rdb/canal/RdbTableDefinition.java b/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/rdb/canal/RdbTableDefinition.java index b2bfbc6b84..5001a8bcb8 100644 --- a/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/rdb/canal/RdbTableDefinition.java +++ b/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/rdb/canal/RdbTableDefinition.java @@ -7,5 +7,6 @@ */ @Data public class RdbTableDefinition { + protected String schemaName; protected String tableName; } diff --git a/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/rdb/canal/mysql/Constants.java b/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/rdb/canal/mysql/Constants.java new file mode 100644 index 0000000000..3025be245b --- /dev/null +++ b/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/rdb/canal/mysql/Constants.java @@ -0,0 +1,5 @@ +package org.apache.eventmesh.common.config.connector.rdb.canal.mysql; + +public class Constants { + public static final String MySQLQuot = "`"; +} diff --git a/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/rdb/canal/mysql/MySQLColumnDef.java b/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/rdb/canal/mysql/MySQLColumnDef.java new file mode 100644 index 0000000000..8f1fcbd741 --- /dev/null +++ b/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/rdb/canal/mysql/MySQLColumnDef.java @@ -0,0 +1,12 @@ +package org.apache.eventmesh.common.config.connector.rdb.canal.mysql; + +import lombok.Data; +import lombok.EqualsAndHashCode; +import org.apache.eventmesh.common.config.connector.rdb.canal.CanalMySQLType; +import org.apache.eventmesh.common.config.connector.rdb.canal.RdbColumnDefinition; + +@Data +@EqualsAndHashCode(callSuper = true) +public class MySQLColumnDef extends RdbColumnDefinition { + private CanalMySQLType type; +} diff --git a/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/rdb/canal/mysql/MysqlTableDef.java b/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/rdb/canal/mysql/MySQLTableDef.java similarity index 59% rename from eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/rdb/canal/mysql/MysqlTableDef.java rename to eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/rdb/canal/mysql/MySQLTableDef.java index 8125c7e5d9..439d80bcd5 100644 --- a/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/rdb/canal/mysql/MysqlTableDef.java +++ b/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/rdb/canal/mysql/MySQLTableDef.java @@ -2,8 +2,10 @@ import lombok.Data; import lombok.EqualsAndHashCode; +import org.apache.eventmesh.common.config.connector.rdb.canal.RdbColumnDefinition; import org.apache.eventmesh.common.config.connector.rdb.canal.RdbTableDefinition; +import java.util.Map; import java.util.Set; /** @@ -11,7 +13,7 @@ */ @Data @EqualsAndHashCode(callSuper = true) -public class MysqlTableDef extends RdbTableDefinition { - private Set colNames; +public class MySQLTableDef extends RdbTableDefinition { private Set primaryKeys; + private Map columnDefinitions; } diff --git a/eventmesh-connectors/eventmesh-connector-canal/build.gradle b/eventmesh-connectors/eventmesh-connector-canal/build.gradle index ccc5acf0ca..77a9968496 100644 --- a/eventmesh-connectors/eventmesh-connector-canal/build.gradle +++ b/eventmesh-connectors/eventmesh-connector-canal/build.gradle @@ -23,6 +23,7 @@ List canal = [ dependencies { api project(":eventmesh-openconnect:eventmesh-openconnect-java") + implementation "org.locationtech.jts" implementation project(":eventmesh-common") implementation canal implementation "com.alibaba:druid:1.2.23" diff --git a/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/DatabaseConnection.java b/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/DatabaseConnection.java index 0d9da7f8be..00f9693be3 100644 --- a/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/DatabaseConnection.java +++ b/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/DatabaseConnection.java @@ -36,42 +36,36 @@ public class DatabaseConnection { public static CanalSinkConfig sinkConfig; + public static DruidDataSource createDruidDataSource(String url, String UserName, String passWord) { + DruidDataSource dataSource = new DruidDataSource(); + dataSource.setUrl(url); + dataSource.setUsername(UserName); + dataSource.setPassword(passWord); + dataSource.setInitialSize(5); + dataSource.setMinIdle(5); + dataSource.setMaxActive(20); + dataSource.setMaxWait(60000); + dataSource.setTimeBetweenEvictionRunsMillis(60000); + dataSource.setMinEvictableIdleTimeMillis(300000); + dataSource.setValidationQuery("SELECT 1"); + dataSource.setTestWhileIdle(true); + dataSource.setTestOnBorrow(false); + dataSource.setTestOnReturn(false); + dataSource.setPoolPreparedStatements(true); + dataSource.setMaxPoolPreparedStatementPerConnectionSize(20); + return dataSource; + } + public static void initSourceConnection() { - sourceDataSource = new DruidDataSource(); - sourceDataSource.setUrl(sourceConfig.getSourceConnectorConfig().getUrl()); - sourceDataSource.setUsername(sourceConfig.getSourceConnectorConfig().getUserName()); - sourceDataSource.setPassword(sourceConfig.getSourceConnectorConfig().getPassWord()); - sourceDataSource.setInitialSize(5); - sourceDataSource.setMinIdle(5); - sourceDataSource.setMaxActive(20); - sourceDataSource.setMaxWait(60000); - sourceDataSource.setTimeBetweenEvictionRunsMillis(60000); - sourceDataSource.setMinEvictableIdleTimeMillis(300000); - sourceDataSource.setValidationQuery("SELECT 1"); - sourceDataSource.setTestWhileIdle(true); - sourceDataSource.setTestOnBorrow(false); - sourceDataSource.setTestOnReturn(false); - sourceDataSource.setPoolPreparedStatements(true); - sourceDataSource.setMaxPoolPreparedStatementPerConnectionSize(20); + sourceDataSource = createDruidDataSource(sourceConfig.getSourceConnectorConfig().getUrl(), + sourceConfig.getSourceConnectorConfig().getUserName(), + sourceConfig.getSourceConnectorConfig().getPassWord()); } public static void initSinkConnection() { - sinkDataSource = new DruidDataSource(); - sinkDataSource.setUrl(sinkConfig.getSinkConnectorConfig().getUrl()); - sinkDataSource.setUsername(sinkConfig.getSinkConnectorConfig().getUserName()); - sinkDataSource.setPassword(sinkConfig.getSinkConnectorConfig().getPassWord()); - sinkDataSource.setInitialSize(5); - sinkDataSource.setMinIdle(5); - sinkDataSource.setMaxActive(20); - sinkDataSource.setMaxWait(60000); - sinkDataSource.setTimeBetweenEvictionRunsMillis(60000); - sinkDataSource.setMinEvictableIdleTimeMillis(300000); - sinkDataSource.setValidationQuery("SELECT 1"); - sinkDataSource.setTestWhileIdle(true); - sinkDataSource.setTestOnBorrow(false); - sinkDataSource.setTestOnReturn(false); - sinkDataSource.setPoolPreparedStatements(true); - sinkDataSource.setMaxPoolPreparedStatementPerConnectionSize(20); + sinkDataSource = createDruidDataSource(sinkConfig.getSinkConnectorConfig().getUrl(), + sinkConfig.getSinkConnectorConfig().getUserName(), + sinkConfig.getSinkConnectorConfig().getPassWord()); } diff --git a/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/SqlUtils.java b/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/SqlUtils.java index f6c4329e23..6d4ca03697 100644 --- a/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/SqlUtils.java +++ b/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/SqlUtils.java @@ -20,6 +20,7 @@ import static org.apache.eventmesh.connector.canal.ByteArrayConverter.SQL_BYTES; import static org.apache.eventmesh.connector.canal.SqlTimestampConverter.SQL_TIMESTAMP; +import com.mysql.cj.MysqlType; import org.apache.commons.beanutils.ConvertUtilsBean; import org.apache.commons.lang.StringUtils; @@ -30,12 +31,15 @@ import java.sql.Blob; import java.sql.Clob; import java.sql.Date; +import java.sql.JDBCType; +import java.sql.PreparedStatement; import java.sql.ResultSet; import java.sql.SQLException; import java.sql.Time; import java.sql.Timestamp; import java.sql.Types; import java.util.HashMap; +import java.util.List; import java.util.Map; import org.slf4j.Logger; @@ -109,6 +113,25 @@ public class SqlUtils { sqlTypeToJavaTypeMap.put(Types.CLOB, String.class); } + public static String genPrepareSqlOfInClause(int size) { + StringBuilder sql = new StringBuilder(); + sql.append("("); + for (int i = 0; i < size; i++) { + sql.append("?"); + if (i < size - 1) { + sql.append(","); + } + } + sql.append(")"); + return sql.toString(); + } + + public static void setInClauseParameters(PreparedStatement preparedStatement, List params) throws SQLException { + for (int i = 0; i < params.size(); i++) { + preparedStatement.setString(i + 1, params.get(i)); + } + } + public static String sqlValueToString(ResultSet rs, int index, int sqlType) throws SQLException { Class requiredType = sqlTypeToJavaTypeMap.get(sqlType); if (requiredType == null) { @@ -293,4 +316,9 @@ public static boolean isTextType(int sqlType) { || sqlType == Types.NCHAR || sqlType == Types.NVARCHAR || sqlType == Types.NCLOB || sqlType == Types.LONGNVARCHAR; } + + public static JDBCType toJDBCType(String connectorDataType) { + MysqlType mysqlType = MysqlType.getByName(connectorDataType); + return JDBCType.valueOf(mysqlType.getJdbcType()); + } } diff --git a/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/source/CanalFullProducer.java b/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/source/CanalFullProducer.java new file mode 100644 index 0000000000..9c3a6d07fb --- /dev/null +++ b/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/source/CanalFullProducer.java @@ -0,0 +1,325 @@ +package org.apache.eventmesh.connector.canal.source; + +import com.mysql.cj.MysqlType; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; +import org.apache.eventmesh.common.config.connector.rdb.canal.RdbColumnDefinition; +import org.apache.eventmesh.common.config.connector.rdb.canal.mysql.Constants; +import org.apache.eventmesh.common.config.connector.rdb.canal.mysql.MySQLTableDef; +import org.apache.eventmesh.common.exception.EventMeshException; +import org.apache.eventmesh.common.utils.JsonUtils; +import org.apache.eventmesh.connector.canal.source.position.TableFullPosition; +import org.apache.eventmesh.openconnect.offsetmgmt.api.data.ConnectRecord; +import org.locationtech.jts.io.WKBReader; + +import javax.sql.DataSource; +import java.math.BigDecimal; +import java.sql.Connection; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.time.Instant; +import java.time.LocalDate; +import java.time.LocalDateTime; +import java.time.LocalTime; +import java.time.ZoneId; +import java.time.format.DateTimeFormatter; +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicReference; +import java.util.concurrent.locks.LockSupport; + + + +@Slf4j +public class CanalFullProducer { + private BlockingQueue> queue; + private final DataSource dataSource; + private final MySQLTableDef tableDefinition; + private final TableFullPosition position; + private static final int LIMIT = 2048; + private final int flushSize; + private final AtomicReference choosePrimaryKey = new AtomicReference<>(null); + private static final DateTimeFormatter DATE_FORMATTER = DateTimeFormatter.ofPattern("yyyy-MM-dd"); + private static final DateTimeFormatter DATE_STAMP_FORMATTER = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss"); + private static final WKBReader WKB_READER = new wk + + public CanalFullProducer(BlockingQueue> queue, DataSource dataSource, + MySQLTableDef tableDefinition, TableFullPosition position, int flushSize) { + this.queue = queue; + this.dataSource = dataSource; + this.tableDefinition = tableDefinition; + this.position = position; + this.flushSize = flushSize; + } + + public void start(AtomicBoolean flag) { + boolean isNextPage = false; + ArrayList records = new ArrayList<>(); + while (flag.get()) { + String scanSql = generateScanSql(tableDefinition, !isNextPage); + log.info("scan sql is [{}] , cur position [{}]", scanSql, + JsonUtils.toJSONString(position.getCurPrimaryKeyCols())); + + try (Connection connection = dataSource.getConnection(); PreparedStatement statement = + connection.prepareStatement(scanSql, ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY)) { + statement.setFetchSize(Integer.MIN_VALUE); + setPrepareStatementValue(statement, position); + try (ResultSet resultSet = statement.executeQuery()) { + while (flag.get() && resultSet.next()) { + + + if (records.size() < flushSize) { + continue; + } + queue.put(records); + records = new ArrayList<>(); + } + } catch (InterruptedException ignore) { + } + } catch (SQLException e) { + log.error("create connection fail", e); + LockSupport.parkNanos(3000 * 1000L); + } + if (!isNextPage) { + isNextPage = true; + } + } + + } + + public Object readColumn(ResultSet rs, String col, MysqlType colType) throws Exception { + switch (colType) { + case TINYINT: + case SMALLINT: + case MEDIUMINT: + case INT: + Long uLong; + if (rs.wasNull()) { + return null; + } else { + uLong = rs.getLong(col); + } + if (uLong.compareTo(2147483647L) > 0) { + return uLong; + } + return uLong.intValue(); + case BIGINT: + String v = rs.getString(col); + if (v == null) { + return null; + } + BigDecimal uBigInt = new BigDecimal(v); + if (uBigInt.compareTo(BigDecimal.valueOf(Long.MAX_VALUE)) > 0) { + return uBigInt; + } + return uBigInt.longValue(); + case FLOAT: + case DOUBLE: + case DECIMAL: + return rs.getBigDecimal(col); + case DATE: + return rs.getObject(col, LocalDate.class); + case TIME: + return rs.getObject(col, LocalTime.class); + case DATETIME: + case TIMESTAMP: + return rs.getObject(col, LocalDateTime.class); + case YEAR: + if (rs.wasNull()) { + return null; + } + return rs.getInt(col); + case CHAR: + case VARCHAR: + case TINYTEXT: + case TEXT: + case MEDIUMTEXT: + case LONGTEXT: + case ENUM: + case SET: + case JSON: + return rs.getString(col); + case BIT: + case BINARY: + case VARBINARY: + case TINYBLOB: + case BLOB: + case MEDIUMBLOB: + case LONGBLOB: + return rs.getBytes(col); + case GEOMETRY: + String wkb = rs.getString(col); + if (wkb == null) { + return null; + } + return safeToGisWKT("0x" + wkb); + default: + return rs.getObject(col); + } + } + + private void refreshPosition() { + + } + + private void setPrepareStatementValue(PreparedStatement statement, TableFullPosition position) throws SQLException { + String colName = choosePrimaryKey.get(); + if (colName == null) { + return; + } + RdbColumnDefinition columnDefinition = tableDefinition.getColumnDefinitions().get(colName); + Object value = position.getCurPrimaryKeyCols().get(choosePrimaryKey); + String str; + switch (columnDefinition.getJdbcType()) { + case BIT: + case TINYINT: + case SMALLINT: + case INTEGER: + case BIGINT: + statement.setBigDecimal(1, new BigDecimal(String.valueOf(value))); + break; + case DECIMAL: + case FLOAT: + case DOUBLE: + case NUMERIC: + statement.setDouble(1, new BigDecimal(String.valueOf(value)).doubleValue()); + break; + case CHAR: + case VARCHAR: + case LONGNVARCHAR: + case NCHAR: + case NVARCHAR: + case LONGVARCHAR: + case CLOB: + case NCLOB: + statement.setString(1, String.valueOf(value)); + break; + case BLOB: + case VARBINARY: + case BINARY: + str = String.valueOf(value); + String hexStr = str; + if (str.startsWith("0x")) { + hexStr = str.substring(str.indexOf("0x")); + } + byte[] bytes = hex2bytes(hexStr); + statement.setBytes(1, bytes); + break; + case DATE: + Instant d; + if (value instanceof Long) { + Long val = (Long)value; + d = Instant.ofEpochMilli(val); + str = d.atZone(ZoneId.systemDefault()).toLocalDateTime().format(DATE_FORMATTER); + } else if (value instanceof Integer) { + Integer val = (Integer)value; + d = Instant.ofEpochMilli((long)val); + str = d.atZone(ZoneId.systemDefault()).toLocalDateTime().format(DATE_FORMATTER); + } else if (value instanceof String) { + str = (String) value; + } else { + if (!(value instanceof LocalDate)) { + throw new IllegalArgumentException("unsupported date class type:" + value.getClass().getSimpleName()); + } + str = ((LocalDate)value).format(DATE_FORMATTER); + } + statement.setString(1, str); + break; + case TIMESTAMP: + if (value instanceof String) { + str = (String)value; + } else { + if (!(value instanceof LocalDateTime)) { + throw new IllegalArgumentException("unsupported timestamp class type:" + value.getClass().getSimpleName()); + } + str = ((LocalDateTime)value).format(DATE_STAMP_FORMATTER); + } + statement.setString(1, str); + break; + default: + throw new EventMeshException(String.format("not support the primary key type [%s]", value.getClass())); + } + } + + public static byte[] hex2bytes(String hexStr) { + if (hexStr == null) + return null; + if (StringUtils.isBlank(hexStr)) { + return new byte[0]; + } + + if (hexStr.length() % 2 == 1) { + hexStr = "0" + hexStr; + } + + int count = hexStr.length() / 2; + byte[] ret = new byte[count]; + for (int i = 0; i < count; i++) { + int index = i * 2; + char c1 = hexStr.charAt(index); + char c2 = hexStr.charAt(index + 1); + if (c1 < '0' || c1 > 'F' || c2 < '0' || c2 > 'F') { + throw new EventMeshException(String.format("illegal byte [%s], [%s]" , c1, c2)); + } + ret[i] = (byte) ((byte)c1 << 4); + ret[i] = (byte) (ret[i] | (byte)(c2)); + } + return ret; + } + + private void generateQueryColumnsSql(StringBuilder builder, Collection rdbColDefs) { + if (rdbColDefs == null || rdbColDefs.isEmpty()) { + builder.append("*"); + return; + } + boolean first = true; + for (RdbColumnDefinition colDef : rdbColDefs) { + if (first) { + first = false; + } else { + builder.append(","); + } + builder.append(Constants.MySQLQuot); + builder.append(colDef.getName()); + builder.append(Constants.MySQLQuot); + } + } + + private String generateScanSql(MySQLTableDef tableDefinition, boolean isEquals) { + StringBuilder builder = new StringBuilder(); + builder.append("select "); + generateQueryColumnsSql(builder, tableDefinition.getColumnDefinitions().values()); + buildWhereSql(builder, tableDefinition, isEquals); + builder.append(" limit " + LIMIT); + return builder.toString(); + } + + private void buildWhereSql(StringBuilder builder, MySQLTableDef tableDefinition, boolean isEquals) { + String colName = null; + for (RdbColumnDefinition col : tableDefinition.getColumnDefinitions().values()) { + if (position.getCurPrimaryKeyCols().get(col.getName()) != null) { + colName = col.getName(); + choosePrimaryKey.set(colName); + break; + } + } + if (colName == null) { + throw new EventMeshException("not support only have one primary key of the timestamp type"); + } + builder.append(" where ") + .append(Constants.MySQLQuot) + .append(colName) + .append(Constants.MySQLQuot); + if (isEquals) { + builder.append(" >= ? "); + } else { + builder.append(" > ? "); + } + builder.append(" order by ").append(Constants.MySQLQuot).append(colName).append(Constants.MySQLQuot) + .append(" asc "); + } +} diff --git a/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/source/connector/CanalSourceConnector.java b/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/source/connector/CanalSourceConnector.java index efdb4e4c95..c1c381c91f 100644 --- a/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/source/connector/CanalSourceConnector.java +++ b/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/source/connector/CanalSourceConnector.java @@ -17,6 +17,7 @@ package org.apache.eventmesh.connector.canal.source.connector; +import com.alibaba.otter.canal.parse.inbound.TableMeta; import org.apache.eventmesh.common.config.connector.Config; import org.apache.eventmesh.common.config.connector.rdb.canal.CanalSourceConfig; import org.apache.eventmesh.connector.canal.source.table.RdbTableMgr; @@ -149,8 +150,7 @@ protected void startEventParserInternal(CanalEventParser parser, boolean isGroup return instance; } }); - tableMgr = RdbTableMgr.getInstance(); - tableMgr.init(sourceConfig.getSourceConnectorConfig()); + tableMgr = new RdbTableMgr(sourceConfig.getSourceConnectorConfig()); } private Canal buildCanal(CanalSourceConfig sourceConfig) { diff --git a/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/source/connector/CanalSourceFullConnector.java b/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/source/connector/CanalSourceFullConnector.java index 79481a69af..b7e15157f0 100644 --- a/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/source/connector/CanalSourceFullConnector.java +++ b/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/source/connector/CanalSourceFullConnector.java @@ -2,28 +2,71 @@ import lombok.extern.slf4j.Slf4j; import org.apache.eventmesh.common.AbstractComponent; +import org.apache.eventmesh.common.EventMeshThreadFactory; import org.apache.eventmesh.common.config.connector.Config; import org.apache.eventmesh.common.config.connector.rdb.canal.CanalSourceFullConfig; +import org.apache.eventmesh.common.config.connector.rdb.canal.JobRdbFullPosition; import org.apache.eventmesh.common.config.connector.rdb.canal.RdbDBDefinition; import org.apache.eventmesh.common.config.connector.rdb.canal.RdbTableDefinition; +import org.apache.eventmesh.common.config.connector.rdb.canal.SourceConnectorConfig; +import org.apache.eventmesh.common.exception.EventMeshException; +import org.apache.eventmesh.connector.canal.DatabaseConnection; +import org.apache.eventmesh.connector.canal.source.position.CanalFullPositionMgr; +import org.apache.eventmesh.connector.canal.source.table.RdbSimpleTable; +import org.apache.eventmesh.connector.canal.source.table.RdbTableMgr; import org.apache.eventmesh.openconnect.api.ConnectorCreateService; import org.apache.eventmesh.openconnect.api.connector.ConnectorContext; import org.apache.eventmesh.openconnect.api.connector.SourceConnectorContext; import org.apache.eventmesh.openconnect.api.source.Source; import org.apache.eventmesh.openconnect.offsetmgmt.api.data.ConnectRecord; +import javax.sql.DataSource; +import java.util.HashMap; import java.util.List; +import java.util.Map; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; @Slf4j public class CanalSourceFullConnector extends AbstractComponent implements Source, ConnectorCreateService { private CanalSourceFullConfig config; + private CanalFullPositionMgr positionMgr; + private RdbTableMgr tableMgr; + private ThreadPoolExecutor executor; + private final Map dataSources = new HashMap<>(); + private final BlockingQueue> queue = new LinkedBlockingQueue<>(); @Override protected void startup() throws Exception { + this.tableMgr.start(); + this.positionMgr.start(); + if (positionMgr.isFinished()) { + log.info("connector [{}] has finished the job", config.getConnectorConfig().getConnectorName()); + return; + } + executor = new ThreadPoolExecutor(config.getParallel(), config.getParallel(),0L, TimeUnit.MILLISECONDS, + new LinkedBlockingQueue<>(), new EventMeshThreadFactory("canal-source-full")); if (config.getConnectorConfig().getDatabases() != null) { for (RdbDBDefinition db : config.getConnectorConfig().getDatabases()) { for (RdbTableDefinition table : db.getTables()) { - log.info("[]"); + log.info("it will create producer of db [{}] table [{}]", db.getSchemaName(), table.getTableName()); + DataSource dataSource = dataSources.computeIfAbsent(db.getSchemaName(), + k -> DatabaseConnection.createDruidDataSource(null, config.getConnectorConfig().getUserName(), + config.getConnectorConfig().getPassWord())); + RdbSimpleTable simpleTable = new RdbSimpleTable(db.getSchemaName(), table.getTableName()); + JobRdbFullPosition position = positionMgr.getPosition(simpleTable); + if (position == null) { + throw new EventMeshException(String.format("db [%s] table [%s] have none position info", + db.getSchemaName(), table.getTableName())); + } + RdbTableDefinition tableDefinition = tableMgr.getTable(simpleTable); + if (tableDefinition == null) { + throw new EventMeshException(String.format("db [%s] table [%s] have none table definition info", + db.getSchemaName(), table.getTableName())); + } + } } } @@ -53,7 +96,8 @@ public void init(Config config) throws Exception { public void init(ConnectorContext connectorContext) throws Exception { SourceConnectorContext sourceConnectorContext = (SourceConnectorContext) connectorContext; this.config = (CanalSourceFullConfig) sourceConnectorContext.getSourceConfig(); - + this.tableMgr = new RdbTableMgr(config.getConnectorConfig()); + this.positionMgr = new CanalFullPositionMgr(config, tableMgr); } @Override diff --git a/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/source/position/CanalFullPositionMgr.java b/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/source/position/CanalFullPositionMgr.java index 70611da632..8003ef1784 100644 --- a/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/source/position/CanalFullPositionMgr.java +++ b/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/source/position/CanalFullPositionMgr.java @@ -1,24 +1,46 @@ package org.apache.eventmesh.connector.canal.source.position; +import com.alibaba.druid.pool.DruidDataSource; import lombok.AllArgsConstructor; import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; import org.apache.eventmesh.common.AbstractComponent; import org.apache.eventmesh.common.config.connector.rdb.canal.CanalSourceFullConfig; +import org.apache.eventmesh.common.config.connector.rdb.canal.RdbColumnDefinition; import org.apache.eventmesh.common.config.connector.rdb.canal.RdbDBDefinition; -import org.apache.eventmesh.common.config.connector.rdb.canal.RdbFullPosition; +import org.apache.eventmesh.common.config.connector.rdb.canal.JobRdbFullPosition; import org.apache.eventmesh.common.config.connector.rdb.canal.RdbTableDefinition; +import org.apache.eventmesh.common.config.connector.rdb.canal.mysql.Constants; +import org.apache.eventmesh.common.config.connector.rdb.canal.mysql.MySQLTableDef; import org.apache.eventmesh.common.remote.offset.RecordPosition; +import org.apache.eventmesh.common.utils.JsonUtils; +import org.apache.eventmesh.connector.canal.DatabaseConnection; import org.apache.eventmesh.connector.canal.source.table.RdbSimpleTable; +import org.apache.eventmesh.connector.canal.source.table.RdbTableMgr; +import javax.sql.DataSource; +import java.sql.JDBCType; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.LinkedHashMap; +import java.util.List; import java.util.Map; import java.util.concurrent.ScheduledThreadPoolExecutor; -@AllArgsConstructor @Slf4j public class CanalFullPositionMgr extends AbstractComponent { + private CanalSourceFullConfig config; private ScheduledThreadPoolExecutor executor; - private Map positions; + private Map positions = new LinkedHashMap<>(); + private RdbTableMgr tableMgr; + + public CanalFullPositionMgr(CanalSourceFullConfig config, RdbTableMgr tableMgr) { + this.config = config; + this.tableMgr = tableMgr; + } @Override protected void startup() throws Exception { @@ -33,31 +55,166 @@ protected void startup() throws Exception { return thread; }, new ScheduledThreadPoolExecutor.DiscardOldestPolicy()); if (config.getStartPosition() != null) { - for (RecordPosition recordPosition : config.getStartPosition()) { + processPositions(config.getStartPosition()); + } + + } + public JobRdbFullPosition getPosition(RdbSimpleTable table) { + return positions.get(table); + } + + public boolean isFinished() { + for (JobRdbFullPosition position : positions.values()) { + if (!position.isFinished()) { + log.info("schema [{}] table [{}] is not finish", position.getSchema(), position.getTableName()); + return false; } } - + return true; } - private void processPositions(CanalSourceFullConfig config) { + private void processPositions(List startPosition) throws SQLException { for (RdbDBDefinition database : config.getConnectorConfig().getDatabases()) { for (RdbTableDefinition table : database.getTables()) { - log.info("init position of data [{}] table [{}]", database.getSchema(), table.getTableName()); - RdbSimpleTable simpleTable = new RdbSimpleTable(database.getSchema(), table.getTableName()); - RdbFullPosition recordPosition = positions.get(simpleTable); + RdbSimpleTable simpleTable = new RdbSimpleTable(database.getSchemaName(), table.getTableName()); + RdbTableDefinition tableDefinition; + if ((tableDefinition = tableMgr.getTable(simpleTable)) == null) { + log.error("db [{}] table [{}] definition is null", database.getSchemaName(), table.getTableName()); + continue; + } + log.info("init position of data [{}] table [{}]", database.getSchemaName(), table.getTableName()); + + JobRdbFullPosition recordPosition = positions.get(simpleTable); if (recordPosition == null || !recordPosition.isFinished()) { - positions.put(simpleTable,initPosition(config, database.getSchema(), table.getTableName(), recordPosition)); + try (DruidDataSource dataSource = DatabaseConnection.createDruidDataSource(null, + config.getConnectorConfig().getUserName(), config.getConnectorConfig().getPassWord())) { + positions.put(simpleTable, initPosition(dataSource, (MySQLTableDef)tableDefinition, recordPosition)); + } } } } } - private RdbFullPosition initPosition(CanalSourceFullConfig config, String db, String table, - RdbFullPosition recordPosition) { + private JobRdbFullPosition initPosition(DataSource dataSource, MySQLTableDef tableDefinition, + JobRdbFullPosition recordPosition) throws SQLException { + TableFullPosition position = new TableFullPosition(); + Map preMinPrimaryKeys = new LinkedHashMap<>(); + Map preMaxPrimaryKeys = new LinkedHashMap<>(); + for (String pk : tableDefinition.getPrimaryKeys()) { + Object min = fetchMinPrimaryKey(dataSource, tableDefinition, preMinPrimaryKeys, pk); + Object max = fetchMaxPrimaryKey(dataSource, tableDefinition, preMaxPrimaryKeys, pk); + preMinPrimaryKeys.put(pk, min); + preMaxPrimaryKeys.put(pk, max); + position.getCurPrimaryKeyCols().put(pk, min); + position.getMinPrimaryKeyCols().put(pk, min); + position.getMaxPrimaryKeyCols().put(pk, max); + } + long rowCount = queryCurTableRowCount(dataSource, tableDefinition); + JobRdbFullPosition jobRdbFullPosition = new JobRdbFullPosition(); + if (recordPosition != null ) { + if (StringUtils.isNotBlank(recordPosition.getCurPrimaryKey())) { + TableFullPosition record = JsonUtils.parseObject(recordPosition.getCurPrimaryKey(), TableFullPosition.class); + if (record != null && record.getCurPrimaryKeyCols() != null && !record.getCurPrimaryKeyCols().isEmpty()) { + position.setCurPrimaryKeyCols(record.getCurPrimaryKeyCols()); + } + } + } + jobRdbFullPosition.setSchema(tableDefinition.getSchemaName()); + jobRdbFullPosition.setTableName(tableDefinition.getTableName()); + jobRdbFullPosition.setMaxCount(rowCount); + jobRdbFullPosition.setCurPrimaryKey(JsonUtils.toJSONString(position.getCurPrimaryKeyCols())); + return jobRdbFullPosition; + } + + + private long queryCurTableRowCount(DataSource datasource, MySQLTableDef tableDefinition) throws SQLException { + String sql = + "select AVG_ROW_LENGTH,DATA_LENGTH from information_schema.TABLES where TABLE_SCHEMA=" + Constants.MySQLQuot + tableDefinition.getSchemaName() + Constants.MySQLQuot + " and TABLE_NAME="+ Constants.MySQLQuot + tableDefinition.getTableName() + Constants.MySQLQuot; + try (Statement statement = datasource.getConnection().createStatement(); ResultSet resultSet = statement.executeQuery(sql)) { + long result = 0L; + if (resultSet.next()) { + long avgRowLength = resultSet.getLong("AVG_ROW_LENGTH"); + long dataLength = resultSet.getLong("DATA_LENGTH"); + result = dataLength / avgRowLength; + } + return result; + } + } + + private void appendPrePrimaryKey(Map preMap, StringBuilder sql) { + if (preMap != null && !preMap.isEmpty()) { + sql.append(" WHERE "); + boolean first = true; + for (Map.Entry entry : preMap.entrySet()) { + if (first) { + first = false; + } else { + sql.append(" AND "); + } + sql.append(Constants.MySQLQuot).append(entry.getKey()).append(Constants.MySQLQuot).append("=?"); + } + } + } + + private void setValue2Statement(PreparedStatement ps, Map preMap, MySQLTableDef tableDefinition) throws SQLException { + if (preMap != null && !preMap.isEmpty()) { + int index = 1; + for (Map.Entry entry : preMap.entrySet()) { + RdbColumnDefinition def = tableDefinition.getColumnDefinitions().get(entry.getKey()); + ps.setObject(index, entry.getValue(), def.getJdbcType().getVendorTypeNumber()); + ++index; + } + } + } + + private Object fetchMinPrimaryKey(DataSource dataSource, MySQLTableDef tableDefinition, + Map prePrimary, String curPrimaryKeyCol) throws SQLException { + StringBuilder builder = new StringBuilder(); + builder.append("SELECT MIN(").append(Constants.MySQLQuot).append(curPrimaryKeyCol).append(Constants.MySQLQuot).append(") min_primary_key FROM") + .append(Constants.MySQLQuot).append(tableDefinition.getSchemaName()).append(Constants.MySQLQuot).append(".").append(Constants.MySQLQuot).append(tableDefinition.getTableName()).append(Constants.MySQLQuot); + appendPrePrimaryKey(prePrimary, builder); + String sql = builder.toString(); + try (PreparedStatement statement = dataSource.getConnection().prepareStatement(sql)){ + setValue2Statement(statement, prePrimary, tableDefinition); + try (ResultSet resultSet = statement.executeQuery()) { + if (resultSet.next()) { + RdbColumnDefinition columnDefinition = tableDefinition.getColumnDefinitions().get(curPrimaryKeyCol); + if (columnDefinition.getJdbcType() == JDBCType.TIMESTAMP) { + return resultSet.getString("min_primary_key"); + } else { + return resultSet.getObject("min_primary_key"); + } + } + } + } + return null; + } + + private Object fetchMaxPrimaryKey(DataSource dataSource, MySQLTableDef tableDefinition, + Map prePrimary, String curPrimaryKeyCol) throws SQLException { + StringBuilder builder = new StringBuilder(); + builder.append("SELECT MAX(").append(Constants.MySQLQuot).append(curPrimaryKeyCol).append(Constants.MySQLQuot).append(") max_primary_key FROM") + .append(Constants.MySQLQuot).append(tableDefinition.getSchemaName()).append(Constants.MySQLQuot).append(".").append(Constants.MySQLQuot).append(tableDefinition.getTableName()).append(Constants.MySQLQuot); + appendPrePrimaryKey(prePrimary, builder); + String sql = builder.toString(); + try (PreparedStatement statement = dataSource.getConnection().prepareStatement(sql)){ + setValue2Statement(statement, prePrimary, tableDefinition); + try (ResultSet resultSet = statement.executeQuery()) { + if (resultSet.next()) { + RdbColumnDefinition columnDefinition = tableDefinition.getColumnDefinitions().get(curPrimaryKeyCol); + if (columnDefinition.getJdbcType() == JDBCType.TIMESTAMP) { + return resultSet.getString("max_primary_key"); + } else { + return resultSet.getObject("max_primary_key"); + } + } + } + } return null; } + @Override protected void shutdown() throws Exception { diff --git a/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/source/position/TableFullPosition.java b/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/source/position/TableFullPosition.java new file mode 100644 index 0000000000..b4b30e7a24 --- /dev/null +++ b/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/source/position/TableFullPosition.java @@ -0,0 +1,13 @@ +package org.apache.eventmesh.connector.canal.source.position; + +import lombok.Data; + +import java.util.LinkedHashMap; +import java.util.Map; + +@Data +public class TableFullPosition { + private Map curPrimaryKeyCols = new LinkedHashMap<>(); + private Map minPrimaryKeyCols = new LinkedHashMap<>(); + private Map maxPrimaryKeyCols = new LinkedHashMap<>(); +} diff --git a/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/source/table/RdbTableMgr.java b/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/source/table/RdbTableMgr.java index c7673880e7..6cacc56122 100644 --- a/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/source/table/RdbTableMgr.java +++ b/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/source/table/RdbTableMgr.java @@ -1,57 +1,160 @@ package org.apache.eventmesh.connector.canal.source.table; -import lombok.AllArgsConstructor; +import com.alibaba.druid.pool.DruidDataSource; +import com.mysql.cj.MysqlType; import lombok.extern.slf4j.Slf4j; import org.apache.eventmesh.common.AbstractComponent; +import org.apache.eventmesh.common.config.connector.rdb.canal.RdbColumnDefinition; import org.apache.eventmesh.common.config.connector.rdb.canal.RdbDBDefinition; import org.apache.eventmesh.common.config.connector.rdb.canal.RdbTableDefinition; import org.apache.eventmesh.common.config.connector.rdb.canal.SourceConnectorConfig; +import org.apache.eventmesh.common.config.connector.rdb.canal.mysql.MySQLColumnDef; +import org.apache.eventmesh.common.config.connector.rdb.canal.mysql.MySQLTableDef; +import org.apache.eventmesh.connector.canal.DatabaseConnection; +import org.apache.eventmesh.connector.canal.SqlUtils; +import javax.sql.DataSource; +import java.sql.JDBCType; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; import java.util.HashMap; +import java.util.HashSet; +import java.util.LinkedHashMap; +import java.util.LinkedList; +import java.util.List; import java.util.Map; -import java.util.concurrent.atomic.AtomicBoolean; +import java.util.stream.Collectors; /** * Description: */ @Slf4j -@AllArgsConstructor public class RdbTableMgr extends AbstractComponent { private SourceConnectorConfig config; - private final Map tables = new HashMap<>(); + private final Map tables = new HashMap<>(); - public static String generate(String ...params) { - return String.join("@", params); - } - - private RdbTableMgr(){ - } - - private static class RdbTableMgrHolder { - private static final RdbTableMgr INSTANCE = new RdbTableMgr(); - } - - public static RdbTableMgr getInstance() { - return RdbTableMgrHolder.INSTANCE; + public RdbTableMgr(SourceConnectorConfig config) { + this.config = config; } - public void init(SourceConnectorConfig config) { - this.config = config; + public RdbTableDefinition getTable(String schema, String tableName) { + return getTable(new RdbSimpleTable(schema, tableName)); } - public RdbTableDefinition getTable(String dbName, String tableName) { - return tables.get(generate(dbName, tableName)); + public RdbTableDefinition getTable(RdbSimpleTable table) { + return tables.get(table); } @Override protected void startup() throws Exception { if (config != null && config.getDatabases() != null) { for (RdbDBDefinition db : config.getDatabases()) { - for (RdbTableDefinition table : db.getTables()) { - tables.put(generate(db.getSchema(), table.getTableName()), table); + if (db.getTables() == null) { + log.warn("init db [{}] position, but it's tables are null", db.getSchemaName()); + continue; } + try (DruidDataSource dataSource = DatabaseConnection.createDruidDataSource(null, + config.getUserName(), config.getPassWord())) { + List tableNames = + db.getTables().stream().map(RdbTableDefinition::getTableName).collect(Collectors.toList()); + Map> primaryKeys = queryTablePrimaryKey(dataSource, tableNames); + Map> columns = queryColumns(dataSource, tableNames); + for (RdbTableDefinition table : db.getTables()) { + MySQLTableDef mysqlTable = new MySQLTableDef(); + mysqlTable.setSchemaName(db.getSchemaName()); + mysqlTable.setTableName(table.getTableName()); + if (primaryKeys == null || primaryKeys.isEmpty() || primaryKeys.get(table.getTableName()) == null) { + log.warn("init db [{}] table [{}] info, and primary keys are empty", db.getSchemaName(), + table.getTableName()); + } else { + mysqlTable.setPrimaryKeys(new HashSet<>(primaryKeys.get(table.getTableName()))); + } + if (columns == null || columns.isEmpty() || columns.get(table.getTableName()) == null) { + log.warn("init db [{}] table [{}] info, and columns are empty", db.getSchemaName(), + table.getTableName()); + } else { + LinkedHashMap cols = new LinkedHashMap<>(); + columns.get(table.getTableName()).forEach(x -> cols.put(x.getName(), x)); + mysqlTable.setColumnDefinitions(cols); + } + + tables.put(new RdbSimpleTable(db.getSchemaName(), table.getTableName()), mysqlTable); + } + } catch (Exception e) { + log.error("init db [{}] tables info fail", db.getSchemaName(), e); + } + + } + } + } + + private Map> queryTablePrimaryKey(DruidDataSource dataSource, List tables) throws SQLException { + Map> primaryKeys = new LinkedHashMap<>(); + String prepareTables = SqlUtils.genPrepareSqlOfInClause(tables.size()); + String sql = "select L.TABLE_NAME,L.COLUMN_NAME,R.CONSTRAINT_TYPE from " + + "INFORMATION_SCHEMA.KEY_COLUMN_USAGE L left join INFORMATION_SCHEMA.TABLE_CONSTRAINTS R on C" + + ".TABLE_SCHEMA = R.TABLE_SCHEMA and L.TABLE_NAME = R.TABLE_NAME and L.CONSTRAINT_CATALOG = R" + + ".CONSTRAINT_CATALOG and L.CONSTRAINT_SCHEMA = R.CONSTRAINT_SCHEMA and L.CONSTRAINT_NAME = R" + + ".CONSTRAINT_NAME where L.TABLE_SCHEMA = ? and L.TABLE_NAME in " + prepareTables + " and R" + + ".CONSTRAINT_TYPE IN ('PRIMARY KEY') order by L.ORDINAL_POSITION asc"; + try (PreparedStatement statement = dataSource.getConnection().prepareStatement(sql)) { + SqlUtils.setInClauseParameters(statement, tables); + ResultSet resultSet = statement.executeQuery(); + if (resultSet == null) { + return null; + } + while (resultSet.next()) { + String tableName = resultSet.getString("TABLE_NAME"); + String colName = resultSet.getString("COLUMN_NAME"); + primaryKeys.compute(tableName, (k, v) -> { + if (v == null) { + v = new LinkedList<>(); + } + v.add(colName); + return v; + }); + } + resultSet.close(); + } + return primaryKeys; + } + + private Map> queryColumns(DataSource dataSource, List tables) throws SQLException { + String prepareTables = SqlUtils.genPrepareSqlOfInClause(tables.size()); + String sql = "select TABLE_SCHEMA,TABLE_NAME,COLUMN_NAME,IS_NULLABLE,DATA_TYPE,CHARACTER_MAXIMUM_LENGTH," + + "CHARACTER_OCTET_LENGTH,NUMERIC_SCALE,NUMERIC_PRECISION,DATETIME_PRECISION,CHARACTER_SET_NAME," + + "COLLATION_NAME,COLUMN_TYPE,COLUMN_DEFAULT,COLUMN_COMMENT,ORDINAL_POSITION,EXTRA from " + + "INFORMATION_SCHEMA.COLUMNS where TABLE_SCHEMA = ? and TABLE_NAME in " + prepareTables + " order by " + + "ORDINAL_POSITION asc"; + Map> cols = new LinkedHashMap<>(); + try (PreparedStatement statement = dataSource.getConnection().prepareStatement(sql)) { + SqlUtils.setInClauseParameters(statement, tables); + ResultSet resultSet = statement.executeQuery(); + if (resultSet == null) { + return null; + } + while (resultSet.next()) { + String tableName = resultSet.getString("TABLE_NAME"); + String colName = resultSet.getString("COLUMN_NAME"); + String dataType = resultSet.getString("DATA_TYPE"); + JDBCType jdbcType = SqlUtils.toJDBCType(dataType); + MysqlType type = MysqlType.getByName(dataType); + MySQLColumnDef col = new MySQLColumnDef(); + col.setJdbcType(jdbcType); + col.setType(type); + col.setName(colName); + cols.compute(tableName, (k, v) -> { + if (v == null) { + v = new LinkedList<>(); + } + v.add(col); + return v; + }); } + resultSet.close(); } + return cols; } @Override From f29dd7be70b13d22b0ce6709272095ccfd499062 Mon Sep 17 00:00:00 2001 From: sodaRyCN <757083350@qq.com> Date: Thu, 11 Jul 2024 16:49:14 +0800 Subject: [PATCH 06/15] close to finish full read and begin full write --- .../connector/rdb/canal/CanalMySQLType.java | 81 ++++---- .../rdb/canal/CanalSinkFullConfig.java | 12 ++ .../rdb/canal/CanalSourceFullConfig.java | 1 + .../rdb/canal/JobRdbFullPosition.java | 5 +- .../offset/canal/CanalFullRecordOffset.java | 18 ++ .../canal/CanalFullRecordPartition.java | 37 ++++ .../connector/canal/DatabaseConnection.java | 4 +- .../eventmesh/connector/canal/SqlUtils.java | 16 +- .../connector/CanalSinkFullConnector.java | 59 ++++++ .../canal/source/CanalFullProducer.java | 188 +++++++++++++----- .../connector/canal/source/EntryParser.java | 32 +-- .../connector/CanalSourceConnector.java | 5 +- .../connector/CanalSourceFullConnector.java | 76 +++++-- .../source/position/CanalFullPositionMgr.java | 89 ++++++--- .../canal/source/table/RdbTableMgr.java | 78 ++++---- 15 files changed, 501 insertions(+), 200 deletions(-) create mode 100644 eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/rdb/canal/CanalSinkFullConfig.java create mode 100644 eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/offset/canal/CanalFullRecordOffset.java create mode 100644 eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/offset/canal/CanalFullRecordPartition.java create mode 100644 eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/sink/connector/CanalSinkFullConnector.java diff --git a/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/rdb/canal/CanalMySQLType.java b/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/rdb/canal/CanalMySQLType.java index 969e631e93..257822810e 100644 --- a/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/rdb/canal/CanalMySQLType.java +++ b/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/rdb/canal/CanalMySQLType.java @@ -2,7 +2,8 @@ import com.mysql.cj.MysqlType; -import java.sql.JDBCType; +import java.util.HashMap; +import java.util.Map; public enum CanalMySQLType { BIT("BIT"), @@ -12,37 +13,38 @@ public enum CanalMySQLType { INT("INT"), BIGINT("BIGINT"), DECIMAL("DECIMAL"), - FLOAT("FLOAT", JDBCType.REAL), - DOUBLE("DOUBLE", JDBCType.DOUBLE), - DATE("DATE", JDBCType.DATE), - DATETIME("DATETIME", JDBCType.TIMESTAMP), - TIMESTAMP("TIMESTAMP", JDBCType.TIMESTAMP), - TIME("TIME", JDBCType.TIME), - YEAR("YEAR", JDBCType.DATE), - CHAR("CHAR", JDBCType.CHAR), - VARCHAR("VARCHAR", JDBCType.VARCHAR), - BINARY("BINARY", JDBCType.BINARY), - VARBINARY("VARBINARY", JDBCType.VARBINARY), - TINYBLOB("TINYBLOB", JDBCType.VARBINARY), - BLOB("BLOB", JDBCType.LONGVARBINARY), - MEDIUMBLOB("MEDIUMBLOB", JDBCType.LONGVARBINARY), - LONGBLOB("LONGBLOB", JDBCType.LONGVARBINARY), - TINYTEXT("TINYTEXT", JDBCType.VARCHAR), - TEXT("TEXT", JDBCType.LONGVARCHAR), - MEDIUMTEXT("MEDIUMTEXT", JDBCType.LONGVARCHAR), - LONGTEXT("LONGTEXT", JDBCType.LONGVARCHAR), - ENUM("ENUM", JDBCType.CHAR), - SET("SET", JDBCType.CHAR), - JSON("JSON", JDBCType.LONGVARCHAR), - GEOMETRY("GEOMETRY", JDBCType.BINARY), - POINT("POINT", JDBCType.BINARY), - LINESTRING("LINESTRING", JDBCType.BINARY), - POLYGON("POLYGON", JDBCType.BINARY), - MULTIPOINT("MULTIPOINT", JDBCType.BINARY), - GEOMETRY_COLLECTION("GEOMETRYCOLLECTION", JDBCType.BINARY), - GEOM_COLLECTION("GEOMCOLLECTION", JDBCType.BINARY), - MULTILINESTRING("MULTILINESTRING", JDBCType.BINARY), - MULTIPOLYGON("MULTIPOLYGON", JDBCType.BINARY); + FLOAT("FLOAT"), + DOUBLE("DOUBLE"), + DATE("DATE"), + DATETIME("DATETIME"), + TIMESTAMP("TIMESTAMP"), + TIME("TIME"), + YEAR("YEAR"), + CHAR("CHAR"), + VARCHAR("VARCHAR"), + BINARY("BINARY"), + VARBINARY("VARBINARY"), + TINYBLOB("TINYBLOB"), + BLOB("BLOB"), + MEDIUMBLOB("MEDIUMBLOB"), + LONGBLOB("LONGBLOB"), + TINYTEXT("TINYTEXT"), + TEXT("TEXT"), + MEDIUMTEXT("MEDIUMTEXT"), + LONGTEXT("LONGTEXT"), + ENUM("ENUM"), + SET("SET"), + JSON("JSON"), + GEOMETRY("GEOMETRY"), + // MysqlType not include the following type + POINT("POINT"), + LINESTRING("LINESTRING"), + POLYGON("POLYGON"), + MULTIPOINT("MULTIPOINT"), + GEOMETRY_COLLECTION("GEOMETRYCOLLECTION"), + GEOM_COLLECTION("GEOMCOLLECTION"), + MULTILINESTRING("MULTILINESTRING"), + MULTIPOLYGON("MULTIPOLYGON"); private final String codeKey; private final MysqlType mysqlType; @@ -51,13 +53,18 @@ public enum CanalMySQLType { this.codeKey = codeKey; this.mysqlType = MysqlType.getByName(codeKey); } - - public static CanalMySQLType valueOfCode(String code) { + private static final Map TYPES = new HashMap<>(); + static { CanalMySQLType[] values = values(); for (CanalMySQLType tableType : values) { - if (tableType.codeKey.equalsIgnoreCase(code)) { - return tableType; - } + TYPES.put(tableType.codeKey, tableType); + } + } + + public static CanalMySQLType valueOfCode(String code) { + CanalMySQLType type = TYPES.get(code.toUpperCase()); + if (type != null) { + return type; } switch (MysqlType.getByName(code)) { case BOOLEAN: diff --git a/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/rdb/canal/CanalSinkFullConfig.java b/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/rdb/canal/CanalSinkFullConfig.java new file mode 100644 index 0000000000..5a6deb77cd --- /dev/null +++ b/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/rdb/canal/CanalSinkFullConfig.java @@ -0,0 +1,12 @@ +package org.apache.eventmesh.common.config.connector.rdb.canal; + +import lombok.Data; +import lombok.EqualsAndHashCode; +import org.apache.eventmesh.common.config.connector.SinkConfig; + + +@Data +@EqualsAndHashCode(callSuper = true) +public class CanalSinkFullConfig extends SinkConfig { + +} diff --git a/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/rdb/canal/CanalSourceFullConfig.java b/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/rdb/canal/CanalSourceFullConfig.java index fcfa6a0e92..6508e49fdb 100644 --- a/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/rdb/canal/CanalSourceFullConfig.java +++ b/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/rdb/canal/CanalSourceFullConfig.java @@ -13,4 +13,5 @@ public class CanalSourceFullConfig extends SourceConfig { private SourceConnectorConfig connectorConfig; private List startPosition; private int parallel; + private int flushSize; } diff --git a/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/rdb/canal/JobRdbFullPosition.java b/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/rdb/canal/JobRdbFullPosition.java index b4aeffd721..5f0f5326ed 100644 --- a/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/rdb/canal/JobRdbFullPosition.java +++ b/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/rdb/canal/JobRdbFullPosition.java @@ -1,15 +1,18 @@ package org.apache.eventmesh.common.config.connector.rdb.canal; import lombok.Data; +import lombok.ToString; import java.math.BigDecimal; @Data +@ToString public class JobRdbFullPosition { private String jobId; private String schema; private String tableName; - private String curPrimaryKey; + private String primaryKeyRecords; private long maxCount; private boolean finished; + private BigDecimal percent; } diff --git a/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/offset/canal/CanalFullRecordOffset.java b/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/offset/canal/CanalFullRecordOffset.java new file mode 100644 index 0000000000..f1f3a7c132 --- /dev/null +++ b/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/offset/canal/CanalFullRecordOffset.java @@ -0,0 +1,18 @@ +package org.apache.eventmesh.common.remote.offset.canal; + +import lombok.Data; +import lombok.EqualsAndHashCode; +import lombok.ToString; +import org.apache.eventmesh.common.config.connector.rdb.canal.JobRdbFullPosition; +import org.apache.eventmesh.common.remote.offset.RecordOffset; + +@Data +@EqualsAndHashCode(callSuper = true) +@ToString +public class CanalFullRecordOffset extends RecordOffset { + private JobRdbFullPosition position; + @Override + public Class getRecordOffsetClass() { + return CanalFullRecordOffset.class; + } +} diff --git a/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/offset/canal/CanalFullRecordPartition.java b/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/offset/canal/CanalFullRecordPartition.java new file mode 100644 index 0000000000..a325444be5 --- /dev/null +++ b/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/offset/canal/CanalFullRecordPartition.java @@ -0,0 +1,37 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.common.remote.offset.canal; + +import lombok.Data; +import lombok.EqualsAndHashCode; +import lombok.ToString; +import org.apache.eventmesh.common.remote.offset.RecordPartition; + + +@Data +@ToString +@EqualsAndHashCode(callSuper = true) +public class CanalFullRecordPartition extends RecordPartition { + private String schema; + private String table; + + @Override + public Class getRecordPartitionClass() { + return CanalFullRecordPartition.class; + } +} diff --git a/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/DatabaseConnection.java b/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/DatabaseConnection.java index 00f9693be3..dc576186ea 100644 --- a/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/DatabaseConnection.java +++ b/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/DatabaseConnection.java @@ -18,14 +18,13 @@ package org.apache.eventmesh.connector.canal; +import com.alibaba.druid.pool.DruidDataSource; import org.apache.eventmesh.common.config.connector.rdb.canal.CanalSinkConfig; import org.apache.eventmesh.common.config.connector.rdb.canal.CanalSourceConfig; import java.sql.Connection; import java.sql.SQLException; -import com.alibaba.druid.pool.DruidDataSource; - public class DatabaseConnection { public static DruidDataSource sourceDataSource; @@ -42,6 +41,7 @@ public static DruidDataSource createDruidDataSource(String url, String UserName, dataSource.setUsername(UserName); dataSource.setPassword(passWord); dataSource.setInitialSize(5); + dataSource.setDriverClassName("com.mysql.cj.jdbc.Driver"); dataSource.setMinIdle(5); dataSource.setMaxActive(20); dataSource.setMaxWait(60000); diff --git a/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/SqlUtils.java b/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/SqlUtils.java index 6d4ca03697..ed72a9eb4d 100644 --- a/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/SqlUtils.java +++ b/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/SqlUtils.java @@ -17,12 +17,11 @@ package org.apache.eventmesh.connector.canal; -import static org.apache.eventmesh.connector.canal.ByteArrayConverter.SQL_BYTES; -import static org.apache.eventmesh.connector.canal.SqlTimestampConverter.SQL_TIMESTAMP; - import com.mysql.cj.MysqlType; import org.apache.commons.beanutils.ConvertUtilsBean; import org.apache.commons.lang.StringUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import java.io.UnsupportedEncodingException; import java.math.BigDecimal; @@ -42,8 +41,8 @@ import java.util.List; import java.util.Map; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; +import static org.apache.eventmesh.connector.canal.ByteArrayConverter.SQL_BYTES; +import static org.apache.eventmesh.connector.canal.SqlTimestampConverter.SQL_TIMESTAMP; public class SqlUtils { @@ -127,8 +126,13 @@ public static String genPrepareSqlOfInClause(int size) { } public static void setInClauseParameters(PreparedStatement preparedStatement, List params) throws SQLException { + setInClauseParameters(preparedStatement, 0, params); + } + + public static void setInClauseParameters(PreparedStatement preparedStatement, int paramIndexStart, + List params) throws SQLException { for (int i = 0; i < params.size(); i++) { - preparedStatement.setString(i + 1, params.get(i)); + preparedStatement.setString(paramIndexStart + i, params.get(i)); } } diff --git a/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/sink/connector/CanalSinkFullConnector.java b/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/sink/connector/CanalSinkFullConnector.java new file mode 100644 index 0000000000..a43f6a4413 --- /dev/null +++ b/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/sink/connector/CanalSinkFullConnector.java @@ -0,0 +1,59 @@ +package org.apache.eventmesh.connector.canal.sink.connector; + +import org.apache.eventmesh.common.config.connector.Config; +import org.apache.eventmesh.common.config.connector.rdb.canal.CanalSinkFullConfig; +import org.apache.eventmesh.openconnect.api.ConnectorCreateService; +import org.apache.eventmesh.openconnect.api.connector.ConnectorContext; +import org.apache.eventmesh.openconnect.api.connector.SinkConnectorContext; +import org.apache.eventmesh.openconnect.api.sink.Sink; +import org.apache.eventmesh.openconnect.offsetmgmt.api.data.ConnectRecord; + +import java.util.List; + +public class CanalSinkFullConnector implements Sink, ConnectorCreateService { + private CanalSinkFullConfig config; + @Override + public void start() throws Exception { + + } + + @Override + public void stop() throws Exception { + + } + + @Override + public Sink create() { + return new CanalSinkFullConnector(); + } + + @Override + public Class configClass() { + return CanalSinkFullConfig.class; + } + + @Override + public void init(Config config) throws Exception { + this.config = (CanalSinkFullConfig) config; + } + + @Override + public void init(ConnectorContext connectorContext) throws Exception { + this.config = (CanalSinkFullConfig)((SinkConnectorContext)connectorContext).getSinkConfig(); + } + + @Override + public void commit(ConnectRecord record) { + + } + + @Override + public String name() { + return null; + } + + @Override + public void put(List sinkRecords) { + + } +} diff --git a/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/source/CanalFullProducer.java b/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/source/CanalFullProducer.java index 9c3a6d07fb..979346512c 100644 --- a/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/source/CanalFullProducer.java +++ b/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/source/CanalFullProducer.java @@ -1,15 +1,20 @@ package org.apache.eventmesh.connector.canal.source; -import com.mysql.cj.MysqlType; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; +import org.apache.eventmesh.common.config.connector.rdb.canal.CanalMySQLType; +import org.apache.eventmesh.common.config.connector.rdb.canal.JobRdbFullPosition; import org.apache.eventmesh.common.config.connector.rdb.canal.RdbColumnDefinition; import org.apache.eventmesh.common.config.connector.rdb.canal.mysql.Constants; +import org.apache.eventmesh.common.config.connector.rdb.canal.mysql.MySQLColumnDef; import org.apache.eventmesh.common.config.connector.rdb.canal.mysql.MySQLTableDef; import org.apache.eventmesh.common.exception.EventMeshException; +import org.apache.eventmesh.common.remote.offset.canal.CanalFullRecordOffset; +import org.apache.eventmesh.common.remote.offset.canal.CanalFullRecordPartition; import org.apache.eventmesh.common.utils.JsonUtils; import org.apache.eventmesh.connector.canal.source.position.TableFullPosition; import org.apache.eventmesh.openconnect.offsetmgmt.api.data.ConnectRecord; +import org.locationtech.jts.geom.GeometryFactory; import org.locationtech.jts.io.WKBReader; import javax.sql.DataSource; @@ -26,14 +31,16 @@ import java.time.format.DateTimeFormatter; import java.util.ArrayList; import java.util.Collection; +import java.util.LinkedHashMap; +import java.util.LinkedList; import java.util.List; +import java.util.Map; import java.util.concurrent.BlockingQueue; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicReference; import java.util.concurrent.locks.LockSupport; - @Slf4j public class CanalFullProducer { private BlockingQueue> queue; @@ -45,7 +52,7 @@ public class CanalFullProducer { private final AtomicReference choosePrimaryKey = new AtomicReference<>(null); private static final DateTimeFormatter DATE_FORMATTER = DateTimeFormatter.ofPattern("yyyy-MM-dd"); private static final DateTimeFormatter DATE_STAMP_FORMATTER = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss"); - private static final WKBReader WKB_READER = new wk + private static final WKBReader WKB_READER = new WKBReader(new GeometryFactory()); public CanalFullProducer(BlockingQueue> queue, DataSource dataSource, MySQLTableDef tableDefinition, TableFullPosition position, int flushSize) { @@ -56,42 +63,108 @@ public CanalFullProducer(BlockingQueue> queue, DataSource da this.flushSize = flushSize; } + public void choosePrimaryKey() { + for (RdbColumnDefinition col : tableDefinition.getColumnDefinitions().values()) { + if (position.getCurPrimaryKeyCols().get(col.getName()) != null) { + choosePrimaryKey.set(col.getName()); + return; + } + } + throw new EventMeshException("illegal: can't pick any primary key"); + } + + public void start(AtomicBoolean flag) { + choosePrimaryKey(); boolean isNextPage = false; - ArrayList records = new ArrayList<>(); + List> rows = new LinkedList<>(); while (flag.get()) { - String scanSql = generateScanSql(tableDefinition, !isNextPage); - log.info("scan sql is [{}] , cur position [{}]", scanSql, - JsonUtils.toJSONString(position.getCurPrimaryKeyCols())); + String scanSql = generateScanSql(!isNextPage); + log.info("scan sql is [{}] , cur position [{}], choose primary key [{}]", scanSql, + JsonUtils.toJSONString(position.getCurPrimaryKeyCols()), choosePrimaryKey.get()); try (Connection connection = dataSource.getConnection(); PreparedStatement statement = connection.prepareStatement(scanSql, ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY)) { statement.setFetchSize(Integer.MIN_VALUE); - setPrepareStatementValue(statement, position); + setPrepareStatementValue(statement); try (ResultSet resultSet = statement.executeQuery()) { + Map lastCol = null; while (flag.get() && resultSet.next()) { - - - if (records.size() < flushSize) { + Map columnValues = new LinkedHashMap<>(); + for (Map.Entry col : + tableDefinition.getColumnDefinitions().entrySet()) { + columnValues.put(col.getKey(), readColumn(resultSet, col.getKey(), + ((MySQLColumnDef) col.getValue()).getType())); + } + lastCol = columnValues; + rows.add(lastCol); + if (rows.size() < flushSize) { continue; } - queue.put(records); - records = new ArrayList<>(); + refreshPosition(lastCol); + commitConnectRecord(rows); + rows = new LinkedList<>(); } + + if (lastCol == null || checkIsScanFinish(lastCol)) { + log.info("full scan db [{}] table [{}] finish", tableDefinition.getSchemaName(), + tableDefinition.getTableName()); + commitConnectRecord(rows); + return; + } + refreshPosition(lastCol); } catch (InterruptedException ignore) { + log.info("full scan db [{}] table [{}] interrupted", tableDefinition.getSchemaName(), + tableDefinition.getTableName()); + Thread.currentThread().interrupt(); + return; } } catch (SQLException e) { - log.error("create connection fail", e); + log.error("catch SQLException fail", e); + LockSupport.parkNanos(3000 * 1000L); + } catch (Exception e) { + log.error("process schema [{}] table [{}] catch unknown exception", tableDefinition.getSchemaName(), + tableDefinition.getTableName(), e); LockSupport.parkNanos(3000 * 1000L); } if (!isNextPage) { isNextPage = true; } } + } + + private void commitConnectRecord(List> rows) throws InterruptedException { + if (rows == null || rows.isEmpty()) { + return; + } + ArrayList records = new ArrayList<>(); + CanalFullRecordOffset offset = new CanalFullRecordOffset(); + JobRdbFullPosition jobRdbFullPosition = new JobRdbFullPosition(); + jobRdbFullPosition.setPrimaryKeyRecords(JsonUtils.toJSONString(position)); + offset.setPosition(jobRdbFullPosition); + CanalFullRecordPartition partition = new CanalFullRecordPartition(); + partition.setSchema(tableDefinition.getSchemaName()); + partition.setTable(tableDefinition.getTableName()); + records.add(new ConnectRecord(partition, offset, System.currentTimeMillis(), rows)); + queue.put(records); + } + private boolean checkIsScanFinish(Map lastCol) { + Object lastPrimaryValue = lastCol.get(choosePrimaryKey.get()); + Object maxPrimaryValue = position.getMaxPrimaryKeyCols().get(choosePrimaryKey.get()); + if (lastPrimaryValue instanceof Number) { + BigDecimal last = new BigDecimal(String.valueOf(lastPrimaryValue)); + BigDecimal max = + new BigDecimal(String.valueOf(maxPrimaryValue)); + return last.compareTo(max) > 0; + } + if (lastPrimaryValue instanceof Comparable) { + return ((Comparable) lastPrimaryValue).compareTo(maxPrimaryValue) > 0; + } + return false; } - public Object readColumn(ResultSet rs, String col, MysqlType colType) throws Exception { + public Object readColumn(ResultSet rs, String col, CanalMySQLType colType) throws Exception { switch (colType) { case TINYINT: case SMALLINT: @@ -103,7 +176,7 @@ public Object readColumn(ResultSet rs, String col, MysqlType colType) throws Exc } else { uLong = rs.getLong(col); } - if (uLong.compareTo(2147483647L) > 0) { + if (uLong.compareTo((long) Integer.MAX_VALUE) > 0) { return uLong; } return uLong.intValue(); @@ -152,27 +225,55 @@ public Object readColumn(ResultSet rs, String col, MysqlType colType) throws Exc case LONGBLOB: return rs.getBytes(col); case GEOMETRY: - String wkb = rs.getString(col); - if (wkb == null) { - return null; - } - return safeToGisWKT("0x" + wkb); + return toGeometry("0x" + rs.getString(col)); + case GEOMETRY_COLLECTION: + case GEOM_COLLECTION: + case POINT: + case LINESTRING: + case POLYGON: + case MULTIPOINT: + case MULTILINESTRING: + case MULTIPOLYGON: + return null; default: return rs.getObject(col); } } - private void refreshPosition() { + protected static String toGeometry(Object value) throws Exception { + if (value == null) { + return null; + } + if (value instanceof String) { + String strVal = (String) value; + if (!strVal.startsWith("0x") && !strVal.startsWith("0X")) { + return (String) value; + } + return WKB_READER.read(hex2bytes(strVal.substring(2))).toText(); + } else if (value instanceof byte[]) { + return WKB_READER.read((byte[]) value).toText(); + } else { + throw new UnsupportedOperationException("class " + value.getClass() + ", value '" + value + "' , " + + "safeToGisWKT" + + " failed."); + } + } + private void refreshPosition(Map lastCol) { + Map nextPosition = new LinkedHashMap<>(); + for (Map.Entry entry : position.getCurPrimaryKeyCols().entrySet()) { + nextPosition.put(entry.getKey(), lastCol.get(entry.getKey())); + } + position.setCurPrimaryKeyCols(nextPosition); } - private void setPrepareStatementValue(PreparedStatement statement, TableFullPosition position) throws SQLException { + private void setPrepareStatementValue(PreparedStatement statement) throws SQLException { String colName = choosePrimaryKey.get(); if (colName == null) { return; } RdbColumnDefinition columnDefinition = tableDefinition.getColumnDefinitions().get(colName); - Object value = position.getCurPrimaryKeyCols().get(choosePrimaryKey); + Object value = position.getCurPrimaryKeyCols().get(colName); String str; switch (columnDefinition.getJdbcType()) { case BIT: @@ -212,12 +313,12 @@ private void setPrepareStatementValue(PreparedStatement statement, TableFullPosi case DATE: Instant d; if (value instanceof Long) { - Long val = (Long)value; + Long val = (Long) value; d = Instant.ofEpochMilli(val); str = d.atZone(ZoneId.systemDefault()).toLocalDateTime().format(DATE_FORMATTER); } else if (value instanceof Integer) { - Integer val = (Integer)value; - d = Instant.ofEpochMilli((long)val); + Integer val = (Integer) value; + d = Instant.ofEpochMilli((long) val); str = d.atZone(ZoneId.systemDefault()).toLocalDateTime().format(DATE_FORMATTER); } else if (value instanceof String) { str = (String) value; @@ -225,18 +326,18 @@ private void setPrepareStatementValue(PreparedStatement statement, TableFullPosi if (!(value instanceof LocalDate)) { throw new IllegalArgumentException("unsupported date class type:" + value.getClass().getSimpleName()); } - str = ((LocalDate)value).format(DATE_FORMATTER); + str = ((LocalDate) value).format(DATE_FORMATTER); } statement.setString(1, str); break; case TIMESTAMP: if (value instanceof String) { - str = (String)value; + str = (String) value; } else { if (!(value instanceof LocalDateTime)) { throw new IllegalArgumentException("unsupported timestamp class type:" + value.getClass().getSimpleName()); } - str = ((LocalDateTime)value).format(DATE_STAMP_FORMATTER); + str = ((LocalDateTime) value).format(DATE_STAMP_FORMATTER); } statement.setString(1, str); break; @@ -263,10 +364,10 @@ public static byte[] hex2bytes(String hexStr) { char c1 = hexStr.charAt(index); char c2 = hexStr.charAt(index + 1); if (c1 < '0' || c1 > 'F' || c2 < '0' || c2 > 'F') { - throw new EventMeshException(String.format("illegal byte [%s], [%s]" , c1, c2)); + throw new EventMeshException(String.format("illegal byte [%s], [%s]", c1, c2)); } - ret[i] = (byte) ((byte)c1 << 4); - ret[i] = (byte) (ret[i] | (byte)(c2)); + ret[i] = (byte) ((byte) c1 << 4); + ret[i] = (byte) (ret[i] | (byte) (c2)); } return ret; } @@ -289,37 +390,30 @@ private void generateQueryColumnsSql(StringBuilder builder, Collection= ? "); } else { builder.append(" > ? "); } - builder.append(" order by ").append(Constants.MySQLQuot).append(colName).append(Constants.MySQLQuot) + builder.append(" order by ").append(Constants.MySQLQuot).append(choosePrimaryKey.get()).append(Constants.MySQLQuot) .append(" asc "); } } diff --git a/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/source/EntryParser.java b/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/source/EntryParser.java index 3ca2f7ec2f..8ef60ff04d 100644 --- a/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/source/EntryParser.java +++ b/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/source/EntryParser.java @@ -22,6 +22,7 @@ import org.apache.eventmesh.connector.canal.model.EventColumn; import org.apache.eventmesh.connector.canal.model.EventColumnIndexComparable; import org.apache.eventmesh.connector.canal.model.EventType; +import org.apache.eventmesh.connector.canal.source.table.RdbTableMgr; import org.apache.commons.lang3.StringUtils; @@ -31,7 +32,6 @@ import java.util.List; import java.util.Map; -import org.apache.eventmesh.connector.canal.source.table.RdbTableMgr; import org.springframework.util.CollectionUtils; import com.alibaba.otter.canal.protocol.CanalEntry; @@ -48,7 +48,8 @@ @Slf4j public class EntryParser { - public static Map> parse(CanalSourceConfig sourceConfig, List datas) { + public static Map> parse(CanalSourceConfig sourceConfig, List datas, + RdbTableMgr tables) { List recordList = new ArrayList<>(); List transactionDataBuffer = new ArrayList<>(); // need check weather the entry is loopback @@ -65,7 +66,7 @@ public static Map> parse(CanalSourceConfig source } break; case TRANSACTIONEND: - parseRecordListWithEntryBuffer(sourceConfig, recordList, transactionDataBuffer); + parseRecordListWithEntryBuffer(sourceConfig, recordList, transactionDataBuffer, tables); if (!recordList.isEmpty()) { recordMap.put(entry.getHeader().getLogfileOffset(), recordList); } @@ -81,10 +82,11 @@ public static Map> parse(CanalSourceConfig source return recordMap; } - private void parseRecordListWithEntryBuffer(CanalSourceConfig sourceConfig, List recordList, - List transactionDataBuffer) { + private static void parseRecordListWithEntryBuffer(CanalSourceConfig sourceConfig, + List recordList, + List transactionDataBuffer, RdbTableMgr tables) { for (Entry bufferEntry : transactionDataBuffer) { - List recordParsedList = internParse(sourceConfig, bufferEntry); + List recordParsedList = internParse(sourceConfig, bufferEntry, tables); if (CollectionUtils.isEmpty(recordParsedList)) { continue; } @@ -100,15 +102,17 @@ private void parseRecordListWithEntryBuffer(CanalSourceConfig sourceConfig, List } } - private boolean checkNeedSync(CanalSourceConfig sourceConfig, RowData rowData) { - Column markedColumn = getColumnIgnoreCase(rowData.getAfterColumnsList(), sourceConfig.getNeedSyncMarkTableColumnName()); + private static boolean checkNeedSync(CanalSourceConfig sourceConfig, RowData rowData) { + Column markedColumn = getColumnIgnoreCase(rowData.getAfterColumnsList(), + sourceConfig.getNeedSyncMarkTableColumnName()); if (markedColumn != null) { - return StringUtils.equalsIgnoreCase(markedColumn.getValue(), sourceConfig.getNeedSyncMarkTableColumnValue()); + return StringUtils.equalsIgnoreCase(markedColumn.getValue(), + sourceConfig.getNeedSyncMarkTableColumnValue()); } return false; } - private Column getColumnIgnoreCase(List columns, String columName) { + private static Column getColumnIgnoreCase(List columns, String columName) { for (Column column : columns) { if (column.getName().equalsIgnoreCase(columName)) { return column; @@ -117,7 +121,8 @@ private Column getColumnIgnoreCase(List columns, String columName) { return null; } - private static List internParse(CanalSourceConfig sourceConfig, Entry entry, RdbTableMgr tableMgr) { + private static List internParse(CanalSourceConfig sourceConfig, Entry entry, + RdbTableMgr tableMgr) { String schemaName = entry.getHeader().getSchemaName(); String tableName = entry.getHeader().getTableName(); if (tableMgr.getTable(schemaName, tableName) == null) { @@ -155,7 +160,8 @@ private static List internParse(CanalSourceConfig sourceConf return recordList; } - private static CanalConnectRecord internParse(CanalSourceConfig canalSourceConfig, Entry entry, RowChange rowChange, RowData rowData) { + private static CanalConnectRecord internParse(CanalSourceConfig canalSourceConfig, Entry entry, + RowChange rowChange, RowData rowData) { CanalConnectRecord canalConnectRecord = new CanalConnectRecord(); canalConnectRecord.setTableName(entry.getHeader().getTableName()); canalConnectRecord.setSchemaName(entry.getHeader().getSchemaName()); @@ -243,7 +249,7 @@ private static CanalConnectRecord internParse(CanalSourceConfig canalSourceConfi } private static void checkUpdateKeyColumns(Map oldKeyColumns, - Map keyColumns) { + Map keyColumns) { if (oldKeyColumns.isEmpty()) { return; } diff --git a/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/source/connector/CanalSourceConnector.java b/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/source/connector/CanalSourceConnector.java index c1c381c91f..a598352f73 100644 --- a/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/source/connector/CanalSourceConnector.java +++ b/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/source/connector/CanalSourceConnector.java @@ -17,16 +17,15 @@ package org.apache.eventmesh.connector.canal.source.connector; -import com.alibaba.otter.canal.parse.inbound.TableMeta; import org.apache.eventmesh.common.config.connector.Config; import org.apache.eventmesh.common.config.connector.rdb.canal.CanalSourceConfig; -import org.apache.eventmesh.connector.canal.source.table.RdbTableMgr; import org.apache.eventmesh.common.remote.offset.RecordPosition; import org.apache.eventmesh.common.remote.offset.canal.CanalRecordOffset; import org.apache.eventmesh.common.remote.offset.canal.CanalRecordPartition; import org.apache.eventmesh.common.utils.JsonUtils; import org.apache.eventmesh.connector.canal.CanalConnectRecord; import org.apache.eventmesh.connector.canal.source.EntryParser; +import org.apache.eventmesh.connector.canal.source.table.RdbTableMgr; import org.apache.eventmesh.openconnect.api.ConnectorCreateService; import org.apache.eventmesh.openconnect.api.connector.ConnectorContext; import org.apache.eventmesh.openconnect.api.connector.SourceConnectorContext; @@ -296,7 +295,7 @@ public List poll() { List result = new ArrayList<>(); // key: Xid offset - Map> connectorRecordMap = EntryParser.parse(sourceConfig, entries); + Map> connectorRecordMap = EntryParser.parse(sourceConfig, entries, tableMgr); if (!connectorRecordMap.isEmpty()) { Set>> entrySet = connectorRecordMap.entrySet(); diff --git a/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/source/connector/CanalSourceFullConnector.java b/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/source/connector/CanalSourceFullConnector.java index b7e15157f0..60a84b4241 100644 --- a/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/source/connector/CanalSourceFullConnector.java +++ b/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/source/connector/CanalSourceFullConnector.java @@ -1,6 +1,5 @@ package org.apache.eventmesh.connector.canal.source.connector; -import lombok.extern.slf4j.Slf4j; import org.apache.eventmesh.common.AbstractComponent; import org.apache.eventmesh.common.EventMeshThreadFactory; import org.apache.eventmesh.common.config.connector.Config; @@ -8,10 +7,13 @@ import org.apache.eventmesh.common.config.connector.rdb.canal.JobRdbFullPosition; import org.apache.eventmesh.common.config.connector.rdb.canal.RdbDBDefinition; import org.apache.eventmesh.common.config.connector.rdb.canal.RdbTableDefinition; -import org.apache.eventmesh.common.config.connector.rdb.canal.SourceConnectorConfig; +import org.apache.eventmesh.common.config.connector.rdb.canal.mysql.MySQLTableDef; import org.apache.eventmesh.common.exception.EventMeshException; +import org.apache.eventmesh.common.utils.JsonUtils; import org.apache.eventmesh.connector.canal.DatabaseConnection; +import org.apache.eventmesh.connector.canal.source.CanalFullProducer; import org.apache.eventmesh.connector.canal.source.position.CanalFullPositionMgr; +import org.apache.eventmesh.connector.canal.source.position.TableFullPosition; import org.apache.eventmesh.connector.canal.source.table.RdbSimpleTable; import org.apache.eventmesh.connector.canal.source.table.RdbTableMgr; import org.apache.eventmesh.openconnect.api.ConnectorCreateService; @@ -20,14 +22,19 @@ import org.apache.eventmesh.openconnect.api.source.Source; import org.apache.eventmesh.openconnect.offsetmgmt.api.data.ConnectRecord; -import javax.sql.DataSource; import java.util.HashMap; +import java.util.LinkedList; import java.util.List; import java.util.Map; import java.util.concurrent.BlockingQueue; import java.util.concurrent.LinkedBlockingQueue; import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; + +import javax.sql.DataSource; + +import lombok.extern.slf4j.Slf4j; @Slf4j public class CanalSourceFullConnector extends AbstractComponent implements Source, ConnectorCreateService { @@ -37,6 +44,7 @@ public class CanalSourceFullConnector extends AbstractComponent implements Sourc private ThreadPoolExecutor executor; private final Map dataSources = new HashMap<>(); private final BlockingQueue> queue = new LinkedBlockingQueue<>(); + private final AtomicBoolean flag = new AtomicBoolean(true); @Override protected void startup() throws Exception { @@ -46,35 +54,46 @@ protected void startup() throws Exception { log.info("connector [{}] has finished the job", config.getConnectorConfig().getConnectorName()); return; } - executor = new ThreadPoolExecutor(config.getParallel(), config.getParallel(),0L, TimeUnit.MILLISECONDS, - new LinkedBlockingQueue<>(), new EventMeshThreadFactory("canal-source-full")); + executor = new ThreadPoolExecutor(config.getParallel(), config.getParallel(), 0L, TimeUnit.MILLISECONDS, + new LinkedBlockingQueue<>(), new EventMeshThreadFactory("canal-source-full")); + List producers = new LinkedList<>(); if (config.getConnectorConfig().getDatabases() != null) { for (RdbDBDefinition db : config.getConnectorConfig().getDatabases()) { for (RdbTableDefinition table : db.getTables()) { - log.info("it will create producer of db [{}] table [{}]", db.getSchemaName(), table.getTableName()); - DataSource dataSource = dataSources.computeIfAbsent(db.getSchemaName(), - k -> DatabaseConnection.createDruidDataSource(null, config.getConnectorConfig().getUserName(), - config.getConnectorConfig().getPassWord())); - RdbSimpleTable simpleTable = new RdbSimpleTable(db.getSchemaName(), table.getTableName()); - JobRdbFullPosition position = positionMgr.getPosition(simpleTable); - if (position == null) { - throw new EventMeshException(String.format("db [%s] table [%s] have none position info", + try { + log.info("it will create producer of db [{}] table [{}]", db.getSchemaName(), table.getTableName()); + DataSource dataSource = dataSources.computeIfAbsent(db.getSchemaName(), + k -> DatabaseConnection.createDruidDataSource(config.getConnectorConfig().getUrl(), + config.getConnectorConfig().getUserName(), + config.getConnectorConfig().getPassWord())); + RdbSimpleTable simpleTable = new RdbSimpleTable(db.getSchemaName(), table.getTableName()); + JobRdbFullPosition position = positionMgr.getPosition(simpleTable); + if (position == null) { + throw new EventMeshException(String.format("db [%s] table [%s] have none position info", db.getSchemaName(), table.getTableName())); - } - RdbTableDefinition tableDefinition = tableMgr.getTable(simpleTable); - if (tableDefinition == null) { - throw new EventMeshException(String.format("db [%s] table [%s] have none table definition info", + } + RdbTableDefinition tableDefinition = tableMgr.getTable(simpleTable); + if (tableDefinition == null) { + throw new EventMeshException(String.format("db [%s] table [%s] have none table definition info", db.getSchemaName(), table.getTableName())); - } + } + producers.add(new CanalFullProducer(queue, dataSource, (MySQLTableDef) tableDefinition, + JsonUtils.parseObject(position.getPrimaryKeyRecords(), TableFullPosition.class), + config.getFlushSize())); + } catch (Exception e) { + log.error("create schema [{}] table [{}] producers fail", db.getSchemaName(), + table.getTableName(), e); + } } } } + producers.forEach(p -> executor.execute(() -> p.start(flag))); } @Override protected void shutdown() throws Exception { - + flag.set(false); } @Override @@ -89,7 +108,7 @@ public Class configClass() { @Override public void init(Config config) throws Exception { - this.config = (CanalSourceFullConfig)config; + this.config = (CanalSourceFullConfig) config; } @Override @@ -102,7 +121,7 @@ public void init(ConnectorContext connectorContext) throws Exception { @Override public void commit(ConnectRecord record) { - + // nothing } @Override @@ -112,6 +131,21 @@ public String name() { @Override public List poll() { + while (flag.get()) { + try { + List records = queue.poll(5, TimeUnit.SECONDS); + if (records == null || records.isEmpty()) { + continue; + } + return records; + } catch (InterruptedException ignore) { + Thread.currentThread().interrupt(); + log.info("[{}] thread interrupted", this.getClass()); + return null; + } + } + log.info("[{}] life flag is stop, so return null", this.getClass()); return null; } + } diff --git a/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/source/position/CanalFullPositionMgr.java b/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/source/position/CanalFullPositionMgr.java index 8003ef1784..fa93871b0f 100644 --- a/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/source/position/CanalFullPositionMgr.java +++ b/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/source/position/CanalFullPositionMgr.java @@ -1,18 +1,18 @@ package org.apache.eventmesh.connector.canal.source.position; import com.alibaba.druid.pool.DruidDataSource; -import lombok.AllArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; import org.apache.eventmesh.common.AbstractComponent; import org.apache.eventmesh.common.config.connector.rdb.canal.CanalSourceFullConfig; +import org.apache.eventmesh.common.config.connector.rdb.canal.JobRdbFullPosition; import org.apache.eventmesh.common.config.connector.rdb.canal.RdbColumnDefinition; import org.apache.eventmesh.common.config.connector.rdb.canal.RdbDBDefinition; -import org.apache.eventmesh.common.config.connector.rdb.canal.JobRdbFullPosition; import org.apache.eventmesh.common.config.connector.rdb.canal.RdbTableDefinition; import org.apache.eventmesh.common.config.connector.rdb.canal.mysql.Constants; import org.apache.eventmesh.common.config.connector.rdb.canal.mysql.MySQLTableDef; import org.apache.eventmesh.common.remote.offset.RecordPosition; +import org.apache.eventmesh.common.remote.offset.canal.CanalFullRecordOffset; import org.apache.eventmesh.common.utils.JsonUtils; import org.apache.eventmesh.connector.canal.DatabaseConnection; import org.apache.eventmesh.connector.canal.source.table.RdbSimpleTable; @@ -25,7 +25,6 @@ import java.sql.SQLException; import java.sql.Statement; import java.util.LinkedHashMap; -import java.util.List; import java.util.Map; import java.util.concurrent.ScheduledThreadPoolExecutor; @@ -54,10 +53,18 @@ protected void startup() throws Exception { thread.setName("task-full-position-timer"); return thread; }, new ScheduledThreadPoolExecutor.DiscardOldestPolicy()); - if (config.getStartPosition() != null) { - processPositions(config.getStartPosition()); - } + prepareRecordPosition(); + initPositions(); + } + public void prepareRecordPosition() { + if (config.getStartPosition() != null && !config.getStartPosition().isEmpty()) { + for (RecordPosition record : config.getStartPosition()) { + CanalFullRecordOffset offset = (CanalFullRecordOffset) record.getRecordOffset(); + RdbSimpleTable table = new RdbSimpleTable(offset.getPosition().getSchema(), offset.getPosition().getTableName()); + positions.put(table, offset.getPosition()); + } + } } public JobRdbFullPosition getPosition(RdbSimpleTable table) { @@ -74,30 +81,39 @@ public boolean isFinished() { return true; } - private void processPositions(List startPosition) throws SQLException { - for (RdbDBDefinition database : config.getConnectorConfig().getDatabases()) { - for (RdbTableDefinition table : database.getTables()) { - RdbSimpleTable simpleTable = new RdbSimpleTable(database.getSchemaName(), table.getTableName()); - RdbTableDefinition tableDefinition; - if ((tableDefinition = tableMgr.getTable(simpleTable)) == null) { - log.error("db [{}] table [{}] definition is null", database.getSchemaName(), table.getTableName()); - continue; - } - log.info("init position of data [{}] table [{}]", database.getSchemaName(), table.getTableName()); - - JobRdbFullPosition recordPosition = positions.get(simpleTable); - if (recordPosition == null || !recordPosition.isFinished()) { - try (DruidDataSource dataSource = DatabaseConnection.createDruidDataSource(null, - config.getConnectorConfig().getUserName(), config.getConnectorConfig().getPassWord())) { - positions.put(simpleTable, initPosition(dataSource, (MySQLTableDef)tableDefinition, recordPosition)); + private void initPositions() { + try (DruidDataSource dataSource = + DatabaseConnection.createDruidDataSource(config.getConnectorConfig().getUrl(), + config.getConnectorConfig().getUserName(), config.getConnectorConfig().getPassWord())) { + for (RdbDBDefinition database : config.getConnectorConfig().getDatabases()) { + for (RdbTableDefinition table : database.getTables()) { + try { + RdbSimpleTable simpleTable = new RdbSimpleTable(database.getSchemaName(), table.getTableName()); + RdbTableDefinition tableDefinition; + if ((tableDefinition = tableMgr.getTable(simpleTable)) == null) { + log.error("db [{}] table [{}] definition is null", database.getSchemaName(), + table.getTableName()); + continue; + } + log.info("init position of data [{}] table [{}]", database.getSchemaName(), + table.getTableName()); + + JobRdbFullPosition recordPosition = positions.get(simpleTable); + if (recordPosition == null || !recordPosition.isFinished()) { + positions.put(simpleTable, fetchTableInfo(dataSource, (MySQLTableDef) tableDefinition, + recordPosition)); + } + } catch (Exception e) { + log.error("process schema [{}] table [{}] position fail", database.getSchemaName(), + table.getTableName(), e); } } } } } - private JobRdbFullPosition initPosition(DataSource dataSource, MySQLTableDef tableDefinition, - JobRdbFullPosition recordPosition) throws SQLException { + private JobRdbFullPosition fetchTableInfo(DataSource dataSource, MySQLTableDef tableDefinition, + JobRdbFullPosition recordPosition) throws SQLException { TableFullPosition position = new TableFullPosition(); Map preMinPrimaryKeys = new LinkedHashMap<>(); Map preMaxPrimaryKeys = new LinkedHashMap<>(); @@ -112,31 +128,36 @@ private JobRdbFullPosition initPosition(DataSource dataSource, MySQLTableDef tab } long rowCount = queryCurTableRowCount(dataSource, tableDefinition); JobRdbFullPosition jobRdbFullPosition = new JobRdbFullPosition(); - if (recordPosition != null ) { - if (StringUtils.isNotBlank(recordPosition.getCurPrimaryKey())) { - TableFullPosition record = JsonUtils.parseObject(recordPosition.getCurPrimaryKey(), TableFullPosition.class); + if (recordPosition != null) { + if (StringUtils.isNotBlank(recordPosition.getPrimaryKeyRecords())) { + TableFullPosition record = JsonUtils.parseObject(recordPosition.getPrimaryKeyRecords(), + TableFullPosition.class); if (record != null && record.getCurPrimaryKeyCols() != null && !record.getCurPrimaryKeyCols().isEmpty()) { position.setCurPrimaryKeyCols(record.getCurPrimaryKeyCols()); } } + jobRdbFullPosition.setPercent(recordPosition.getPercent()); } jobRdbFullPosition.setSchema(tableDefinition.getSchemaName()); jobRdbFullPosition.setTableName(tableDefinition.getTableName()); jobRdbFullPosition.setMaxCount(rowCount); - jobRdbFullPosition.setCurPrimaryKey(JsonUtils.toJSONString(position.getCurPrimaryKeyCols())); + jobRdbFullPosition.setPrimaryKeyRecords(JsonUtils.toJSONString(position)); return jobRdbFullPosition; } private long queryCurTableRowCount(DataSource datasource, MySQLTableDef tableDefinition) throws SQLException { String sql = - "select AVG_ROW_LENGTH,DATA_LENGTH from information_schema.TABLES where TABLE_SCHEMA=" + Constants.MySQLQuot + tableDefinition.getSchemaName() + Constants.MySQLQuot + " and TABLE_NAME="+ Constants.MySQLQuot + tableDefinition.getTableName() + Constants.MySQLQuot; - try (Statement statement = datasource.getConnection().createStatement(); ResultSet resultSet = statement.executeQuery(sql)) { + "select `AVG_ROW_LENGTH`,`DATA_LENGTH` from information_schema.TABLES where `TABLE_SCHEMA`='" + tableDefinition.getSchemaName() +"' and `TABLE_NAME`='" + tableDefinition.getTableName() +"'"; + try (Statement statement = datasource.getConnection().createStatement(); ResultSet resultSet = + statement.executeQuery(sql)) { long result = 0L; if (resultSet.next()) { long avgRowLength = resultSet.getLong("AVG_ROW_LENGTH"); long dataLength = resultSet.getLong("DATA_LENGTH"); - result = dataLength / avgRowLength; + if (avgRowLength != 0L) { + result = dataLength / avgRowLength; + } } return result; } @@ -175,7 +196,8 @@ private Object fetchMinPrimaryKey(DataSource dataSource, MySQLTableDef tableDefi .append(Constants.MySQLQuot).append(tableDefinition.getSchemaName()).append(Constants.MySQLQuot).append(".").append(Constants.MySQLQuot).append(tableDefinition.getTableName()).append(Constants.MySQLQuot); appendPrePrimaryKey(prePrimary, builder); String sql = builder.toString(); - try (PreparedStatement statement = dataSource.getConnection().prepareStatement(sql)){ + log.info("fetch min primary sql [{}]", sql); + try (PreparedStatement statement = dataSource.getConnection().prepareStatement(sql)) { setValue2Statement(statement, prePrimary, tableDefinition); try (ResultSet resultSet = statement.executeQuery()) { if (resultSet.next()) { @@ -198,7 +220,8 @@ private Object fetchMaxPrimaryKey(DataSource dataSource, MySQLTableDef tableDefi .append(Constants.MySQLQuot).append(tableDefinition.getSchemaName()).append(Constants.MySQLQuot).append(".").append(Constants.MySQLQuot).append(tableDefinition.getTableName()).append(Constants.MySQLQuot); appendPrePrimaryKey(prePrimary, builder); String sql = builder.toString(); - try (PreparedStatement statement = dataSource.getConnection().prepareStatement(sql)){ + log.info("fetch max primary sql [{}]", sql); + try (PreparedStatement statement = dataSource.getConnection().prepareStatement(sql)) { setValue2Statement(statement, prePrimary, tableDefinition); try (ResultSet resultSet = statement.executeQuery()) { if (resultSet.next()) { diff --git a/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/source/table/RdbTableMgr.java b/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/source/table/RdbTableMgr.java index 6cacc56122..18d134db96 100644 --- a/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/source/table/RdbTableMgr.java +++ b/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/source/table/RdbTableMgr.java @@ -1,9 +1,9 @@ package org.apache.eventmesh.connector.canal.source.table; import com.alibaba.druid.pool.DruidDataSource; -import com.mysql.cj.MysqlType; import lombok.extern.slf4j.Slf4j; import org.apache.eventmesh.common.AbstractComponent; +import org.apache.eventmesh.common.config.connector.rdb.canal.CanalMySQLType; import org.apache.eventmesh.common.config.connector.rdb.canal.RdbColumnDefinition; import org.apache.eventmesh.common.config.connector.rdb.canal.RdbDBDefinition; import org.apache.eventmesh.common.config.connector.rdb.canal.RdbTableDefinition; @@ -18,20 +18,20 @@ import java.sql.PreparedStatement; import java.sql.ResultSet; import java.sql.SQLException; +import java.util.Collections; import java.util.HashMap; import java.util.HashSet; import java.util.LinkedHashMap; import java.util.LinkedList; import java.util.List; import java.util.Map; -import java.util.stream.Collectors; /** * Description: */ @Slf4j public class RdbTableMgr extends AbstractComponent { - private SourceConnectorConfig config; + private final SourceConnectorConfig config; private final Map tables = new HashMap<>(); public RdbTableMgr(SourceConnectorConfig config) { @@ -47,59 +47,63 @@ public RdbTableDefinition getTable(RdbSimpleTable table) { } @Override - protected void startup() throws Exception { + protected void startup() { if (config != null && config.getDatabases() != null) { for (RdbDBDefinition db : config.getDatabases()) { if (db.getTables() == null) { log.warn("init db [{}] position, but it's tables are null", db.getSchemaName()); continue; } - try (DruidDataSource dataSource = DatabaseConnection.createDruidDataSource(null, + try (DruidDataSource dataSource = DatabaseConnection.createDruidDataSource(config.getUrl(), config.getUserName(), config.getPassWord())) { - List tableNames = - db.getTables().stream().map(RdbTableDefinition::getTableName).collect(Collectors.toList()); - Map> primaryKeys = queryTablePrimaryKey(dataSource, tableNames); - Map> columns = queryColumns(dataSource, tableNames); for (RdbTableDefinition table : db.getTables()) { - MySQLTableDef mysqlTable = new MySQLTableDef(); - mysqlTable.setSchemaName(db.getSchemaName()); - mysqlTable.setTableName(table.getTableName()); - if (primaryKeys == null || primaryKeys.isEmpty() || primaryKeys.get(table.getTableName()) == null) { - log.warn("init db [{}] table [{}] info, and primary keys are empty", db.getSchemaName(), - table.getTableName()); - } else { - mysqlTable.setPrimaryKeys(new HashSet<>(primaryKeys.get(table.getTableName()))); - } - if (columns == null || columns.isEmpty() || columns.get(table.getTableName()) == null) { - log.warn("init db [{}] table [{}] info, and columns are empty", db.getSchemaName(), - table.getTableName()); - } else { - LinkedHashMap cols = new LinkedHashMap<>(); - columns.get(table.getTableName()).forEach(x -> cols.put(x.getName(), x)); - mysqlTable.setColumnDefinitions(cols); - } + try { + MySQLTableDef mysqlTable = new MySQLTableDef(); + mysqlTable.setSchemaName(db.getSchemaName()); + mysqlTable.setTableName(table.getTableName()); + List tables = Collections.singletonList(table.getTableName()); + Map> primaryKeys = queryTablePrimaryKey(dataSource, db.getSchemaName(), + tables); + Map> columns = queryColumns(dataSource, db.getSchemaName(), tables); + if (primaryKeys == null || primaryKeys.isEmpty() || primaryKeys.get(table.getTableName()) == null) { + log.warn("init db [{}] table [{}] info, and primary keys are empty", db.getSchemaName(), + table.getTableName()); + } else { + mysqlTable.setPrimaryKeys(new HashSet<>(primaryKeys.get(table.getTableName()))); + } + if (columns == null || columns.isEmpty() || columns.get(table.getTableName()) == null) { + log.warn("init db [{}] table [{}] info, and columns are empty", db.getSchemaName(), + table.getTableName()); + } else { + LinkedHashMap cols = new LinkedHashMap<>(); + columns.get(table.getTableName()).forEach(x -> cols.put(x.getName(), x)); + mysqlTable.setColumnDefinitions(cols); + } - tables.put(new RdbSimpleTable(db.getSchemaName(), table.getTableName()), mysqlTable); + this.tables.put(new RdbSimpleTable(db.getSchemaName(), table.getTableName()), mysqlTable); + } catch (Exception e) { + log.error("init rdb table schema [{}] table [{}] fail", db.getSchemaName(), + table.getTableName(), e); + } } - } catch (Exception e) { - log.error("init db [{}] tables info fail", db.getSchemaName(), e); } - } } } - private Map> queryTablePrimaryKey(DruidDataSource dataSource, List tables) throws SQLException { + private Map> queryTablePrimaryKey(DruidDataSource dataSource, String schema, + List tables) throws SQLException { Map> primaryKeys = new LinkedHashMap<>(); String prepareTables = SqlUtils.genPrepareSqlOfInClause(tables.size()); String sql = "select L.TABLE_NAME,L.COLUMN_NAME,R.CONSTRAINT_TYPE from " + - "INFORMATION_SCHEMA.KEY_COLUMN_USAGE L left join INFORMATION_SCHEMA.TABLE_CONSTRAINTS R on C" + + "INFORMATION_SCHEMA.KEY_COLUMN_USAGE L left join INFORMATION_SCHEMA.TABLE_CONSTRAINTS R on L" + ".TABLE_SCHEMA = R.TABLE_SCHEMA and L.TABLE_NAME = R.TABLE_NAME and L.CONSTRAINT_CATALOG = R" + ".CONSTRAINT_CATALOG and L.CONSTRAINT_SCHEMA = R.CONSTRAINT_SCHEMA and L.CONSTRAINT_NAME = R" + ".CONSTRAINT_NAME where L.TABLE_SCHEMA = ? and L.TABLE_NAME in " + prepareTables + " and R" + ".CONSTRAINT_TYPE IN ('PRIMARY KEY') order by L.ORDINAL_POSITION asc"; try (PreparedStatement statement = dataSource.getConnection().prepareStatement(sql)) { - SqlUtils.setInClauseParameters(statement, tables); + statement.setString(1, schema); + SqlUtils.setInClauseParameters(statement, 2, tables); ResultSet resultSet = statement.executeQuery(); if (resultSet == null) { return null; @@ -120,7 +124,7 @@ private Map> queryTablePrimaryKey(DruidDataSource dataSourc return primaryKeys; } - private Map> queryColumns(DataSource dataSource, List tables) throws SQLException { + private Map> queryColumns(DataSource dataSource, String schema, List tables) throws SQLException { String prepareTables = SqlUtils.genPrepareSqlOfInClause(tables.size()); String sql = "select TABLE_SCHEMA,TABLE_NAME,COLUMN_NAME,IS_NULLABLE,DATA_TYPE,CHARACTER_MAXIMUM_LENGTH," + "CHARACTER_OCTET_LENGTH,NUMERIC_SCALE,NUMERIC_PRECISION,DATETIME_PRECISION,CHARACTER_SET_NAME," + @@ -129,7 +133,8 @@ private Map> queryColumns(DataSource dataSource, Li "ORDINAL_POSITION asc"; Map> cols = new LinkedHashMap<>(); try (PreparedStatement statement = dataSource.getConnection().prepareStatement(sql)) { - SqlUtils.setInClauseParameters(statement, tables); + statement.setString(1, schema); + SqlUtils.setInClauseParameters(statement, 2, tables); ResultSet resultSet = statement.executeQuery(); if (resultSet == null) { return null; @@ -139,10 +144,9 @@ private Map> queryColumns(DataSource dataSource, Li String colName = resultSet.getString("COLUMN_NAME"); String dataType = resultSet.getString("DATA_TYPE"); JDBCType jdbcType = SqlUtils.toJDBCType(dataType); - MysqlType type = MysqlType.getByName(dataType); MySQLColumnDef col = new MySQLColumnDef(); col.setJdbcType(jdbcType); - col.setType(type); + col.setType(CanalMySQLType.valueOfCode(dataType)); col.setName(colName); cols.compute(tableName, (k, v) -> { if (v == null) { From dcfc348e857b34f0beea6c002ab6c1b47cfe9f82 Mon Sep 17 00:00:00 2001 From: sodaRyCN <757083350@qq.com> Date: Fri, 12 Jul 2024 18:21:20 +0800 Subject: [PATCH 07/15] do full write --- .../config/connector/rdb/JdbcConfig.java | 22 +++++ .../rdb/canal/CanalSinkFullConfig.java | 5 +- .../rdb/canal/SinkConnectorConfig.java | 21 +--- .../rdb/canal/SourceConnectorConfig.java | 22 +---- .../connector/canal/DatabaseConnection.java | 24 ++--- .../sink/connector/CanalSinkConnector.java | 14 +-- .../connector/CanalSinkFullConnector.java | 35 ++++++- .../{ => connector}/CanalFullProducer.java | 18 ++-- .../connector/CanalSourceConnector.java | 3 +- .../connector/CanalSourceFullConnector.java | 39 +++++--- .../source/position/CanalFullPositionMgr.java | 95 +++++++++--------- .../canal/source/table/RdbTableMgr.java | 97 ++++++++++--------- 12 files changed, 226 insertions(+), 169 deletions(-) create mode 100644 eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/rdb/JdbcConfig.java rename eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/source/{ => connector}/CanalFullProducer.java (99%) diff --git a/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/rdb/JdbcConfig.java b/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/rdb/JdbcConfig.java new file mode 100644 index 0000000000..70c18a221a --- /dev/null +++ b/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/rdb/JdbcConfig.java @@ -0,0 +1,22 @@ +package org.apache.eventmesh.common.config.connector.rdb; + +import org.apache.eventmesh.common.config.connector.rdb.canal.RdbDBDefinition; + +import java.util.Set; + +import lombok.Data; + +@Data +public class JdbcConfig { + private String url; + + private String dbAddress; + + private int dbPort; + + private String userName; + + private String passWord; + + private Set databases; +} diff --git a/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/rdb/canal/CanalSinkFullConfig.java b/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/rdb/canal/CanalSinkFullConfig.java index 5a6deb77cd..2143368dd9 100644 --- a/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/rdb/canal/CanalSinkFullConfig.java +++ b/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/rdb/canal/CanalSinkFullConfig.java @@ -1,12 +1,13 @@ package org.apache.eventmesh.common.config.connector.rdb.canal; +import org.apache.eventmesh.common.config.connector.SinkConfig; + import lombok.Data; import lombok.EqualsAndHashCode; -import org.apache.eventmesh.common.config.connector.SinkConfig; @Data @EqualsAndHashCode(callSuper = true) public class CanalSinkFullConfig extends SinkConfig { - + private final SinkConnectorConfig sinkConfig; } diff --git a/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/rdb/canal/SinkConnectorConfig.java b/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/rdb/canal/SinkConnectorConfig.java index 1124bb1425..761cdba4bb 100644 --- a/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/rdb/canal/SinkConnectorConfig.java +++ b/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/rdb/canal/SinkConnectorConfig.java @@ -17,27 +17,16 @@ package org.apache.eventmesh.common.config.connector.rdb.canal; +import org.apache.eventmesh.common.config.connector.rdb.JdbcConfig; + import lombok.Data; +import lombok.EqualsAndHashCode; /** * Configuration parameters for a sink connector. */ @Data -public class SinkConnectorConfig { - +@EqualsAndHashCode(callSuper = true) +public class SinkConnectorConfig extends JdbcConfig { private String connectorName; - - private String url; - - private String dbAddress; - - private int dbPort; - - private String userName; - - private String passWord; - - private String schemaName; - - private String tableName; } diff --git a/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/rdb/canal/SourceConnectorConfig.java b/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/rdb/canal/SourceConnectorConfig.java index 01aa01fe09..9a95696a0d 100644 --- a/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/rdb/canal/SourceConnectorConfig.java +++ b/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/rdb/canal/SourceConnectorConfig.java @@ -17,28 +17,16 @@ package org.apache.eventmesh.common.config.connector.rdb.canal; -import lombok.Data; +import org.apache.eventmesh.common.config.connector.rdb.JdbcConfig; -import java.util.Set; +import lombok.Data; +import lombok.EqualsAndHashCode; /** * Represents the configuration for a database connector. */ @Data -public class SourceConnectorConfig { - +@EqualsAndHashCode(callSuper = true) +public class SourceConnectorConfig extends JdbcConfig { private String connectorName; - - private String url; - - private String dbAddress; - - private int dbPort; - - private String userName; - - private String passWord; - - private Set databases; - } diff --git a/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/DatabaseConnection.java b/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/DatabaseConnection.java index dc576186ea..6fbda08d83 100644 --- a/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/DatabaseConnection.java +++ b/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/DatabaseConnection.java @@ -18,22 +18,23 @@ package org.apache.eventmesh.connector.canal; -import com.alibaba.druid.pool.DruidDataSource; -import org.apache.eventmesh.common.config.connector.rdb.canal.CanalSinkConfig; -import org.apache.eventmesh.common.config.connector.rdb.canal.CanalSourceConfig; +import org.apache.eventmesh.common.config.connector.rdb.canal.SinkConnectorConfig; +import org.apache.eventmesh.common.config.connector.rdb.canal.SourceConnectorConfig; import java.sql.Connection; import java.sql.SQLException; +import com.alibaba.druid.pool.DruidDataSource; + public class DatabaseConnection { public static DruidDataSource sourceDataSource; public static DruidDataSource sinkDataSource; - public static CanalSourceConfig sourceConfig; + public static SourceConnectorConfig sourceConfig; - public static CanalSinkConfig sinkConfig; + public static SinkConnectorConfig sinkConfig; public static DruidDataSource createDruidDataSource(String url, String UserName, String passWord) { DruidDataSource dataSource = new DruidDataSource(); @@ -41,7 +42,6 @@ public static DruidDataSource createDruidDataSource(String url, String UserName, dataSource.setUsername(UserName); dataSource.setPassword(passWord); dataSource.setInitialSize(5); - dataSource.setDriverClassName("com.mysql.cj.jdbc.Driver"); dataSource.setMinIdle(5); dataSource.setMaxActive(20); dataSource.setMaxWait(60000); @@ -57,15 +57,15 @@ public static DruidDataSource createDruidDataSource(String url, String UserName, } public static void initSourceConnection() { - sourceDataSource = createDruidDataSource(sourceConfig.getSourceConnectorConfig().getUrl(), - sourceConfig.getSourceConnectorConfig().getUserName(), - sourceConfig.getSourceConnectorConfig().getPassWord()); + sourceDataSource = createDruidDataSource(sourceConfig.getUrl(), + sourceConfig.getUserName(), + sourceConfig.getPassWord()); } public static void initSinkConnection() { - sinkDataSource = createDruidDataSource(sinkConfig.getSinkConnectorConfig().getUrl(), - sinkConfig.getSinkConnectorConfig().getUserName(), - sinkConfig.getSinkConnectorConfig().getPassWord()); + sinkDataSource = createDruidDataSource(sinkConfig.getUrl(), + sinkConfig.getUserName(), + sinkConfig.getPassWord()); } diff --git a/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/sink/connector/CanalSinkConnector.java b/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/sink/connector/CanalSinkConnector.java index 1888e204ac..8f9df7595b 100644 --- a/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/sink/connector/CanalSinkConnector.java +++ b/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/sink/connector/CanalSinkConnector.java @@ -31,6 +31,7 @@ import org.apache.eventmesh.connector.canal.sink.DbLoadData; import org.apache.eventmesh.connector.canal.sink.DbLoadData.TableLoadData; import org.apache.eventmesh.connector.canal.sink.DbLoadMerger; +import org.apache.eventmesh.connector.canal.source.table.RdbTableMgr; import org.apache.eventmesh.openconnect.api.ConnectorCreateService; import org.apache.eventmesh.openconnect.api.connector.ConnectorContext; import org.apache.eventmesh.openconnect.api.connector.SinkConnectorContext; @@ -88,6 +89,7 @@ public class CanalSinkConnector implements Sink, ConnectorCreateService { private int batchSize = 50; private boolean useBatch = true; + private RdbTableMgr tableMgr; @Override public Class configClass() { @@ -107,12 +109,13 @@ public void init(ConnectorContext connectorContext) throws Exception { this.sinkConfig = (CanalSinkConfig) sinkConnectorContext.getSinkConfig(); this.batchSize = sinkConfig.getBatchSize(); this.useBatch = sinkConfig.getUseBatch(); - DatabaseConnection.sinkConfig = this.sinkConfig; + DatabaseConnection.sinkConfig = this.sinkConfig.getSinkConnectorConfig(); DatabaseConnection.initSinkConnection(); jdbcTemplate = new JdbcTemplate(DatabaseConnection.sinkDataSource); dbDialect = new MysqlDialect(jdbcTemplate, new DefaultLobHandler()); interceptor = new SqlBuilderLoadInterceptor(); interceptor.setDbDialect(dbDialect); + tableMgr = new RdbTableMgr(sinkConfig.getSinkConnectorConfig(), DatabaseConnection.sinkDataSource); executor = new ThreadPoolExecutor(sinkConfig.getPoolSize(), sinkConfig.getPoolSize(), 0L, @@ -124,7 +127,7 @@ public void init(ConnectorContext connectorContext) throws Exception { @Override public void start() throws Exception { - + tableMgr.start(); } @Override @@ -147,7 +150,7 @@ public void put(List sinkRecords) { DbLoadContext context = new DbLoadContext(); for (ConnectRecord connectRecord : sinkRecords) { List canalConnectRecordList = (List) connectRecord.getData(); - canalConnectRecordList = filterRecord(canalConnectRecordList, sinkConfig); + canalConnectRecordList = filterRecord(canalConnectRecordList); if (isDdlDatas(canalConnectRecordList)) { doDdl(context, canalConnectRecordList); } else { @@ -179,10 +182,9 @@ private boolean isDdlDatas(List canalConnectRecordList) { return result; } - private List filterRecord(List canalConnectRecordList, CanalSinkConfig sinkConfig) { + private List filterRecord(List canalConnectRecordList) { return canalConnectRecordList.stream() - .filter(record -> sinkConfig.getSinkConnectorConfig().getSchemaName().equalsIgnoreCase(record.getSchemaName()) - && sinkConfig.getSinkConnectorConfig().getTableName().equalsIgnoreCase(record.getTableName())) + .filter(record -> tableMgr.getTable(record.getSchemaName(), record.getTableName()) != null) .collect(Collectors.toList()); } diff --git a/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/sink/connector/CanalSinkFullConnector.java b/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/sink/connector/CanalSinkFullConnector.java index a43f6a4413..fb49cc4473 100644 --- a/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/sink/connector/CanalSinkFullConnector.java +++ b/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/sink/connector/CanalSinkFullConnector.java @@ -2,19 +2,33 @@ import org.apache.eventmesh.common.config.connector.Config; import org.apache.eventmesh.common.config.connector.rdb.canal.CanalSinkFullConfig; +import org.apache.eventmesh.common.exception.EventMeshException; +import org.apache.eventmesh.connector.canal.DatabaseConnection; +import org.apache.eventmesh.connector.canal.source.table.RdbTableMgr; import org.apache.eventmesh.openconnect.api.ConnectorCreateService; import org.apache.eventmesh.openconnect.api.connector.ConnectorContext; import org.apache.eventmesh.openconnect.api.connector.SinkConnectorContext; import org.apache.eventmesh.openconnect.api.sink.Sink; import org.apache.eventmesh.openconnect.offsetmgmt.api.data.ConnectRecord; +import java.sql.SQLException; import java.util.List; +import java.util.concurrent.locks.LockSupport; +import com.alibaba.druid.pool.DruidPooledConnection; + +import lombok.extern.slf4j.Slf4j; + +@Slf4j public class CanalSinkFullConnector implements Sink, ConnectorCreateService { private CanalSinkFullConfig config; + private RdbTableMgr tableMgr; @Override public void start() throws Exception { - + if (config.getSinkConfig() == null) { + throw new EventMeshException(String.format("[%s] sink config is null", this.getClass())); + } + tableMgr.start(); } @Override @@ -35,11 +49,20 @@ public Class configClass() { @Override public void init(Config config) throws Exception { this.config = (CanalSinkFullConfig) config; + init(); } @Override public void init(ConnectorContext connectorContext) throws Exception { this.config = (CanalSinkFullConfig)((SinkConnectorContext)connectorContext).getSinkConfig(); + init(); + } + + private void init() { + DatabaseConnection.sinkConfig = this.config.getSinkConfig(); + DatabaseConnection.initSinkConnection(); + + tableMgr = new RdbTableMgr(this.config.getSinkConfig(), DatabaseConnection.sinkDataSource); } @Override @@ -54,6 +77,14 @@ public String name() { @Override public void put(List sinkRecords) { - + if (sinkRecords == null || sinkRecords.isEmpty()) { + return; + } + try(DruidPooledConnection connection = DatabaseConnection.sinkDataSource.getConnection()) { + connection.prepareStatement(); + } catch (SQLException e) { + log.warn("create sink connection "); + LockSupport.parkNanos(3000 * 1000L); + } } } diff --git a/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/source/CanalFullProducer.java b/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/source/connector/CanalFullProducer.java similarity index 99% rename from eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/source/CanalFullProducer.java rename to eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/source/connector/CanalFullProducer.java index 979346512c..40575f0547 100644 --- a/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/source/CanalFullProducer.java +++ b/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/source/connector/CanalFullProducer.java @@ -1,7 +1,5 @@ -package org.apache.eventmesh.connector.canal.source; +package org.apache.eventmesh.connector.canal.source.connector; -import lombok.extern.slf4j.Slf4j; -import org.apache.commons.lang3.StringUtils; import org.apache.eventmesh.common.config.connector.rdb.canal.CanalMySQLType; import org.apache.eventmesh.common.config.connector.rdb.canal.JobRdbFullPosition; import org.apache.eventmesh.common.config.connector.rdb.canal.RdbColumnDefinition; @@ -14,10 +12,9 @@ import org.apache.eventmesh.common.utils.JsonUtils; import org.apache.eventmesh.connector.canal.source.position.TableFullPosition; import org.apache.eventmesh.openconnect.offsetmgmt.api.data.ConnectRecord; -import org.locationtech.jts.geom.GeometryFactory; -import org.locationtech.jts.io.WKBReader; -import javax.sql.DataSource; +import org.apache.commons.lang3.StringUtils; + import java.math.BigDecimal; import java.sql.Connection; import java.sql.PreparedStatement; @@ -40,6 +37,13 @@ import java.util.concurrent.atomic.AtomicReference; import java.util.concurrent.locks.LockSupport; +import javax.sql.DataSource; + +import org.locationtech.jts.geom.GeometryFactory; +import org.locationtech.jts.io.WKBReader; + +import lombok.extern.slf4j.Slf4j; + @Slf4j public class CanalFullProducer { @@ -396,6 +400,8 @@ private String generateScanSql(boolean isEquals) { generateQueryColumnsSql(builder, tableDefinition.getColumnDefinitions().values()); builder.append(" from "); builder.append(Constants.MySQLQuot); + builder.append(tableDefinition.getSchemaName()); + builder.append("."); builder.append(tableDefinition.getTableName()); builder.append(Constants.MySQLQuot); buildWhereSql(builder, tableDefinition, isEquals); diff --git a/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/source/connector/CanalSourceConnector.java b/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/source/connector/CanalSourceConnector.java index a598352f73..4b96177319 100644 --- a/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/source/connector/CanalSourceConnector.java +++ b/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/source/connector/CanalSourceConnector.java @@ -24,6 +24,7 @@ import org.apache.eventmesh.common.remote.offset.canal.CanalRecordPartition; import org.apache.eventmesh.common.utils.JsonUtils; import org.apache.eventmesh.connector.canal.CanalConnectRecord; +import org.apache.eventmesh.connector.canal.DatabaseConnection; import org.apache.eventmesh.connector.canal.source.EntryParser; import org.apache.eventmesh.connector.canal.source.table.RdbTableMgr; import org.apache.eventmesh.openconnect.api.ConnectorCreateService; @@ -149,7 +150,7 @@ protected void startEventParserInternal(CanalEventParser parser, boolean isGroup return instance; } }); - tableMgr = new RdbTableMgr(sourceConfig.getSourceConnectorConfig()); + tableMgr = new RdbTableMgr(sourceConfig.getSourceConnectorConfig(), DatabaseConnection.sourceDataSource); } private Canal buildCanal(CanalSourceConfig sourceConfig) { diff --git a/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/source/connector/CanalSourceFullConnector.java b/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/source/connector/CanalSourceFullConnector.java index 60a84b4241..b24a8b36e1 100644 --- a/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/source/connector/CanalSourceFullConnector.java +++ b/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/source/connector/CanalSourceFullConnector.java @@ -11,7 +11,6 @@ import org.apache.eventmesh.common.exception.EventMeshException; import org.apache.eventmesh.common.utils.JsonUtils; import org.apache.eventmesh.connector.canal.DatabaseConnection; -import org.apache.eventmesh.connector.canal.source.CanalFullProducer; import org.apache.eventmesh.connector.canal.source.position.CanalFullPositionMgr; import org.apache.eventmesh.connector.canal.source.position.TableFullPosition; import org.apache.eventmesh.connector.canal.source.table.RdbSimpleTable; @@ -22,18 +21,14 @@ import org.apache.eventmesh.openconnect.api.source.Source; import org.apache.eventmesh.openconnect.offsetmgmt.api.data.ConnectRecord; -import java.util.HashMap; import java.util.LinkedList; import java.util.List; -import java.util.Map; import java.util.concurrent.BlockingQueue; import java.util.concurrent.LinkedBlockingQueue; import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; -import javax.sql.DataSource; - import lombok.extern.slf4j.Slf4j; @Slf4j @@ -42,7 +37,6 @@ public class CanalSourceFullConnector extends AbstractComponent implements Sourc private CanalFullPositionMgr positionMgr; private RdbTableMgr tableMgr; private ThreadPoolExecutor executor; - private final Map dataSources = new HashMap<>(); private final BlockingQueue> queue = new LinkedBlockingQueue<>(); private final AtomicBoolean flag = new AtomicBoolean(true); @@ -62,10 +56,6 @@ protected void startup() throws Exception { for (RdbTableDefinition table : db.getTables()) { try { log.info("it will create producer of db [{}] table [{}]", db.getSchemaName(), table.getTableName()); - DataSource dataSource = dataSources.computeIfAbsent(db.getSchemaName(), - k -> DatabaseConnection.createDruidDataSource(config.getConnectorConfig().getUrl(), - config.getConnectorConfig().getUserName(), - config.getConnectorConfig().getPassWord())); RdbSimpleTable simpleTable = new RdbSimpleTable(db.getSchemaName(), table.getTableName()); JobRdbFullPosition position = positionMgr.getPosition(simpleTable); if (position == null) { @@ -78,7 +68,7 @@ protected void startup() throws Exception { db.getSchemaName(), table.getTableName())); } - producers.add(new CanalFullProducer(queue, dataSource, (MySQLTableDef) tableDefinition, + producers.add(new CanalFullProducer(queue, DatabaseConnection.sourceDataSource, (MySQLTableDef) tableDefinition, JsonUtils.parseObject(position.getPrimaryKeyRecords(), TableFullPosition.class), config.getFlushSize())); } catch (Exception e) { @@ -94,6 +84,22 @@ protected void startup() throws Exception { @Override protected void shutdown() throws Exception { flag.set(false); + if (!executor.isShutdown()) { + executor.shutdown(); + try { + if (!executor.awaitTermination(5, TimeUnit.SECONDS)) { + log.warn("wait thread pool shutdown timeout, it will shutdown now"); + executor.shutdownNow(); + } + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + log.info("shutdown thread pool fail"); + } + } + if (DatabaseConnection.sourceDataSource != null) { + DatabaseConnection.sourceDataSource.close(); + log.info("data source has been closed"); + } } @Override @@ -109,14 +115,21 @@ public Class configClass() { @Override public void init(Config config) throws Exception { this.config = (CanalSourceFullConfig) config; + init(); + } + + private void init() { + DatabaseConnection.sourceConfig = this.config.getConnectorConfig(); + DatabaseConnection.initSourceConnection(); + this.tableMgr = new RdbTableMgr(config.getConnectorConfig(), DatabaseConnection.sourceDataSource); + this.positionMgr = new CanalFullPositionMgr(config, tableMgr); } @Override public void init(ConnectorContext connectorContext) throws Exception { SourceConnectorContext sourceConnectorContext = (SourceConnectorContext) connectorContext; this.config = (CanalSourceFullConfig) sourceConnectorContext.getSourceConfig(); - this.tableMgr = new RdbTableMgr(config.getConnectorConfig()); - this.positionMgr = new CanalFullPositionMgr(config, tableMgr); + init(); } @Override diff --git a/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/source/position/CanalFullPositionMgr.java b/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/source/position/CanalFullPositionMgr.java index fa93871b0f..c95c50b5ce 100644 --- a/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/source/position/CanalFullPositionMgr.java +++ b/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/source/position/CanalFullPositionMgr.java @@ -1,8 +1,5 @@ package org.apache.eventmesh.connector.canal.source.position; -import com.alibaba.druid.pool.DruidDataSource; -import lombok.extern.slf4j.Slf4j; -import org.apache.commons.lang3.StringUtils; import org.apache.eventmesh.common.AbstractComponent; import org.apache.eventmesh.common.config.connector.rdb.canal.CanalSourceFullConfig; import org.apache.eventmesh.common.config.connector.rdb.canal.JobRdbFullPosition; @@ -18,7 +15,8 @@ import org.apache.eventmesh.connector.canal.source.table.RdbSimpleTable; import org.apache.eventmesh.connector.canal.source.table.RdbTableMgr; -import javax.sql.DataSource; +import org.apache.commons.lang3.StringUtils; + import java.sql.JDBCType; import java.sql.PreparedStatement; import java.sql.ResultSet; @@ -26,15 +24,18 @@ import java.sql.Statement; import java.util.LinkedHashMap; import java.util.Map; -import java.util.concurrent.ScheduledThreadPoolExecutor; + +import javax.sql.DataSource; + +import lombok.extern.slf4j.Slf4j; @Slf4j public class CanalFullPositionMgr extends AbstractComponent { - private CanalSourceFullConfig config; - private ScheduledThreadPoolExecutor executor; - private Map positions = new LinkedHashMap<>(); - private RdbTableMgr tableMgr; + private final CanalSourceFullConfig config; +// private ScheduledThreadPoolExecutor executor; + private final Map positions = new LinkedHashMap<>(); + private final RdbTableMgr tableMgr; public CanalFullPositionMgr(CanalSourceFullConfig config, RdbTableMgr tableMgr) { this.config = config; @@ -47,12 +48,12 @@ protected void startup() throws Exception { log.info("config or database is null"); return; } - executor = new ScheduledThreadPoolExecutor(1, r -> { - Thread thread = new Thread(r); - thread.setDaemon(true); - thread.setName("task-full-position-timer"); - return thread; - }, new ScheduledThreadPoolExecutor.DiscardOldestPolicy()); +// executor = new ScheduledThreadPoolExecutor(1, r -> { +// Thread thread = new Thread(r); +// thread.setDaemon(true); +// thread.setName("task-full-position-timer"); +// return thread; +// }, new ScheduledThreadPoolExecutor.DiscardOldestPolicy()); prepareRecordPosition(); initPositions(); } @@ -82,32 +83,29 @@ public boolean isFinished() { } private void initPositions() { - try (DruidDataSource dataSource = - DatabaseConnection.createDruidDataSource(config.getConnectorConfig().getUrl(), - config.getConnectorConfig().getUserName(), config.getConnectorConfig().getPassWord())) { - for (RdbDBDefinition database : config.getConnectorConfig().getDatabases()) { - for (RdbTableDefinition table : database.getTables()) { - try { - RdbSimpleTable simpleTable = new RdbSimpleTable(database.getSchemaName(), table.getTableName()); - RdbTableDefinition tableDefinition; - if ((tableDefinition = tableMgr.getTable(simpleTable)) == null) { - log.error("db [{}] table [{}] definition is null", database.getSchemaName(), - table.getTableName()); - continue; - } - log.info("init position of data [{}] table [{}]", database.getSchemaName(), - table.getTableName()); - - JobRdbFullPosition recordPosition = positions.get(simpleTable); - if (recordPosition == null || !recordPosition.isFinished()) { - positions.put(simpleTable, fetchTableInfo(dataSource, (MySQLTableDef) tableDefinition, - recordPosition)); - } - } catch (Exception e) { - log.error("process schema [{}] table [{}] position fail", database.getSchemaName(), - table.getTableName(), e); + for (RdbDBDefinition database : config.getConnectorConfig().getDatabases()) { + for (RdbTableDefinition table : database.getTables()) { + try { + RdbSimpleTable simpleTable = new RdbSimpleTable(database.getSchemaName(), table.getTableName()); + RdbTableDefinition tableDefinition; + if ((tableDefinition = tableMgr.getTable(simpleTable)) == null) { + log.error("db [{}] table [{}] definition is null", database.getSchemaName(), + table.getTableName()); + continue; } + log.info("init position of data [{}] table [{}]", database.getSchemaName(), + table.getTableName()); + + JobRdbFullPosition recordPosition = positions.get(simpleTable); + if (recordPosition == null || !recordPosition.isFinished()) { + positions.put(simpleTable, fetchTableInfo(DatabaseConnection.sourceDataSource, (MySQLTableDef) tableDefinition, + recordPosition)); + } + } catch (Exception e) { + log.error("process schema [{}] table [{}] position fail", database.getSchemaName(), + table.getTableName(), e); } + } } } @@ -131,7 +129,7 @@ private JobRdbFullPosition fetchTableInfo(DataSource dataSource, MySQLTableDef t if (recordPosition != null) { if (StringUtils.isNotBlank(recordPosition.getPrimaryKeyRecords())) { TableFullPosition record = JsonUtils.parseObject(recordPosition.getPrimaryKeyRecords(), - TableFullPosition.class); + TableFullPosition.class); if (record != null && record.getCurPrimaryKeyCols() != null && !record.getCurPrimaryKeyCols().isEmpty()) { position.setCurPrimaryKeyCols(record.getCurPrimaryKeyCols()); } @@ -148,9 +146,10 @@ private JobRdbFullPosition fetchTableInfo(DataSource dataSource, MySQLTableDef t private long queryCurTableRowCount(DataSource datasource, MySQLTableDef tableDefinition) throws SQLException { String sql = - "select `AVG_ROW_LENGTH`,`DATA_LENGTH` from information_schema.TABLES where `TABLE_SCHEMA`='" + tableDefinition.getSchemaName() +"' and `TABLE_NAME`='" + tableDefinition.getTableName() +"'"; + "select `AVG_ROW_LENGTH`,`DATA_LENGTH` from information_schema.TABLES where `TABLE_SCHEMA`='" + tableDefinition.getSchemaName() + + "' and `TABLE_NAME`='" + tableDefinition.getTableName() + "'"; try (Statement statement = datasource.getConnection().createStatement(); ResultSet resultSet = - statement.executeQuery(sql)) { + statement.executeQuery(sql)) { long result = 0L; if (resultSet.next()) { long avgRowLength = resultSet.getLong("AVG_ROW_LENGTH"); @@ -192,8 +191,10 @@ private void setValue2Statement(PreparedStatement ps, Map preMap private Object fetchMinPrimaryKey(DataSource dataSource, MySQLTableDef tableDefinition, Map prePrimary, String curPrimaryKeyCol) throws SQLException { StringBuilder builder = new StringBuilder(); - builder.append("SELECT MIN(").append(Constants.MySQLQuot).append(curPrimaryKeyCol).append(Constants.MySQLQuot).append(") min_primary_key FROM") - .append(Constants.MySQLQuot).append(tableDefinition.getSchemaName()).append(Constants.MySQLQuot).append(".").append(Constants.MySQLQuot).append(tableDefinition.getTableName()).append(Constants.MySQLQuot); + builder.append("SELECT MIN(").append(Constants.MySQLQuot).append(curPrimaryKeyCol).append(Constants.MySQLQuot) + .append(") min_primary_key FROM") + .append(Constants.MySQLQuot).append(tableDefinition.getSchemaName()).append(Constants.MySQLQuot).append(".").append(Constants.MySQLQuot) + .append(tableDefinition.getTableName()).append(Constants.MySQLQuot); appendPrePrimaryKey(prePrimary, builder); String sql = builder.toString(); log.info("fetch min primary sql [{}]", sql); @@ -216,8 +217,10 @@ private Object fetchMinPrimaryKey(DataSource dataSource, MySQLTableDef tableDefi private Object fetchMaxPrimaryKey(DataSource dataSource, MySQLTableDef tableDefinition, Map prePrimary, String curPrimaryKeyCol) throws SQLException { StringBuilder builder = new StringBuilder(); - builder.append("SELECT MAX(").append(Constants.MySQLQuot).append(curPrimaryKeyCol).append(Constants.MySQLQuot).append(") max_primary_key FROM") - .append(Constants.MySQLQuot).append(tableDefinition.getSchemaName()).append(Constants.MySQLQuot).append(".").append(Constants.MySQLQuot).append(tableDefinition.getTableName()).append(Constants.MySQLQuot); + builder.append("SELECT MAX(").append(Constants.MySQLQuot).append(curPrimaryKeyCol).append(Constants.MySQLQuot) + .append(") max_primary_key FROM") + .append(Constants.MySQLQuot).append(tableDefinition.getSchemaName()).append(Constants.MySQLQuot).append(".").append(Constants.MySQLQuot) + .append(tableDefinition.getTableName()).append(Constants.MySQLQuot); appendPrePrimaryKey(prePrimary, builder); String sql = builder.toString(); log.info("fetch max primary sql [{}]", sql); diff --git a/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/source/table/RdbTableMgr.java b/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/source/table/RdbTableMgr.java index 18d134db96..6df431d18d 100644 --- a/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/source/table/RdbTableMgr.java +++ b/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/source/table/RdbTableMgr.java @@ -1,19 +1,15 @@ package org.apache.eventmesh.connector.canal.source.table; -import com.alibaba.druid.pool.DruidDataSource; -import lombok.extern.slf4j.Slf4j; import org.apache.eventmesh.common.AbstractComponent; +import org.apache.eventmesh.common.config.connector.rdb.JdbcConfig; import org.apache.eventmesh.common.config.connector.rdb.canal.CanalMySQLType; import org.apache.eventmesh.common.config.connector.rdb.canal.RdbColumnDefinition; import org.apache.eventmesh.common.config.connector.rdb.canal.RdbDBDefinition; import org.apache.eventmesh.common.config.connector.rdb.canal.RdbTableDefinition; -import org.apache.eventmesh.common.config.connector.rdb.canal.SourceConnectorConfig; import org.apache.eventmesh.common.config.connector.rdb.canal.mysql.MySQLColumnDef; import org.apache.eventmesh.common.config.connector.rdb.canal.mysql.MySQLTableDef; -import org.apache.eventmesh.connector.canal.DatabaseConnection; import org.apache.eventmesh.connector.canal.SqlUtils; -import javax.sql.DataSource; import java.sql.JDBCType; import java.sql.PreparedStatement; import java.sql.ResultSet; @@ -26,16 +22,23 @@ import java.util.List; import java.util.Map; +import javax.sql.DataSource; + +import lombok.extern.slf4j.Slf4j; + /** * Description: */ @Slf4j + public class RdbTableMgr extends AbstractComponent { - private final SourceConnectorConfig config; + private final JdbcConfig config; private final Map tables = new HashMap<>(); + private final DataSource dataSource; - public RdbTableMgr(SourceConnectorConfig config) { + public RdbTableMgr(JdbcConfig config, DataSource dataSource) { this.config = config; + this.dataSource = dataSource; } public RdbTableDefinition getTable(String schema, String tableName) { @@ -54,53 +57,51 @@ protected void startup() { log.warn("init db [{}] position, but it's tables are null", db.getSchemaName()); continue; } - try (DruidDataSource dataSource = DatabaseConnection.createDruidDataSource(config.getUrl(), - config.getUserName(), config.getPassWord())) { - for (RdbTableDefinition table : db.getTables()) { - try { - MySQLTableDef mysqlTable = new MySQLTableDef(); - mysqlTable.setSchemaName(db.getSchemaName()); - mysqlTable.setTableName(table.getTableName()); - List tables = Collections.singletonList(table.getTableName()); - Map> primaryKeys = queryTablePrimaryKey(dataSource, db.getSchemaName(), - tables); - Map> columns = queryColumns(dataSource, db.getSchemaName(), tables); - if (primaryKeys == null || primaryKeys.isEmpty() || primaryKeys.get(table.getTableName()) == null) { - log.warn("init db [{}] table [{}] info, and primary keys are empty", db.getSchemaName(), - table.getTableName()); - } else { - mysqlTable.setPrimaryKeys(new HashSet<>(primaryKeys.get(table.getTableName()))); - } - if (columns == null || columns.isEmpty() || columns.get(table.getTableName()) == null) { - log.warn("init db [{}] table [{}] info, and columns are empty", db.getSchemaName(), - table.getTableName()); - } else { - LinkedHashMap cols = new LinkedHashMap<>(); - columns.get(table.getTableName()).forEach(x -> cols.put(x.getName(), x)); - mysqlTable.setColumnDefinitions(cols); - } - - this.tables.put(new RdbSimpleTable(db.getSchemaName(), table.getTableName()), mysqlTable); - } catch (Exception e) { - log.error("init rdb table schema [{}] table [{}] fail", db.getSchemaName(), - table.getTableName(), e); + for (RdbTableDefinition table : db.getTables()) { + try { + MySQLTableDef mysqlTable = new MySQLTableDef(); + mysqlTable.setSchemaName(db.getSchemaName()); + mysqlTable.setTableName(table.getTableName()); + List tables = Collections.singletonList(table.getTableName()); + Map> primaryKeys = queryTablePrimaryKey(db.getSchemaName(), + tables); + Map> columns = queryColumns(db.getSchemaName(), tables); + if (primaryKeys == null || primaryKeys.isEmpty() || primaryKeys.get(table.getTableName()) == null) { + log.warn("init db [{}] table [{}] info, and primary keys are empty", db.getSchemaName(), + table.getTableName()); + } else { + mysqlTable.setPrimaryKeys(new HashSet<>(primaryKeys.get(table.getTableName()))); + } + if (columns == null || columns.isEmpty() || columns.get(table.getTableName()) == null) { + log.warn("init db [{}] table [{}] info, and columns are empty", db.getSchemaName(), + table.getTableName()); + } else { + LinkedHashMap cols = new LinkedHashMap<>(); + columns.get(table.getTableName()).forEach(x -> cols.put(x.getName(), x)); + mysqlTable.setColumnDefinitions(cols); } + + this.tables.put(new RdbSimpleTable(db.getSchemaName(), table.getTableName()), mysqlTable); + } catch (Exception e) { + log.error("init rdb table schema [{}] table [{}] fail", db.getSchemaName(), + table.getTableName(), e); } } + } } } - private Map> queryTablePrimaryKey(DruidDataSource dataSource, String schema, + private Map> queryTablePrimaryKey(String schema, List tables) throws SQLException { Map> primaryKeys = new LinkedHashMap<>(); String prepareTables = SqlUtils.genPrepareSqlOfInClause(tables.size()); String sql = "select L.TABLE_NAME,L.COLUMN_NAME,R.CONSTRAINT_TYPE from " + - "INFORMATION_SCHEMA.KEY_COLUMN_USAGE L left join INFORMATION_SCHEMA.TABLE_CONSTRAINTS R on L" + - ".TABLE_SCHEMA = R.TABLE_SCHEMA and L.TABLE_NAME = R.TABLE_NAME and L.CONSTRAINT_CATALOG = R" + - ".CONSTRAINT_CATALOG and L.CONSTRAINT_SCHEMA = R.CONSTRAINT_SCHEMA and L.CONSTRAINT_NAME = R" + - ".CONSTRAINT_NAME where L.TABLE_SCHEMA = ? and L.TABLE_NAME in " + prepareTables + " and R" + - ".CONSTRAINT_TYPE IN ('PRIMARY KEY') order by L.ORDINAL_POSITION asc"; + "INFORMATION_SCHEMA.KEY_COLUMN_USAGE L left join INFORMATION_SCHEMA.TABLE_CONSTRAINTS R on L" + + ".TABLE_SCHEMA = R.TABLE_SCHEMA and L.TABLE_NAME = R.TABLE_NAME and L.CONSTRAINT_CATALOG = R" + + ".CONSTRAINT_CATALOG and L.CONSTRAINT_SCHEMA = R.CONSTRAINT_SCHEMA and L.CONSTRAINT_NAME = R" + + ".CONSTRAINT_NAME where L.TABLE_SCHEMA = ? and L.TABLE_NAME in " + prepareTables + " and R" + + ".CONSTRAINT_TYPE IN ('PRIMARY KEY') order by L.ORDINAL_POSITION asc"; try (PreparedStatement statement = dataSource.getConnection().prepareStatement(sql)) { statement.setString(1, schema); SqlUtils.setInClauseParameters(statement, 2, tables); @@ -124,13 +125,13 @@ private Map> queryTablePrimaryKey(DruidDataSource dataSourc return primaryKeys; } - private Map> queryColumns(DataSource dataSource, String schema, List tables) throws SQLException { + private Map> queryColumns(String schema, List tables) throws SQLException { String prepareTables = SqlUtils.genPrepareSqlOfInClause(tables.size()); String sql = "select TABLE_SCHEMA,TABLE_NAME,COLUMN_NAME,IS_NULLABLE,DATA_TYPE,CHARACTER_MAXIMUM_LENGTH," + - "CHARACTER_OCTET_LENGTH,NUMERIC_SCALE,NUMERIC_PRECISION,DATETIME_PRECISION,CHARACTER_SET_NAME," + - "COLLATION_NAME,COLUMN_TYPE,COLUMN_DEFAULT,COLUMN_COMMENT,ORDINAL_POSITION,EXTRA from " + - "INFORMATION_SCHEMA.COLUMNS where TABLE_SCHEMA = ? and TABLE_NAME in " + prepareTables + " order by " + - "ORDINAL_POSITION asc"; + "CHARACTER_OCTET_LENGTH,NUMERIC_SCALE,NUMERIC_PRECISION,DATETIME_PRECISION,CHARACTER_SET_NAME," + + "COLLATION_NAME,COLUMN_TYPE,COLUMN_DEFAULT,COLUMN_COMMENT,ORDINAL_POSITION,EXTRA from " + + "INFORMATION_SCHEMA.COLUMNS where TABLE_SCHEMA = ? and TABLE_NAME in " + prepareTables + " order by " + + "ORDINAL_POSITION asc"; Map> cols = new LinkedHashMap<>(); try (PreparedStatement statement = dataSource.getConnection().prepareStatement(sql)) { statement.setString(1, schema); From 59e16c97915deb8ca6865b5e31c0089fde1dd271 Mon Sep 17 00:00:00 2001 From: sodaRyCN <757083350@qq.com> Date: Fri, 12 Jul 2024 22:08:53 +0800 Subject: [PATCH 08/15] prepare for full write --- .../canal/sink/connector/CanalSinkFullConnector.java | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/sink/connector/CanalSinkFullConnector.java b/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/sink/connector/CanalSinkFullConnector.java index fb49cc4473..dbad92625c 100644 --- a/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/sink/connector/CanalSinkFullConnector.java +++ b/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/sink/connector/CanalSinkFullConnector.java @@ -1,5 +1,7 @@ package org.apache.eventmesh.connector.canal.sink.connector; +import com.alibaba.druid.pool.DruidPooledConnection; +import lombok.extern.slf4j.Slf4j; import org.apache.eventmesh.common.config.connector.Config; import org.apache.eventmesh.common.config.connector.rdb.canal.CanalSinkFullConfig; import org.apache.eventmesh.common.exception.EventMeshException; @@ -15,10 +17,6 @@ import java.util.List; import java.util.concurrent.locks.LockSupport; -import com.alibaba.druid.pool.DruidPooledConnection; - -import lombok.extern.slf4j.Slf4j; - @Slf4j public class CanalSinkFullConnector implements Sink, ConnectorCreateService { private CanalSinkFullConfig config; @@ -81,7 +79,6 @@ public void put(List sinkRecords) { return; } try(DruidPooledConnection connection = DatabaseConnection.sinkDataSource.getConnection()) { - connection.prepareStatement(); } catch (SQLException e) { log.warn("create sink connection "); LockSupport.parkNanos(3000 * 1000L); From bdd91e16d2f0a137c52a31ea00e35f0fb60a4970 Mon Sep 17 00:00:00 2001 From: sodaRyCN <757083350@qq.com> Date: Wed, 17 Jul 2024 17:59:02 +0800 Subject: [PATCH 09/15] close to finish full read and begin full write --- .../rdb/canal/mysql/MySQLTableDef.java | 8 +- .../canal/CanalFullRecordPartition.java | 5 +- .../eventmesh/connector/canal/SqlUtils.java | 235 ++++++++++++++ .../connector/CanalSinkFullConnector.java | 286 +++++++++++++++++- .../source/connector/CanalFullProducer.java | 60 +--- .../canal/source/table/RdbTableMgr.java | 3 +- 6 files changed, 533 insertions(+), 64 deletions(-) diff --git a/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/rdb/canal/mysql/MySQLTableDef.java b/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/rdb/canal/mysql/MySQLTableDef.java index 439d80bcd5..0aa9a0f0d7 100644 --- a/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/rdb/canal/mysql/MySQLTableDef.java +++ b/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/rdb/canal/mysql/MySQLTableDef.java @@ -1,13 +1,13 @@ package org.apache.eventmesh.common.config.connector.rdb.canal.mysql; -import lombok.Data; -import lombok.EqualsAndHashCode; -import org.apache.eventmesh.common.config.connector.rdb.canal.RdbColumnDefinition; import org.apache.eventmesh.common.config.connector.rdb.canal.RdbTableDefinition; import java.util.Map; import java.util.Set; +import lombok.Data; +import lombok.EqualsAndHashCode; + /** * Description: */ @@ -15,5 +15,5 @@ @EqualsAndHashCode(callSuper = true) public class MySQLTableDef extends RdbTableDefinition { private Set primaryKeys; - private Map columnDefinitions; + private Map columnDefinitions; } diff --git a/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/offset/canal/CanalFullRecordPartition.java b/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/offset/canal/CanalFullRecordPartition.java index a325444be5..73626fa78f 100644 --- a/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/offset/canal/CanalFullRecordPartition.java +++ b/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/offset/canal/CanalFullRecordPartition.java @@ -17,18 +17,17 @@ package org.apache.eventmesh.common.remote.offset.canal; +import org.apache.eventmesh.common.remote.offset.RecordPartition; + import lombok.Data; import lombok.EqualsAndHashCode; import lombok.ToString; -import org.apache.eventmesh.common.remote.offset.RecordPartition; @Data @ToString @EqualsAndHashCode(callSuper = true) public class CanalFullRecordPartition extends RecordPartition { - private String schema; - private String table; @Override public Class getRecordPartitionClass() { diff --git a/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/SqlUtils.java b/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/SqlUtils.java index ed72a9eb4d..3bbadd5382 100644 --- a/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/SqlUtils.java +++ b/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/SqlUtils.java @@ -20,6 +20,9 @@ import com.mysql.cj.MysqlType; import org.apache.commons.beanutils.ConvertUtilsBean; import org.apache.commons.lang.StringUtils; + +import org.locationtech.jts.geom.GeometryFactory; +import org.locationtech.jts.io.WKBReader; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -37,6 +40,7 @@ import java.sql.Time; import java.sql.Timestamp; import java.sql.Types; +import java.time.LocalDateTime; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -44,6 +48,8 @@ import static org.apache.eventmesh.connector.canal.ByteArrayConverter.SQL_BYTES; import static org.apache.eventmesh.connector.canal.SqlTimestampConverter.SQL_TIMESTAMP; +import org.apache.eventmesh.common.exception.EventMeshException; + public class SqlUtils { public static final String REQUIRED_FIELD_NULL_SUBSTITUTE = " "; @@ -52,6 +58,7 @@ public class SqlUtils { private static final Map> sqlTypeToJavaTypeMap = new HashMap>(); private static final ConvertUtilsBean convertUtilsBean = new ConvertUtilsBean(); private static final Logger log = LoggerFactory.getLogger(SqlUtils.class); + private static final WKBReader WKB_READER = new WKBReader(new GeometryFactory()); static { // regist Converter @@ -325,4 +332,232 @@ public static JDBCType toJDBCType(String connectorDataType) { MysqlType mysqlType = MysqlType.getByName(connectorDataType); return JDBCType.valueOf(mysqlType.getJdbcType()); } + + private static BigDecimal toBigDecimal(Object value) { + if (value == null) { + return null; + } + if (value instanceof String) { + String strValue = (String) value; + if (!org.apache.commons.lang3.StringUtils.isNotBlank(strValue)) { + return null; + } + try { + return new BigDecimal(strValue); + } catch (Exception e) { + if ("true".equals(strValue)) { + return BigDecimal.ONE; + } + if ("false".equals(strValue)) { + return BigDecimal.ZERO; + } + return new BigDecimal(strValue); + } + } else if (value instanceof Number) { + if (value instanceof BigDecimal) { + return (BigDecimal) value; + } + if (value instanceof Integer) { + return BigDecimal.valueOf(((Integer) value).longValue()); + } + if (value instanceof Long) { + return BigDecimal.valueOf(((Long) value)); + } + if (value instanceof Double) { + return BigDecimal.valueOf(((Double) value)); + } + if (value instanceof Float) { + return BigDecimal.valueOf(((Float) value).doubleValue()); + } + if (value instanceof BigInteger) { + return new BigDecimal((BigInteger)value); + } + if (value instanceof Byte) { + return BigDecimal.valueOf(((Byte) value).longValue()); + } + if (value instanceof Short) { + return BigDecimal.valueOf(((Short) value).longValue()); + } + return null; + } else if (value instanceof Boolean) { + return Boolean.TRUE.equals(value) ? BigDecimal.ONE : BigDecimal.ZERO; + } else { + throw new UnsupportedOperationException("class " + value.getClass() + ", value '" + value + "' , parse to big decimal failed."); + } + } + + protected static Double toDouble(Object value) { + if (value == null) { + return null; + } + if (value instanceof String) { + String strValue = (String) value; + if (org.apache.commons.lang3.StringUtils.isBlank(strValue)) { + return null; + } + try { + return Double.parseDouble(strValue); + } catch (Exception e) { + if ("true".equals(strValue)) { + return 1.0d; + } + if ("false".equals(strValue)) { + return 0.0d; + } + return new BigDecimal(strValue).doubleValue(); + } + } else if (value instanceof Number) { + return ((Number) value).doubleValue(); + } else { + if (value instanceof Boolean) { + return Boolean.TRUE.equals(value) ? 1.0d : 0.0d; + } + throw new UnsupportedOperationException("class " + value.getClass() + ", value '" + value + "' , parse to double failed."); + } + } + + protected static Long toLong(Object value) { + if (value == null) { + return null; + } + if (value instanceof String) { + String strValue = (String) value; + if (org.apache.commons.lang3.StringUtils.isBlank(strValue)) { + return null; + } + try { + return Long.parseLong(strValue); + } catch (Exception e) { + try { + return Long.decode(strValue); + } catch (Exception e2) { + if ("true".equals(strValue)) { + return 1L; + } + if ("false".equals(strValue)) { + return 0L; + } + return new BigDecimal(strValue).longValue(); + } + } + } else if (value instanceof Number) { + return ((Number) value).longValue(); + } else { + if (value instanceof Boolean) { + return Boolean.TRUE.equals(value) ? 1L : 0L; + } + throw new UnsupportedOperationException(value.getClass() + ", value '" + value + "' , parse to long failed."); + } + } + + protected static boolean isZeroTime(Object value) { + if (value == null || org.apache.commons.lang3.StringUtils.isBlank(value.toString())) { + return false; + } + return value.toString().startsWith("0000-00-00"); + } + + protected static LocalDateTime safeToLocalDateTime(Object value) { + return DateTimeUtils.safeToLocalDateTime(value); + } + + public static boolean isHexNumber(String str) { + boolean flag = true; + if (str.startsWith("0x") || str.startsWith("0X")) { + str = str.substring(2); + } + int i = 0; + while (true) { + if (i < str.length()) { + char cc = str.charAt(i); + if (cc != '0' && cc != '1' && cc != '2' && cc != '3' && cc != '4' && cc != '5' && cc != '6' && cc != '7' && cc != '8' && cc != '9' && cc != 'A' && cc != 'B' && cc != 'C' && cc != 'D' && cc != 'E' && cc != 'F' && cc != 'a' && cc != 'b' && cc != 'c' && cc != 'd' && cc != 'e' && cc != 'f') { + flag = false; + break; + } + i++; + } else { + break; + } + } + return flag; + } + + protected static byte[] safeToBytes(Object value) { + if (value == null) { + return null; + } + if (value instanceof String) { + String strVal = (String) value; + if ((strVal.startsWith("0x") || strVal.startsWith("0X")) && isHexNumber(strVal)) { + return hex2bytes(strVal.substring(2)); + } + return ((String) value).getBytes(StandardCharsets.ISO_8859_1); + } else if (value instanceof byte[]) { + return (byte[]) value; + } else { + throw new UnsupportedOperationException("class " + value.getClass() + ", value '" + value + "' , safeToBytes failed."); + } + } + + public static String toGeometry(Object value) throws Exception { + if (value == null) { + return null; + } + if (value instanceof String) { + String strVal = (String) value; + if (!strVal.startsWith("0x") && !strVal.startsWith("0X")) { + return (String) value; + } + return WKB_READER.read(hex2bytes(strVal.substring(2))).toText(); + } else if (value instanceof byte[]) { + return WKB_READER.read((byte[]) value).toText(); + } else { + throw new UnsupportedOperationException("class " + value.getClass() + ", value '" + value + "' , " + + "safeToGisWKT" + + " failed."); + } + } + + public static byte[] hex2bytes(String hexStr) { + if (hexStr == null) + return null; + if (org.apache.commons.lang3.StringUtils.isBlank(hexStr)) { + return new byte[0]; + } + + if (hexStr.length() % 2 == 1) { + hexStr = "0" + hexStr; + } + + int count = hexStr.length() / 2; + byte[] ret = new byte[count]; + for (int i = 0; i < count; i++) { + int index = i * 2; + char c1 = hexStr.charAt(index); + char c2 = hexStr.charAt(index + 1); + if (c1 < '0' || c1 > 'F' || c2 < '0' || c2 > 'F') { + throw new EventMeshException(String.format("illegal byte [%s], [%s]", c1, c2)); + } + ret[i] = (byte) ((byte) c1 << 4); + ret[i] = (byte) (ret[i] | (byte) (c2)); + } + return ret; + } + + protected static String toGisWKT(Object value) throws Exception { + if (value == null) { + return null; + } + if (value instanceof String) { + String strVal = (String) value; + if (!strVal.startsWith("0x") && !strVal.startsWith("0X")) { + return (String) value; + } + return WKB_READER.read(hex2bytes(strVal.substring(2))).toText(); + } else if (value instanceof byte[]) { + return WKB_READER.read((byte[]) value).toText(); + } else { + throw new UnsupportedOperationException("class " + value.getClass() + ", value '" + value + "' , parse to gis failed."); + } + } } diff --git a/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/sink/connector/CanalSinkFullConnector.java b/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/sink/connector/CanalSinkFullConnector.java index dbad92625c..c3d2f67002 100644 --- a/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/sink/connector/CanalSinkFullConnector.java +++ b/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/sink/connector/CanalSinkFullConnector.java @@ -4,7 +4,11 @@ import lombok.extern.slf4j.Slf4j; import org.apache.eventmesh.common.config.connector.Config; import org.apache.eventmesh.common.config.connector.rdb.canal.CanalSinkFullConfig; +import org.apache.eventmesh.common.config.connector.rdb.canal.mysql.Constants; +import org.apache.eventmesh.common.config.connector.rdb.canal.mysql.MySQLColumnDef; +import org.apache.eventmesh.common.config.connector.rdb.canal.mysql.MySQLTableDef; import org.apache.eventmesh.common.exception.EventMeshException; +import org.apache.eventmesh.common.remote.offset.canal.CanalFullRecordOffset; import org.apache.eventmesh.connector.canal.DatabaseConnection; import org.apache.eventmesh.connector.canal.source.table.RdbTableMgr; import org.apache.eventmesh.openconnect.api.ConnectorCreateService; @@ -13,19 +17,33 @@ import org.apache.eventmesh.openconnect.api.sink.Sink; import org.apache.eventmesh.openconnect.offsetmgmt.api.data.ConnectRecord; +import org.apache.commons.lang3.StringUtils; + +import java.math.BigDecimal; +import java.math.BigInteger; +import java.nio.charset.StandardCharsets; +import java.sql.PreparedStatement; import java.sql.SQLException; +import java.sql.Types; +import java.time.LocalDateTime; +import java.util.ArrayList; import java.util.List; +import java.util.Map; import java.util.concurrent.locks.LockSupport; +import org.locationtech.jts.io.WKBReader; + +import com.alibaba.druid.pool.DruidPooledConnection; + +import lombok.extern.slf4j.Slf4j; + @Slf4j public class CanalSinkFullConnector implements Sink, ConnectorCreateService { private CanalSinkFullConfig config; private RdbTableMgr tableMgr; @Override public void start() throws Exception { - if (config.getSinkConfig() == null) { - throw new EventMeshException(String.format("[%s] sink config is null", this.getClass())); - } + tableMgr.start(); } @@ -57,6 +75,9 @@ public void init(ConnectorContext connectorContext) throws Exception { } private void init() { + if (config.getSinkConfig() == null) { + throw new EventMeshException(String.format("[%s] sink config is null", this.getClass())); + } DatabaseConnection.sinkConfig = this.config.getSinkConfig(); DatabaseConnection.initSinkConnection(); @@ -75,13 +96,268 @@ public String name() { @Override public void put(List sinkRecords) { - if (sinkRecords == null || sinkRecords.isEmpty()) { + if (sinkRecords == null || sinkRecords.isEmpty() || sinkRecords.get(0) == null) { + if (log.isDebugEnabled()) { + log.debug("[{}] got sink records are none", this.getClass()); + } + return; + } + ConnectRecord record = sinkRecords.get(0); + List> data = (List>)record.getData(); + if (data == null || data.isEmpty()) { + if (log.isDebugEnabled()) { + log.debug("[{}] got rows data is none", this.getClass()); + } + return; + } + CanalFullRecordOffset offset = (CanalFullRecordOffset) record.getPosition().getRecordOffset(); + if (offset == null || offset.getPosition() == null) { + if (log.isDebugEnabled()) { + log.debug("[{}] got canal full offset is none", this.getClass()); + } return; } - try(DruidPooledConnection connection = DatabaseConnection.sinkDataSource.getConnection()) { + + MySQLTableDef tableDefinition = (MySQLTableDef)tableMgr.getTable(offset.getPosition().getSchema(), offset.getPosition().getTableName()); + if (tableDefinition == null) { + log.warn("target schema [{}] table [{}] is not exists", offset.getPosition().getSchema(), offset.getPosition().getTableName()); + return; + } + List cols = new ArrayList<>(tableDefinition.getColumnDefinitions().values()); + String sql = generateInsertPrepareSql(offset.getPosition().getSchema(), offset.getPosition().getTableName(), + cols); + + + try(DruidPooledConnection connection = DatabaseConnection.sinkDataSource.getConnection();PreparedStatement statement = + connection.prepareStatement(sql)) { + for (Map col : data) { + setPrepareParams(statement, col, cols); + statement.addBatch(); + } + statement.executeBatch(); } catch (SQLException e) { log.warn("create sink connection "); LockSupport.parkNanos(3000 * 1000L); + } catch (Exception e) { + // todo rollback + } + } + + private void setPrepareParams(PreparedStatement preparedStatement, Map col, List columnDefs) throws Exception { + for (int i = 0; i <= columnDefs.size(); i++) { + writeColumn(preparedStatement, i + 1, columnDefs.get(i), col.get(columnDefs.get(i))); } } + + public void writeColumn(PreparedStatement ps, int index, MySQLColumnDef colType, Object value) throws Exception { + LocalDateTime colValue; + LocalDateTime colValue2; + if (colType == null) { + String colVal = null; + if (value != null) { + colVal = value.toString(); + } + if (colVal == null) { + ps.setNull(index, Types.VARCHAR); + } else { + ps.setString(index, colVal); + } + } else if (value == null) { + ps.setNull(index, colType.getJdbcType().getVendorTypeNumber()); + } else { + switch (colType.getType()) { + case TINYINT: + case SMALLINT: + case MEDIUMINT: + case INT: + Long longValue = toLong(value); + if (longValue == null) { + ps.setNull(index, 4); + return; + } else { + ps.setLong(index, longValue); + return; + } + case BIGINT: + case DECIMAL: + BigDecimal bigDecimalValue = toBigDecimal(value); + if (bigDecimalValue == null) { + ps.setNull(index, 3); + return; + } else { + ps.setBigDecimal(index, bigDecimalValue); + return; + } + case FLOAT: + case DOUBLE: + Double colValue6 = toDouble(value); + if (colValue6 == null) { + ps.setNull(index, 8); + return; + } else { + ps.setDouble(index, colValue6); + return; + } + case DATE: + case DATETIME: + case TIMESTAMP: + if (!isZeroTime(value)) { + try { + colValue2 = safeToLocalDateTime(value); + } catch (Exception e) { + ps.setString(index, CrwUtils.convertToString(value)); + return; + } + } else if (StringUtils.isNotBlank(options.getDefaultZeroDate())) { + colValue2 = safeToLocalDateTime(options.getDefaultZeroDate()); + } else { + ps.setObject(index, value); + return; + } + if (colValue2 == null) { + ps.setNull(index, 93); + return; + } else { + ps.setString(index, WellKnowFormat.WKF_DATE_TIME24_S9.format(colValue2)); + return; + } + case TIME: + String colValue7 = safeToMySqlTime(value, options); + if (StringUtils.isBlank(colValue7)) { + ps.setNull(index, 12); + return; + } else { + ps.setString(index, colValue7); + return; + } + case YEAR: + if (!isZeroTime(value)) { + colValue = safeToLocalDateTime(value); + } else if (StringUtils.isNotBlank(options.getDefaultZeroDate())) { + colValue = safeToLocalDateTime(options.getDefaultZeroDate()); + } else { + ps.setInt(index, 0); + return; + } + if (colValue == null) { + ps.setNull(index, 4); + return; + } else { + ps.setInt(index, colValue.getYear()); + return; + } + case CHAR: + case VARCHAR: + case TINYTEXT: + case TEXT: + case MEDIUMTEXT: + case LONGTEXT: + case ENUM: + case SET: + String colValue8 = safeToString(value); + if (colValue8 == null) { + ps.setNull(index, 12); + return; + } else { + ps.setString(index, colValue8); + return; + } + case JSON: + String colValue9 = safeToJson(value); + if (colValue9 == null) { + ps.setNull(index, 12); + return; + } else { + ps.setString(index, colValue9); + return; + } + case BIT: + if (value instanceof Boolean) { + byte[] bArr = new byte[1]; + bArr[0] = (byte) (Boolean.TRUE.equals(value) ? 1 : 0); + ps.setBytes(index, bArr); + return; + } else if (value instanceof Number) { + ps.setBytes(index, numberToBinaryArray((Number) value)); + return; + } else if ((value instanceof byte[]) || value.toString().startsWith("0x") || value.toString().startsWith("0X")) { + byte[] colValue10 = safeToBytes(value); + if (colValue10 == null || colValue10.length == 0) { + ps.setNull(index, -7); + return; + } else { + ps.setBytes(index, colValue10); + return; + } + } else { + ps.setBytes(index, numberToBinaryArray(safeToInt(value))); + return; + } + case BINARY: + case VARBINARY: + case TINYBLOB: + case BLOB: + case MEDIUMBLOB: + case LONGBLOB: + byte[] colValue11 = safeToBytes(value); + if (colValue11 == null) { + ps.setNull(index, -2); + return; + } else { + ps.setBytes(index, colValue11); + return; + } + case GEOMETRY: + case GEOMETRY_COLLECTION: + case GEOM_COLLECTION: + String colValue12 = safeToGisWKT(value); + if (colValue12 == null) { + ps.setNull(index, 12); + return; + } + if (colValue12.length() >= 5 && StringUtils.startsWithIgnoreCase(colValue12.substring(0, 5), "SRID=")) { + colValue12 = colValue12.substring(colValue12.indexOf(59) + 1); + } + ps.setString(index, colValue12); + return; + case POINT: + case LINESTRING: + case POLYGON: + case MULTIPOINT: + case MULTILINESTRING: + case MULTIPOLYGON: + ps.setNull(index, MysqlErrorNumbers.ER_INVALID_GROUP_FUNC_USE); + return; + default: + throw new UnsupportedOperationException("columnType '" + colType + "' Unsupported."); + } + } + } + + private String generateInsertPrepareSql(String schema, String table, List cols) { + StringBuilder builder = new StringBuilder(); + builder.append("INSERT IGNORE INTO "); + builder.append(Constants.MySQLQuot); + builder.append(schema); + builder.append("."); + builder.append(table); + builder.append(Constants.MySQLQuot); + StringBuilder columns = new StringBuilder(); + StringBuilder values = new StringBuilder(); + for (MySQLColumnDef colInfo : cols) { + if (columns.length() > 0) { + columns.append(", "); + values.append(", "); + } + String wrapName = Constants.MySQLQuot + colInfo.getName() + Constants.MySQLQuot; + columns.append(wrapName); + values.append("?"); + } + builder.append("(").append(columns).append(")"); + builder.append(" VALUES "); + builder.append("(").append(values).append(")"); + return builder.toString(); + } + + } diff --git a/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/source/connector/CanalFullProducer.java b/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/source/connector/CanalFullProducer.java index 40575f0547..adb55eefd6 100644 --- a/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/source/connector/CanalFullProducer.java +++ b/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/source/connector/CanalFullProducer.java @@ -10,6 +10,7 @@ import org.apache.eventmesh.common.remote.offset.canal.CanalFullRecordOffset; import org.apache.eventmesh.common.remote.offset.canal.CanalFullRecordPartition; import org.apache.eventmesh.common.utils.JsonUtils; +import org.apache.eventmesh.connector.canal.SqlUtils; import org.apache.eventmesh.connector.canal.source.position.TableFullPosition; import org.apache.eventmesh.openconnect.offsetmgmt.api.data.ConnectRecord; @@ -56,7 +57,7 @@ public class CanalFullProducer { private final AtomicReference choosePrimaryKey = new AtomicReference<>(null); private static final DateTimeFormatter DATE_FORMATTER = DateTimeFormatter.ofPattern("yyyy-MM-dd"); private static final DateTimeFormatter DATE_STAMP_FORMATTER = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss"); - private static final WKBReader WKB_READER = new WKBReader(new GeometryFactory()); + public CanalFullProducer(BlockingQueue> queue, DataSource dataSource, MySQLTableDef tableDefinition, TableFullPosition position, int flushSize) { @@ -95,10 +96,10 @@ public void start(AtomicBoolean flag) { Map lastCol = null; while (flag.get() && resultSet.next()) { Map columnValues = new LinkedHashMap<>(); - for (Map.Entry col : + for (Map.Entry col : tableDefinition.getColumnDefinitions().entrySet()) { columnValues.put(col.getKey(), readColumn(resultSet, col.getKey(), - ((MySQLColumnDef) col.getValue()).getType())); + col.getValue().getType())); } lastCol = columnValues; rows.add(lastCol); @@ -145,10 +146,10 @@ private void commitConnectRecord(List> rows) throws Interrup CanalFullRecordOffset offset = new CanalFullRecordOffset(); JobRdbFullPosition jobRdbFullPosition = new JobRdbFullPosition(); jobRdbFullPosition.setPrimaryKeyRecords(JsonUtils.toJSONString(position)); + jobRdbFullPosition.setTableName(tableDefinition.getTableName()); + jobRdbFullPosition.setSchema(tableDefinition.getSchemaName()); offset.setPosition(jobRdbFullPosition); CanalFullRecordPartition partition = new CanalFullRecordPartition(); - partition.setSchema(tableDefinition.getSchemaName()); - partition.setTable(tableDefinition.getTableName()); records.add(new ConnectRecord(partition, offset, System.currentTimeMillis(), rows)); queue.put(records); } @@ -229,7 +230,7 @@ public Object readColumn(ResultSet rs, String col, CanalMySQLType colType) throw case LONGBLOB: return rs.getBytes(col); case GEOMETRY: - return toGeometry("0x" + rs.getString(col)); + return SqlUtils.toGeometry("0x" + rs.getString(col)); case GEOMETRY_COLLECTION: case GEOM_COLLECTION: case POINT: @@ -244,24 +245,7 @@ public Object readColumn(ResultSet rs, String col, CanalMySQLType colType) throw } } - protected static String toGeometry(Object value) throws Exception { - if (value == null) { - return null; - } - if (value instanceof String) { - String strVal = (String) value; - if (!strVal.startsWith("0x") && !strVal.startsWith("0X")) { - return (String) value; - } - return WKB_READER.read(hex2bytes(strVal.substring(2))).toText(); - } else if (value instanceof byte[]) { - return WKB_READER.read((byte[]) value).toText(); - } else { - throw new UnsupportedOperationException("class " + value.getClass() + ", value '" + value + "' , " + - "safeToGisWKT" + - " failed."); - } - } + private void refreshPosition(Map lastCol) { Map nextPosition = new LinkedHashMap<>(); @@ -311,7 +295,7 @@ private void setPrepareStatementValue(PreparedStatement statement) throws SQLExc if (str.startsWith("0x")) { hexStr = str.substring(str.indexOf("0x")); } - byte[] bytes = hex2bytes(hexStr); + byte[] bytes = SqlUtils.hex2bytes(hexStr); statement.setBytes(1, bytes); break; case DATE: @@ -350,33 +334,9 @@ private void setPrepareStatementValue(PreparedStatement statement) throws SQLExc } } - public static byte[] hex2bytes(String hexStr) { - if (hexStr == null) - return null; - if (StringUtils.isBlank(hexStr)) { - return new byte[0]; - } - - if (hexStr.length() % 2 == 1) { - hexStr = "0" + hexStr; - } - int count = hexStr.length() / 2; - byte[] ret = new byte[count]; - for (int i = 0; i < count; i++) { - int index = i * 2; - char c1 = hexStr.charAt(index); - char c2 = hexStr.charAt(index + 1); - if (c1 < '0' || c1 > 'F' || c2 < '0' || c2 > 'F') { - throw new EventMeshException(String.format("illegal byte [%s], [%s]", c1, c2)); - } - ret[i] = (byte) ((byte) c1 << 4); - ret[i] = (byte) (ret[i] | (byte) (c2)); - } - return ret; - } - private void generateQueryColumnsSql(StringBuilder builder, Collection rdbColDefs) { + private void generateQueryColumnsSql(StringBuilder builder, Collection rdbColDefs) { if (rdbColDefs == null || rdbColDefs.isEmpty()) { builder.append("*"); return; diff --git a/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/source/table/RdbTableMgr.java b/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/source/table/RdbTableMgr.java index 6df431d18d..470de5f39a 100644 --- a/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/source/table/RdbTableMgr.java +++ b/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/source/table/RdbTableMgr.java @@ -3,7 +3,6 @@ import org.apache.eventmesh.common.AbstractComponent; import org.apache.eventmesh.common.config.connector.rdb.JdbcConfig; import org.apache.eventmesh.common.config.connector.rdb.canal.CanalMySQLType; -import org.apache.eventmesh.common.config.connector.rdb.canal.RdbColumnDefinition; import org.apache.eventmesh.common.config.connector.rdb.canal.RdbDBDefinition; import org.apache.eventmesh.common.config.connector.rdb.canal.RdbTableDefinition; import org.apache.eventmesh.common.config.connector.rdb.canal.mysql.MySQLColumnDef; @@ -76,7 +75,7 @@ protected void startup() { log.warn("init db [{}] table [{}] info, and columns are empty", db.getSchemaName(), table.getTableName()); } else { - LinkedHashMap cols = new LinkedHashMap<>(); + LinkedHashMap cols = new LinkedHashMap<>(); columns.get(table.getTableName()).forEach(x -> cols.put(x.getName(), x)); mysqlTable.setColumnDefinitions(cols); } From 3a080396ab511ff234a194529d12393085ce3b7a Mon Sep 17 00:00:00 2001 From: sodaRyCN <757083350@qq.com> Date: Fri, 19 Jul 2024 18:10:10 +0800 Subject: [PATCH 10/15] close to finish full read and begin full write --- .../rdb/canal/CanalSinkFullConfig.java | 3 +- .../connector/rdb/canal/RdbDBDefinition.java | 4 +- .../common/remote/offset/RecordPosition.java | 4 + .../eventmesh-connector-canal/build.gradle | 3 +- .../eventmesh/connector/canal/SqlUtils.java | 416 ++++++++++++++++-- .../connector/CanalSinkFullConnector.java | 127 +++--- .../source/connector/CanalFullProducer.java | 32 +- 7 files changed, 477 insertions(+), 112 deletions(-) diff --git a/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/rdb/canal/CanalSinkFullConfig.java b/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/rdb/canal/CanalSinkFullConfig.java index 2143368dd9..33161e07ad 100644 --- a/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/rdb/canal/CanalSinkFullConfig.java +++ b/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/rdb/canal/CanalSinkFullConfig.java @@ -9,5 +9,6 @@ @Data @EqualsAndHashCode(callSuper = true) public class CanalSinkFullConfig extends SinkConfig { - private final SinkConnectorConfig sinkConfig; + private SinkConnectorConfig sinkConfig; + private String zeroDate; } diff --git a/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/rdb/canal/RdbDBDefinition.java b/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/rdb/canal/RdbDBDefinition.java index d6b08503dd..93ab696ad5 100644 --- a/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/rdb/canal/RdbDBDefinition.java +++ b/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/rdb/canal/RdbDBDefinition.java @@ -1,9 +1,9 @@ package org.apache.eventmesh.common.config.connector.rdb.canal; -import lombok.Data; - import java.util.Set; +import lombok.Data; + /** * Description: as class name */ diff --git a/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/offset/RecordPosition.java b/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/offset/RecordPosition.java index c3c2d5dd7a..5f45390b73 100644 --- a/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/offset/RecordPosition.java +++ b/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/offset/RecordPosition.java @@ -19,6 +19,8 @@ import org.apache.eventmesh.common.remote.offset.S3.S3RecordOffset; import org.apache.eventmesh.common.remote.offset.S3.S3RecordPartition; +import org.apache.eventmesh.common.remote.offset.canal.CanalFullRecordOffset; +import org.apache.eventmesh.common.remote.offset.canal.CanalFullRecordPartition; import org.apache.eventmesh.common.remote.offset.canal.CanalRecordOffset; import org.apache.eventmesh.common.remote.offset.canal.CanalRecordPartition; import org.apache.eventmesh.common.remote.offset.file.FileRecordOffset; @@ -40,6 +42,7 @@ public class RecordPosition { @JsonTypeInfo(use = JsonTypeInfo.Id.CLASS) @JsonSubTypes({ @JsonSubTypes.Type(value = CanalRecordPartition.class, name = "CanalRecordPartition"), + @JsonSubTypes.Type(value = CanalFullRecordPartition.class, name = "CanalFullRecordPartition"), @JsonSubTypes.Type(value = FileRecordPartition.class, name = "FileRecordPartition"), @JsonSubTypes.Type(value = S3RecordPartition.class, name = "S3RecordPartition"), @JsonSubTypes.Type(value = KafkaRecordPartition.class, name = "KafkaRecordPartition"), @@ -53,6 +56,7 @@ public class RecordPosition { @JsonTypeInfo(use = JsonTypeInfo.Id.CLASS) @JsonSubTypes({ @JsonSubTypes.Type(value = CanalRecordOffset.class, name = "CanalRecordOffset"), + @JsonSubTypes.Type(value = CanalFullRecordOffset.class, name = "CanalFullRecordOffset"), @JsonSubTypes.Type(value = FileRecordOffset.class, name = "FileRecordOffset"), @JsonSubTypes.Type(value = S3RecordOffset.class, name = "S3RecordOffset"), @JsonSubTypes.Type(value = KafkaRecordOffset.class, name = "KafkaRecordOffset"), diff --git a/eventmesh-connectors/eventmesh-connector-canal/build.gradle b/eventmesh-connectors/eventmesh-connector-canal/build.gradle index 77a9968496..94d85ee22b 100644 --- a/eventmesh-connectors/eventmesh-connector-canal/build.gradle +++ b/eventmesh-connectors/eventmesh-connector-canal/build.gradle @@ -26,7 +26,8 @@ dependencies { implementation "org.locationtech.jts" implementation project(":eventmesh-common") implementation canal - implementation "com.alibaba:druid:1.2.23" + implementation "com.alibaba:druid" + implementation 'com.mysql:mysql-connector-j' compileOnly 'org.projectlombok:lombok' annotationProcessor 'org.projectlombok:lombok' testImplementation "org.mockito:mockito-core" diff --git a/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/SqlUtils.java b/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/SqlUtils.java index 3bbadd5382..27afd22735 100644 --- a/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/SqlUtils.java +++ b/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/SqlUtils.java @@ -17,15 +17,12 @@ package org.apache.eventmesh.connector.canal; -import com.mysql.cj.MysqlType; +import static org.apache.eventmesh.connector.canal.ByteArrayConverter.SQL_BYTES; +import static org.apache.eventmesh.connector.canal.SqlTimestampConverter.SQL_TIMESTAMP; + import org.apache.commons.beanutils.ConvertUtilsBean; import org.apache.commons.lang.StringUtils; -import org.locationtech.jts.geom.GeometryFactory; -import org.locationtech.jts.io.WKBReader; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - import java.io.UnsupportedEncodingException; import java.math.BigDecimal; import java.math.BigInteger; @@ -40,25 +37,37 @@ import java.sql.Time; import java.sql.Timestamp; import java.sql.Types; +import java.time.DateTimeException; +import java.time.Duration; +import java.time.Instant; +import java.time.LocalDate; import java.time.LocalDateTime; +import java.time.LocalTime; +import java.time.OffsetDateTime; +import java.time.ZoneId; +import java.time.temporal.Temporal; import java.util.HashMap; import java.util.List; import java.util.Map; -import static org.apache.eventmesh.connector.canal.ByteArrayConverter.SQL_BYTES; -import static org.apache.eventmesh.connector.canal.SqlTimestampConverter.SQL_TIMESTAMP; +import org.locationtech.jts.geom.GeometryFactory; +import org.locationtech.jts.io.WKBReader; -import org.apache.eventmesh.common.exception.EventMeshException; +import com.mysql.cj.Constants; +import com.mysql.cj.MysqlType; +import com.taobao.tddl.dbsync.binlog.LogBuffer; public class SqlUtils { public static final String REQUIRED_FIELD_NULL_SUBSTITUTE = " "; - public static final String SQLDATE_FORMAT = "yyyy-MM-dd"; - public static final String TIMESTAMP_FORMAT = "yyyy-MM-dd HH:mm:ss"; private static final Map> sqlTypeToJavaTypeMap = new HashMap>(); private static final ConvertUtilsBean convertUtilsBean = new ConvertUtilsBean(); - private static final Logger log = LoggerFactory.getLogger(SqlUtils.class); private static final WKBReader WKB_READER = new WKBReader(new GeometryFactory()); + private static final GeometryFactory GEOMETRY_FACTORY = new GeometryFactory(); + private static final BigDecimal NANO_SEC = new BigDecimal(LogBuffer.DIG_BASE); + private static final LocalDateTime BASE = LocalDateTime.of(1970, 1, 1, 0, 0, 0, 0); + private static final long ONE_HOUR = 3600; + private static final long ONE_MINUTE = 60; static { // regist Converter @@ -333,7 +342,7 @@ public static JDBCType toJDBCType(String connectorDataType) { return JDBCType.valueOf(mysqlType.getJdbcType()); } - private static BigDecimal toBigDecimal(Object value) { + public static BigDecimal toBigDecimal(Object value) { if (value == null) { return null; } @@ -386,7 +395,7 @@ private static BigDecimal toBigDecimal(Object value) { } } - protected static Double toDouble(Object value) { + public static Double toDouble(Object value) { if (value == null) { return null; } @@ -416,7 +425,7 @@ protected static Double toDouble(Object value) { } } - protected static Long toLong(Object value) { + public static Long toLong(Object value) { if (value == null) { return null; } @@ -450,15 +459,332 @@ protected static Long toLong(Object value) { } } - protected static boolean isZeroTime(Object value) { + public static boolean isZeroTime(Object value) { if (value == null || org.apache.commons.lang3.StringUtils.isBlank(value.toString())) { return false; } return value.toString().startsWith("0000-00-00"); } - protected static LocalDateTime safeToLocalDateTime(Object value) { - return DateTimeUtils.safeToLocalDateTime(value); + public static String removeZone(String datetime) { + if (datetime == null || datetime.length() == 0) { + return datetime; + } + int len = datetime.length(); + if (datetime.charAt(len - 1) == 'Z' || datetime.charAt(len - 1) == 'z') { + return datetime.substring(0, len - 1).trim(); + } + if (len >= 7) { + char checkCharAt1 = datetime.charAt(len - 2); + char checkCharAt2 = datetime.charAt(len - 3); + char checkCharAt3 = datetime.charAt(len - 6); + char checkCharAt4 = datetime.charAt(len - 5); + char checkCharAt5 = len >= 9 ? datetime.charAt(len - 9) : ' '; + char checkCharAt6 = datetime.charAt(len - 7); + boolean matchTest1 = (checkCharAt1 == '+' || checkCharAt1 == '-') && len >= 10; + boolean matchTest2 = (checkCharAt2 == '+' || checkCharAt2 == '-') && len >= 11; + boolean matchTest3 = (checkCharAt3 == '+' || checkCharAt3 == '-') && checkCharAt2 == ':'; + boolean matchTest4 = (checkCharAt4 == '+' || checkCharAt4 == '-') && checkCharAt2 == ':'; + boolean matchTest5 = (checkCharAt5 == '+' || checkCharAt5 == '-') && checkCharAt2 == ':' && checkCharAt3 == ':'; + boolean matchTest6 = checkCharAt6 == '+' || checkCharAt6 == '-'; + boolean matchTest7 = checkCharAt4 == '+' || checkCharAt4 == '-'; + if (matchTest1) { + return datetime.substring(0, len - 2).trim(); + } + if (matchTest2) { + return datetime.substring(0, len - 3).trim(); + } + if (matchTest3) { + return datetime.substring(0, len - 6).trim(); + } + if (matchTest4) { + return datetime.substring(0, len - 5).trim(); + } + if (matchTest5) { + return datetime.substring(0, len - 9).trim(); + } + if (matchTest6) { + return datetime.substring(0, len - 7).trim(); + } + if (matchTest7) { + return datetime.substring(0, len - 5).trim(); + } + } + return datetime; + } + + private static LocalDateTime toLocalDateTime(String value) { + if (value.trim().length() >= 4) { + String dateStr2 = removeZone(value); + int len = dateStr2.length(); + if (len == 4) { + return LocalDateTime.of(Integer.parseInt(dateStr2), 1, 1, 0, 0, 0, 0); + } + if (dateStr2.charAt(4) == '-') { + switch (len) { + case 7: + String[] dataParts = dateStr2.split("-"); + return LocalDateTime.of(Integer.parseInt(dataParts[0]), Integer.parseInt(dataParts[1]), 1, 0, 0, 0, 0); + case 8: + case 9: + case 11: + case 12: + case 14: + case 15: + case 17: + case 18: + default: + String[] dataTime = dateStr2.split(" "); + String[] dataParts2 = dataTime[0].split("-"); + String[] timeParts = dataTime[1].split(":"); + String[] secondParts = timeParts[2].split("\\."); + secondParts[1] = StringUtils.rightPad(secondParts[1], 9, Constants.CJ_MINOR_VERSION); + return LocalDateTime.of(Integer.parseInt(dataParts2[0]), Integer.parseInt(dataParts2[1]), Integer.parseInt(dataParts2[2]), + Integer.parseInt(timeParts[0]), Integer.parseInt(timeParts[1]), Integer.parseInt(secondParts[0]), Integer.parseInt(secondParts[1])); + case 10: + String[] dataParts3 = dateStr2.split("-"); + return LocalDateTime.of(Integer.parseInt(dataParts3[0]), Integer.parseInt(dataParts3[1]), Integer.parseInt(dataParts3[2]), 0, 0, 0, 0); + case 13: + String[] dataTime2 = dateStr2.split(" "); + String[] dataParts4 = dataTime2[0].split("-"); + return LocalDateTime.of(Integer.parseInt(dataParts4[0]), Integer.parseInt(dataParts4[1]), Integer.parseInt(dataParts4[2]), Integer.parseInt(dataTime2[1]), 0, 0, 0); + case 16: + String[] dataTime3 = dateStr2.split(" "); + String[] dataParts5 = dataTime3[0].split("-"); + String[] timeParts2 = dataTime3[1].split(":"); + return LocalDateTime.of(Integer.parseInt(dataParts5[0]), Integer.parseInt(dataParts5[1]), Integer.parseInt(dataParts5[2]), Integer.parseInt(timeParts2[0]), Integer.parseInt(timeParts2[1]), 0, 0); + case 19: + String[] dataTime4 = dateStr2.split(" "); + String[] dataParts6 = dataTime4[0].split("-"); + String[] timeParts3 = dataTime4[1].split(":"); + return LocalDateTime.of(Integer.parseInt(dataParts6[0]), Integer.parseInt(dataParts6[1]), Integer.parseInt(dataParts6[2]), Integer.parseInt(timeParts3[0]), Integer.parseInt(timeParts3[1]), Integer.parseInt(timeParts3[2]), 0); + } + } else if (dateStr2.charAt(2) == ':') { + switch (len) { + case 5: + String[] timeParts4 = dateStr2.split(":"); + return LocalDateTime.of(0, 1, 1, Integer.parseInt(timeParts4[0]), Integer.parseInt(timeParts4[1]), 0, 0); + case 8: + String[] timeParts5 = dateStr2.split(":"); + return LocalDateTime.of(0, 1, 1, Integer.parseInt(timeParts5[0]), Integer.parseInt(timeParts5[1]), Integer.parseInt(timeParts5[2]), 0); + default: + String[] timeParts6 = dateStr2.split(":"); + String[] secondParts2 = timeParts6[2].split("\\."); + secondParts2[1] = StringUtils.rightPad(secondParts2[1], 9, Constants.CJ_MINOR_VERSION); + return LocalDateTime.of(0, 1, 1, Integer.parseInt(timeParts6[0]), Integer.parseInt(timeParts6[1]), Integer.parseInt(secondParts2[0]), Integer.parseInt(secondParts2[1])); + } + } else { + throw new UnsupportedOperationException(value.getClass() + ", value '" + value + "' , parse to local date time failed."); + } + } else if (StringUtils.isNumeric(value)) { + return LocalDateTime.of(Integer.parseInt(value), 1, 1, 0, 0, 0, 0); + } else { + throw new DateTimeException(value + " format error."); + } + } + + public static String bytes2hex(byte[] b) { + if (b == null) { + return null; + } + if (b.length == 0) { + return ""; + } + StringBuilder hs = new StringBuilder(); + for (byte element : b) { + String stmp = Integer.toHexString(element & 255).toUpperCase(); + if (stmp.length() == 1) { + hs.append(Constants.CJ_MINOR_VERSION); + hs.append(stmp); + } else { + hs.append(stmp); + } + } + return hs.toString(); + } + + public static String convertToString(Object value) { + if (value == null) { + return null; + } + if (value instanceof String) { + return (String) value; + } + if (value instanceof BigInteger) { + return value.toString(); + } + if (value instanceof BigDecimal) { + return ((BigDecimal) value).toPlainString(); + } + if (value instanceof Number) { + return new BigDecimal(value.toString()).toPlainString(); + } + if (value instanceof Boolean) { + return Boolean.TRUE.equals(value) ? "1" : "0"; + } + if (value instanceof byte[]) { + return "0x" + bytes2hex((byte[]) value); + } + if (value instanceof Timestamp) { + long nanos = ((Timestamp) value).getNanos(); + value = Instant.ofEpochMilli(((Timestamp) value).getTime() - (nanos / 1000000)).plusNanos(nanos).atZone(ZoneId.systemDefault()).toLocalDateTime(); + } else if (value instanceof Date) { + value = ((Date) value).toLocalDate().atTime(0, 0); + } else if (value instanceof Time) { + value = LocalDateTime.of(LocalDate.of(1970, 1, 1), Instant.ofEpochMilli(((Time) value).getTime()).atZone(ZoneId.systemDefault()).toLocalTime()); + } else if (value instanceof java.util.Date) { + value = ((java.util.Date) value).toInstant().atZone(ZoneId.systemDefault()).toLocalDateTime(); + } if (value instanceof LocalDateTime) { + return coverLocalDateTime2String((LocalDateTime) value); + } else if (value instanceof OffsetDateTime) { + OffsetDateTime zone = (OffsetDateTime) value; + String datetimeStr = coverLocalDateTime2String(zone.toLocalDateTime()); + String zonedStr = zone.getOffset().toString(); + if ("Z".equals(zonedStr)) { + return datetimeStr + "+00:00"; + } + return datetimeStr + zonedStr; + } else if (!(value instanceof LocalTime)) { + return value.toString(); + } else { + LocalTime local3 = (LocalTime) value; + return String.format("%02d:%02d:%02d", local3.getHour(), local3.getMinute(), local3.getSecond()); + } + } + + + private static String coverLocalDateTime2String(LocalDateTime localDateTime) { + LocalDate localDate = localDateTime.toLocalDate(); + LocalTime localTime = localDateTime.toLocalTime(); + int year = localDate.getYear(); + int month = localDate.getMonthValue(); + int day = localDate.getDayOfMonth(); + int hour = localTime.getHour(); + int minute = localTime.getMinute(); + int second = localTime.getSecond(); + int nano = localTime.getNano(); + return nano == 0 ? String.format("%04d-%02d-%02d %02d:%02d:%02d", year, month, day, hour, minute, second) : + String.format("%04d-%02d-%02d %02d:%02d:%02d.%s", year, month, day, hour, minute, second, new BigDecimal(nano).divide(NANO_SEC).toPlainString().substring(2)); + } + + public static String toMySqlTime(Object value) { + if (value == null || StringUtils.isBlank(value.toString())) { + return null; + } + if (value instanceof String) { + return value.toString(); + } + LocalDateTime localTime = toLocalDateTime(value); + if (BASE.isBefore(localTime) || BASE.isEqual(localTime)) { + long diffHours = Duration.between(BASE, localTime).toHours(); + if (localTime.getNano() == 0) { + return String.format("%02d:%02d:%02d", diffHours, localTime.getMinute(), localTime.getSecond()); + } + return String.format("%02d:%02d:%02d.%s", diffHours, localTime.getMinute(), localTime.getSecond(), + Integer.parseInt(trimEnd(String.valueOf(localTime.getNano()), '0'))); + } + Duration duration = Duration.between(localTime, BASE); + long totalSecond = duration.getSeconds(); + long hours = totalSecond / ONE_HOUR; + long remaining = totalSecond - (hours * ONE_HOUR); + long minutes = remaining / ONE_MINUTE; + remaining = remaining - (minutes * ONE_MINUTE); + if (duration.getNano() == 0) { + return String.format("-%02d:%02d:%02d", hours, minutes, remaining); + } + return String.format("-%02d:%02d:%02d.%s", hours, minutes, remaining, Integer.parseInt(trimEnd(String.valueOf(duration.getNano()), '0'))); + } + + public static String trimEnd(String str, char trimChar) { + if (str == null || str.isEmpty()) { + return str; + } + char[] val = str.toCharArray(); + int len = val.length; + while (0 < len && val[len - 1] == trimChar) { + len--; + } + return len < val.length ? str.substring(0, len) : str; + } + + public static byte[] numberToBinaryArray(Number number) { + BigInteger bigInt = BigInteger.valueOf(number.longValue()); + int size = (bigInt.bitLength() + 7) / 8; + byte[] result = new byte[size]; + byte[] bigIntBytes = bigInt.toByteArray(); + int start = bigInt.bitLength() % 8 == 0 ? 1 : 0; + int length = Math.min(bigIntBytes.length - start, size); + System.arraycopy(bigIntBytes, start, result, size - length, length); + return result; + } + + public static Integer toInt(Object value) { + if (value == null) { + return null; + } + if (value instanceof String) { + String strValue = ((String) value).toLowerCase(); + if (StringUtils.isBlank(strValue)) { + return null; + } + try { + return Integer.parseInt(strValue); + } catch (Exception e) { + try { + return Integer.decode(strValue); + } catch (Exception e2) { + if ("true".equals(strValue)) { + return 1; + } + if ("false".equals(strValue)) { + return 0; + } + return new BigDecimal(strValue).intValue(); + } + } + } else if (value instanceof Number) { + return ((Number) value).intValue(); + } else { + if (value instanceof Boolean) { + return Boolean.TRUE.equals(value) ? 1 : 0; + } + throw new UnsupportedOperationException("class " + value.getClass() + ", value '" + value + "' , parse to int failed."); + } + } + + public static LocalDateTime toLocalDateTime(Object value) { + if (value == null || StringUtils.isBlank(value.toString())) { + return null; + } + if (value instanceof Temporal) { + if (value instanceof LocalDateTime) { + return (LocalDateTime) value; + } + if (value instanceof OffsetDateTime) { + return ((OffsetDateTime) value).toLocalDateTime(); + } + if (value instanceof LocalTime) { + return LocalDateTime.of(LocalDate.of(1970, 1, 1), (LocalTime) value); + } else if (value instanceof LocalDate) { + return LocalDateTime.of((LocalDate) value, LocalTime.of(0, 0)); + } else { + throw new UnsupportedOperationException(value.getClass() + ", value '" + value + "' , parse local date time failed."); + } + } else if (!(value instanceof java.util.Date)) { + return toLocalDateTime(value.toString()); + } else { + if (value instanceof Timestamp) { + long nanos = ((Timestamp) value).getNanos(); + return Instant.ofEpochMilli(((Timestamp) value).getTime() - (nanos / 1000000)).plusNanos(nanos).atZone(ZoneId.systemDefault()).toLocalDateTime(); + } else if (value instanceof java.sql.Date) { + return ((java.sql.Date) value).toLocalDate().atTime(0, 0); + } else { + if (!(value instanceof Time)) { + return ((java.util.Date) value).toInstant().atZone(ZoneId.systemDefault()).toLocalDateTime(); + } + return LocalDateTime.of(LocalDate.of(1970, 1, 1), Instant.ofEpochMilli(((Time) value).getTime()).atZone(ZoneId.systemDefault()).toLocalTime()); + } + } } public static boolean isHexNumber(String str) { @@ -482,7 +808,7 @@ public static boolean isHexNumber(String str) { return flag; } - protected static byte[] safeToBytes(Object value) { + public static byte[] toBytes(Object value) { if (value == null) { return null; } @@ -495,7 +821,7 @@ protected static byte[] safeToBytes(Object value) { } else if (value instanceof byte[]) { return (byte[]) value; } else { - throw new UnsupportedOperationException("class " + value.getClass() + ", value '" + value + "' , safeToBytes failed."); + throw new UnsupportedOperationException("class " + value.getClass() + ", value '" + value + "' , parse to bytes failed."); } } @@ -508,13 +834,12 @@ public static String toGeometry(Object value) throws Exception { if (!strVal.startsWith("0x") && !strVal.startsWith("0X")) { return (String) value; } - return WKB_READER.read(hex2bytes(strVal.substring(2))).toText(); + return new WKBReader(GEOMETRY_FACTORY).read(hex2bytes(strVal.substring(2))).toText(); } else if (value instanceof byte[]) { - return WKB_READER.read((byte[]) value).toText(); + return new WKBReader(GEOMETRY_FACTORY).read((byte[]) value).toText(); } else { throw new UnsupportedOperationException("class " + value.getClass() + ", value '" + value + "' , " + - "safeToGisWKT" + - " failed."); + "parse to geometry failed."); } } @@ -535,16 +860,51 @@ public static byte[] hex2bytes(String hexStr) { int index = i * 2; char c1 = hexStr.charAt(index); char c2 = hexStr.charAt(index + 1); - if (c1 < '0' || c1 > 'F' || c2 < '0' || c2 > 'F') { - throw new EventMeshException(String.format("illegal byte [%s], [%s]", c1, c2)); - } ret[i] = (byte) ((byte) c1 << 4); ret[i] = (byte) (ret[i] | (byte) (c2)); } return ret; } - protected static String toGisWKT(Object value) throws Exception { + private static byte toByte(char src) { + switch (Character.toUpperCase(src)) { + case '0': + return 0; + case '1': + return 1; + case '2': + return 2; + case '3': + return 3; + case '4': + return 4; + case '5': + return 5; + case '6': + return 6; + case '7': + return 7; + case '8': + return 8; + case '9': + return 9; + case 'A': + return 10; + case 'B': + return 11; + case 'C': + return 12; + case 'D': + return 13; + case 'E': + return 14; + case 'F': + return 15; + } + throw new IllegalStateException("0-F"); + } + + public static String toGisWKT(Object value) throws Exception { if (value == null) { return null; } diff --git a/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/sink/connector/CanalSinkFullConnector.java b/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/sink/connector/CanalSinkFullConnector.java index c3d2f67002..da6245440b 100644 --- a/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/sink/connector/CanalSinkFullConnector.java +++ b/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/sink/connector/CanalSinkFullConnector.java @@ -10,6 +10,7 @@ import org.apache.eventmesh.common.exception.EventMeshException; import org.apache.eventmesh.common.remote.offset.canal.CanalFullRecordOffset; import org.apache.eventmesh.connector.canal.DatabaseConnection; +import org.apache.eventmesh.connector.canal.SqlUtils; import org.apache.eventmesh.connector.canal.source.table.RdbTableMgr; import org.apache.eventmesh.openconnect.api.ConnectorCreateService; import org.apache.eventmesh.openconnect.api.connector.ConnectorContext; @@ -20,20 +21,18 @@ import org.apache.commons.lang3.StringUtils; import java.math.BigDecimal; -import java.math.BigInteger; -import java.nio.charset.StandardCharsets; import java.sql.PreparedStatement; import java.sql.SQLException; import java.sql.Types; import java.time.LocalDateTime; +import java.time.format.DateTimeFormatter; import java.util.ArrayList; import java.util.List; import java.util.Map; import java.util.concurrent.locks.LockSupport; -import org.locationtech.jts.io.WKBReader; - import com.alibaba.druid.pool.DruidPooledConnection; +import com.mysql.cj.exceptions.MysqlErrorNumbers; import lombok.extern.slf4j.Slf4j; @@ -41,9 +40,9 @@ public class CanalSinkFullConnector implements Sink, ConnectorCreateService { private CanalSinkFullConfig config; private RdbTableMgr tableMgr; + private final DateTimeFormatter dataTimePattern = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss.SSSSSSSSS"); @Override public void start() throws Exception { - tableMgr.start(); } @@ -127,7 +126,6 @@ public void put(List sinkRecords) { String sql = generateInsertPrepareSql(offset.getPosition().getSchema(), offset.getPosition().getTableName(), cols); - try(DruidPooledConnection connection = DatabaseConnection.sinkDataSource.getConnection();PreparedStatement statement = connection.prepareStatement(sql)) { for (Map col : data) { @@ -136,22 +134,21 @@ public void put(List sinkRecords) { } statement.executeBatch(); } catch (SQLException e) { - log.warn("create sink connection "); + log.warn("sink connector write fail", e); LockSupport.parkNanos(3000 * 1000L); } catch (Exception e) { + log.error("", e); // todo rollback } } private void setPrepareParams(PreparedStatement preparedStatement, Map col, List columnDefs) throws Exception { - for (int i = 0; i <= columnDefs.size(); i++) { - writeColumn(preparedStatement, i + 1, columnDefs.get(i), col.get(columnDefs.get(i))); + for (int i = 0; i < columnDefs.size(); i++) { + writeColumn(preparedStatement, i + 1, columnDefs.get(i), col.get(columnDefs.get(i).getName())); } } public void writeColumn(PreparedStatement ps, int index, MySQLColumnDef colType, Object value) throws Exception { - LocalDateTime colValue; - LocalDateTime colValue2; if (colType == null) { String colVal = null; if (value != null) { @@ -170,7 +167,7 @@ public void writeColumn(PreparedStatement ps, int index, MySQLColumnDef colType, case SMALLINT: case MEDIUMINT: case INT: - Long longValue = toLong(value); + Long longValue = SqlUtils.toLong(value); if (longValue == null) { ps.setNull(index, 4); return; @@ -180,7 +177,7 @@ public void writeColumn(PreparedStatement ps, int index, MySQLColumnDef colType, } case BIGINT: case DECIMAL: - BigDecimal bigDecimalValue = toBigDecimal(value); + BigDecimal bigDecimalValue = SqlUtils.toBigDecimal(value); if (bigDecimalValue == null) { ps.setNull(index, 3); return; @@ -190,62 +187,61 @@ public void writeColumn(PreparedStatement ps, int index, MySQLColumnDef colType, } case FLOAT: case DOUBLE: - Double colValue6 = toDouble(value); - if (colValue6 == null) { + Double doubleValue = SqlUtils.toDouble(value); + if (doubleValue == null) { ps.setNull(index, 8); - return; } else { - ps.setDouble(index, colValue6); - return; + ps.setDouble(index, doubleValue); } + return; case DATE: case DATETIME: case TIMESTAMP: - if (!isZeroTime(value)) { + LocalDateTime dateValue = null; + if (!SqlUtils.isZeroTime(value)) { try { - colValue2 = safeToLocalDateTime(value); + dateValue = SqlUtils.toLocalDateTime(value); } catch (Exception e) { - ps.setString(index, CrwUtils.convertToString(value)); + ps.setString(index, SqlUtils.convertToString(value)); return; } - } else if (StringUtils.isNotBlank(options.getDefaultZeroDate())) { - colValue2 = safeToLocalDateTime(options.getDefaultZeroDate()); + } else if (StringUtils.isNotBlank(config.getZeroDate())) { + dateValue = SqlUtils.toLocalDateTime(config.getZeroDate()); } else { ps.setObject(index, value); return; } - if (colValue2 == null) { - ps.setNull(index, 93); - return; + if (dateValue == null) { + ps.setNull(index, Types.TIMESTAMP); } else { - ps.setString(index, WellKnowFormat.WKF_DATE_TIME24_S9.format(colValue2)); - return; + ps.setString(index, dataTimePattern.format(dateValue)); } + return; case TIME: - String colValue7 = safeToMySqlTime(value, options); - if (StringUtils.isBlank(colValue7)) { + String timeValue = SqlUtils.toMySqlTime(value); + if (StringUtils.isBlank(timeValue)) { ps.setNull(index, 12); return; } else { - ps.setString(index, colValue7); + ps.setString(index, timeValue); return; } case YEAR: - if (!isZeroTime(value)) { - colValue = safeToLocalDateTime(value); - } else if (StringUtils.isNotBlank(options.getDefaultZeroDate())) { - colValue = safeToLocalDateTime(options.getDefaultZeroDate()); + LocalDateTime yearValue = null; + if (!SqlUtils.isZeroTime(value)) { + yearValue = SqlUtils.toLocalDateTime(value); + } else if (StringUtils.isNotBlank(config.getZeroDate())) { + yearValue = SqlUtils.toLocalDateTime(config.getZeroDate()); } else { ps.setInt(index, 0); return; } - if (colValue == null) { + if (yearValue == null) { ps.setNull(index, 4); - return; } else { - ps.setInt(index, colValue.getYear()); - return; + ps.setInt(index, yearValue.getYear()); } + return; case CHAR: case VARCHAR: case TINYTEXT: @@ -254,23 +250,22 @@ public void writeColumn(PreparedStatement ps, int index, MySQLColumnDef colType, case LONGTEXT: case ENUM: case SET: - String colValue8 = safeToString(value); - if (colValue8 == null) { - ps.setNull(index, 12); + String strValue = value.toString(); + if (strValue == null) { + ps.setNull(index, Types.VARCHAR); return; } else { - ps.setString(index, colValue8); + ps.setString(index, strValue); return; } case JSON: - String colValue9 = safeToJson(value); - if (colValue9 == null) { - ps.setNull(index, 12); - return; + String jsonValue = value.toString(); + if (jsonValue == null) { + ps.setNull(index, Types.VARCHAR); } else { - ps.setString(index, colValue9); - return; + ps.setString(index, jsonValue); } + return; case BIT: if (value instanceof Boolean) { byte[] bArr = new byte[1]; @@ -278,19 +273,19 @@ public void writeColumn(PreparedStatement ps, int index, MySQLColumnDef colType, ps.setBytes(index, bArr); return; } else if (value instanceof Number) { - ps.setBytes(index, numberToBinaryArray((Number) value)); + ps.setBytes(index, SqlUtils.numberToBinaryArray((Number) value)); return; } else if ((value instanceof byte[]) || value.toString().startsWith("0x") || value.toString().startsWith("0X")) { - byte[] colValue10 = safeToBytes(value); - if (colValue10 == null || colValue10.length == 0) { - ps.setNull(index, -7); + byte[] bArr = SqlUtils.toBytes(value); + if (bArr == null || bArr.length == 0) { + ps.setNull(index, Types.BIT); return; } else { - ps.setBytes(index, colValue10); + ps.setBytes(index, bArr); return; } } else { - ps.setBytes(index, numberToBinaryArray(safeToInt(value))); + ps.setBytes(index, SqlUtils.numberToBinaryArray(SqlUtils.toInt(value))); return; } case BINARY: @@ -299,26 +294,26 @@ public void writeColumn(PreparedStatement ps, int index, MySQLColumnDef colType, case BLOB: case MEDIUMBLOB: case LONGBLOB: - byte[] colValue11 = safeToBytes(value); - if (colValue11 == null) { - ps.setNull(index, -2); + byte[] binaryValue = SqlUtils.toBytes(value); + if (binaryValue == null) { + ps.setNull(index, Types.BINARY); return; } else { - ps.setBytes(index, colValue11); + ps.setBytes(index, binaryValue); return; } case GEOMETRY: case GEOMETRY_COLLECTION: case GEOM_COLLECTION: - String colValue12 = safeToGisWKT(value); - if (colValue12 == null) { - ps.setNull(index, 12); + String geoValue = SqlUtils.toGisWKT(value); + if (geoValue == null) { + ps.setNull(index, Types.VARCHAR); return; } - if (colValue12.length() >= 5 && StringUtils.startsWithIgnoreCase(colValue12.substring(0, 5), "SRID=")) { - colValue12 = colValue12.substring(colValue12.indexOf(59) + 1); + if (geoValue.length() >= 5 && StringUtils.startsWithIgnoreCase(geoValue.substring(0, 5), "SRID=")) { + geoValue = geoValue.substring(geoValue.indexOf(59) + 1); } - ps.setString(index, colValue12); + ps.setString(index, geoValue); return; case POINT: case LINESTRING: @@ -339,7 +334,9 @@ private String generateInsertPrepareSql(String schema, String table, List> rows = new LinkedList<>(); while (flag.get()) { - String scanSql = generateScanSql(!isNextPage); - log.info("scan sql is [{}] , cur position [{}], choose primary key [{}]", scanSql, - JsonUtils.toJSONString(position.getCurPrimaryKeyCols()), choosePrimaryKey.get()); + String scanSql = generateScanSql(isFirstSelect); + log.info("scan sql is [{}] , cur position [{}]", scanSql, JsonUtils.toJSONString(position.getCurPrimaryKeyCols())); try (Connection connection = dataSource.getConnection(); PreparedStatement statement = connection.prepareStatement(scanSql, ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY)) { @@ -132,8 +128,8 @@ public void start(AtomicBoolean flag) { tableDefinition.getTableName(), e); LockSupport.parkNanos(3000 * 1000L); } - if (!isNextPage) { - isNextPage = true; + if (isFirstSelect) { + isFirstSelect = false; } } } @@ -230,7 +226,11 @@ public Object readColumn(ResultSet rs, String col, CanalMySQLType colType) throw case LONGBLOB: return rs.getBytes(col); case GEOMETRY: - return SqlUtils.toGeometry("0x" + rs.getString(col)); + String geo = rs.getString(col); + if (col == null) { + return null; + } + return SqlUtils.toGeometry("0x" + geo); case GEOMETRY_COLLECTION: case GEOM_COLLECTION: case POINT: @@ -354,22 +354,24 @@ private void generateQueryColumnsSql(StringBuilder builder, Collection Date: Tue, 23 Jul 2024 10:38:08 +0800 Subject: [PATCH 11/15] close to finish full read and begin full write --- .../connector/rdb/canal/CanalMySQLType.java | 27 +++- .../eventmesh/connector/canal/SqlUtils.java | 153 +++++++++--------- .../connector/CanalSinkFullConnector.java | 73 ++++++--- .../source/connector/CanalFullProducer.java | 55 +++---- 4 files changed, 177 insertions(+), 131 deletions(-) diff --git a/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/rdb/canal/CanalMySQLType.java b/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/rdb/canal/CanalMySQLType.java index 257822810e..e14ea78c82 100644 --- a/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/rdb/canal/CanalMySQLType.java +++ b/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/rdb/canal/CanalMySQLType.java @@ -1,10 +1,10 @@ package org.apache.eventmesh.common.config.connector.rdb.canal; -import com.mysql.cj.MysqlType; - import java.util.HashMap; import java.util.Map; +import com.mysql.cj.MysqlType; + public enum CanalMySQLType { BIT("BIT"), TINYINT("TINYINT"), @@ -61,6 +61,29 @@ public enum CanalMySQLType { } } + public String genPrepareStatement4Insert() { + switch (this) { + case GEOMETRY: + case GEOM_COLLECTION: + case GEOMETRY_COLLECTION: + return "ST_GEOMFROMTEXT(?)"; + case POINT: + return "ST_PointFromText(?)"; + case LINESTRING: + return "ST_LineStringFromText(?)"; + case POLYGON: + return "ST_PolygonFromText(?)"; + case MULTIPOINT: + return "ST_MultiPointFromText(?)"; + case MULTILINESTRING: + return "ST_MultiLineStringFromText(?)"; + case MULTIPOLYGON: + return "ST_MultiPolygonFromText(?)"; + default: + return "?"; + } + } + public static CanalMySQLType valueOfCode(String code) { CanalMySQLType type = TYPES.get(code.toUpperCase()); if (type != null) { diff --git a/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/SqlUtils.java b/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/SqlUtils.java index 27afd22735..05b6b1836f 100644 --- a/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/SqlUtils.java +++ b/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/SqlUtils.java @@ -52,6 +52,7 @@ import org.locationtech.jts.geom.GeometryFactory; import org.locationtech.jts.io.WKBReader; +import org.locationtech.jts.io.WKTReader; import com.mysql.cj.Constants; import com.mysql.cj.MysqlType; @@ -62,8 +63,8 @@ public class SqlUtils { public static final String REQUIRED_FIELD_NULL_SUBSTITUTE = " "; private static final Map> sqlTypeToJavaTypeMap = new HashMap>(); private static final ConvertUtilsBean convertUtilsBean = new ConvertUtilsBean(); - private static final WKBReader WKB_READER = new WKBReader(new GeometryFactory()); private static final GeometryFactory GEOMETRY_FACTORY = new GeometryFactory(); + private static final WKBReader WKB_READER = new WKBReader(GEOMETRY_FACTORY); private static final BigDecimal NANO_SEC = new BigDecimal(LogBuffer.DIG_BASE); private static final LocalDateTime BASE = LocalDateTime.of(1970, 1, 1, 0, 0, 0, 0); private static final long ONE_HOUR = 3600; @@ -379,7 +380,7 @@ public static BigDecimal toBigDecimal(Object value) { return BigDecimal.valueOf(((Float) value).doubleValue()); } if (value instanceof BigInteger) { - return new BigDecimal((BigInteger)value); + return new BigDecimal((BigInteger) value); } if (value instanceof Byte) { return BigDecimal.valueOf(((Byte) value).longValue()); @@ -540,24 +541,29 @@ private static LocalDateTime toLocalDateTime(String value) { String[] secondParts = timeParts[2].split("\\."); secondParts[1] = StringUtils.rightPad(secondParts[1], 9, Constants.CJ_MINOR_VERSION); return LocalDateTime.of(Integer.parseInt(dataParts2[0]), Integer.parseInt(dataParts2[1]), Integer.parseInt(dataParts2[2]), - Integer.parseInt(timeParts[0]), Integer.parseInt(timeParts[1]), Integer.parseInt(secondParts[0]), Integer.parseInt(secondParts[1])); + Integer.parseInt(timeParts[0]), Integer.parseInt(timeParts[1]), Integer.parseInt(secondParts[0]), + Integer.parseInt(secondParts[1])); case 10: String[] dataParts3 = dateStr2.split("-"); - return LocalDateTime.of(Integer.parseInt(dataParts3[0]), Integer.parseInt(dataParts3[1]), Integer.parseInt(dataParts3[2]), 0, 0, 0, 0); + return LocalDateTime.of(Integer.parseInt(dataParts3[0]), Integer.parseInt(dataParts3[1]), Integer.parseInt(dataParts3[2]), 0, + 0, 0, 0); case 13: String[] dataTime2 = dateStr2.split(" "); String[] dataParts4 = dataTime2[0].split("-"); - return LocalDateTime.of(Integer.parseInt(dataParts4[0]), Integer.parseInt(dataParts4[1]), Integer.parseInt(dataParts4[2]), Integer.parseInt(dataTime2[1]), 0, 0, 0); + return LocalDateTime.of(Integer.parseInt(dataParts4[0]), Integer.parseInt(dataParts4[1]), Integer.parseInt(dataParts4[2]), + Integer.parseInt(dataTime2[1]), 0, 0, 0); case 16: String[] dataTime3 = dateStr2.split(" "); String[] dataParts5 = dataTime3[0].split("-"); String[] timeParts2 = dataTime3[1].split(":"); - return LocalDateTime.of(Integer.parseInt(dataParts5[0]), Integer.parseInt(dataParts5[1]), Integer.parseInt(dataParts5[2]), Integer.parseInt(timeParts2[0]), Integer.parseInt(timeParts2[1]), 0, 0); + return LocalDateTime.of(Integer.parseInt(dataParts5[0]), Integer.parseInt(dataParts5[1]), Integer.parseInt(dataParts5[2]), + Integer.parseInt(timeParts2[0]), Integer.parseInt(timeParts2[1]), 0, 0); case 19: String[] dataTime4 = dateStr2.split(" "); String[] dataParts6 = dataTime4[0].split("-"); String[] timeParts3 = dataTime4[1].split(":"); - return LocalDateTime.of(Integer.parseInt(dataParts6[0]), Integer.parseInt(dataParts6[1]), Integer.parseInt(dataParts6[2]), Integer.parseInt(timeParts3[0]), Integer.parseInt(timeParts3[1]), Integer.parseInt(timeParts3[2]), 0); + return LocalDateTime.of(Integer.parseInt(dataParts6[0]), Integer.parseInt(dataParts6[1]), Integer.parseInt(dataParts6[2]), + Integer.parseInt(timeParts3[0]), Integer.parseInt(timeParts3[1]), Integer.parseInt(timeParts3[2]), 0); } } else if (dateStr2.charAt(2) == ':') { switch (len) { @@ -566,12 +572,14 @@ private static LocalDateTime toLocalDateTime(String value) { return LocalDateTime.of(0, 1, 1, Integer.parseInt(timeParts4[0]), Integer.parseInt(timeParts4[1]), 0, 0); case 8: String[] timeParts5 = dateStr2.split(":"); - return LocalDateTime.of(0, 1, 1, Integer.parseInt(timeParts5[0]), Integer.parseInt(timeParts5[1]), Integer.parseInt(timeParts5[2]), 0); + return LocalDateTime.of(0, 1, 1, Integer.parseInt(timeParts5[0]), Integer.parseInt(timeParts5[1]), + Integer.parseInt(timeParts5[2]), 0); default: String[] timeParts6 = dateStr2.split(":"); String[] secondParts2 = timeParts6[2].split("\\."); secondParts2[1] = StringUtils.rightPad(secondParts2[1], 9, Constants.CJ_MINOR_VERSION); - return LocalDateTime.of(0, 1, 1, Integer.parseInt(timeParts6[0]), Integer.parseInt(timeParts6[1]), Integer.parseInt(secondParts2[0]), Integer.parseInt(secondParts2[1])); + return LocalDateTime.of(0, 1, 1, Integer.parseInt(timeParts6[0]), Integer.parseInt(timeParts6[1]), + Integer.parseInt(secondParts2[0]), Integer.parseInt(secondParts2[1])); } } else { throw new UnsupportedOperationException(value.getClass() + ", value '" + value + "' , parse to local date time failed."); @@ -627,14 +635,17 @@ public static String convertToString(Object value) { } if (value instanceof Timestamp) { long nanos = ((Timestamp) value).getNanos(); - value = Instant.ofEpochMilli(((Timestamp) value).getTime() - (nanos / 1000000)).plusNanos(nanos).atZone(ZoneId.systemDefault()).toLocalDateTime(); + value = Instant.ofEpochMilli(((Timestamp) value).getTime() - (nanos / 1000000)).plusNanos(nanos).atZone(ZoneId.systemDefault()) + .toLocalDateTime(); } else if (value instanceof Date) { value = ((Date) value).toLocalDate().atTime(0, 0); } else if (value instanceof Time) { - value = LocalDateTime.of(LocalDate.of(1970, 1, 1), Instant.ofEpochMilli(((Time) value).getTime()).atZone(ZoneId.systemDefault()).toLocalTime()); + value = LocalDateTime.of(LocalDate.of(1970, 1, 1), + Instant.ofEpochMilli(((Time) value).getTime()).atZone(ZoneId.systemDefault()).toLocalTime()); } else if (value instanceof java.util.Date) { value = ((java.util.Date) value).toInstant().atZone(ZoneId.systemDefault()).toLocalDateTime(); - } if (value instanceof LocalDateTime) { + } + if (value instanceof LocalDateTime) { return coverLocalDateTime2String((LocalDateTime) value); } else if (value instanceof OffsetDateTime) { OffsetDateTime zone = (OffsetDateTime) value; @@ -664,7 +675,8 @@ private static String coverLocalDateTime2String(LocalDateTime localDateTime) { int second = localTime.getSecond(); int nano = localTime.getNano(); return nano == 0 ? String.format("%04d-%02d-%02d %02d:%02d:%02d", year, month, day, hour, minute, second) : - String.format("%04d-%02d-%02d %02d:%02d:%02d.%s", year, month, day, hour, minute, second, new BigDecimal(nano).divide(NANO_SEC).toPlainString().substring(2)); + String.format("%04d-%02d-%02d %02d:%02d:%02d.%s", year, month, day, hour, minute, second, + new BigDecimal(nano).divide(NANO_SEC).toPlainString().substring(2)); } public static String toMySqlTime(Object value) { @@ -775,14 +787,16 @@ public static LocalDateTime toLocalDateTime(Object value) { } else { if (value instanceof Timestamp) { long nanos = ((Timestamp) value).getNanos(); - return Instant.ofEpochMilli(((Timestamp) value).getTime() - (nanos / 1000000)).plusNanos(nanos).atZone(ZoneId.systemDefault()).toLocalDateTime(); + return Instant.ofEpochMilli(((Timestamp) value).getTime() - (nanos / 1000000)).plusNanos(nanos).atZone(ZoneId.systemDefault()) + .toLocalDateTime(); } else if (value instanceof java.sql.Date) { return ((java.sql.Date) value).toLocalDate().atTime(0, 0); } else { if (!(value instanceof Time)) { return ((java.util.Date) value).toInstant().atZone(ZoneId.systemDefault()).toLocalDateTime(); } - return LocalDateTime.of(LocalDate.of(1970, 1, 1), Instant.ofEpochMilli(((Time) value).getTime()).atZone(ZoneId.systemDefault()).toLocalTime()); + return LocalDateTime.of(LocalDate.of(1970, 1, 1), + Instant.ofEpochMilli(((Time) value).getTime()).atZone(ZoneId.systemDefault()).toLocalTime()); } } } @@ -796,7 +810,9 @@ public static boolean isHexNumber(String str) { while (true) { if (i < str.length()) { char cc = str.charAt(i); - if (cc != '0' && cc != '1' && cc != '2' && cc != '3' && cc != '4' && cc != '5' && cc != '6' && cc != '7' && cc != '8' && cc != '9' && cc != 'A' && cc != 'B' && cc != 'C' && cc != 'D' && cc != 'E' && cc != 'F' && cc != 'a' && cc != 'b' && cc != 'c' && cc != 'd' && cc != 'e' && cc != 'f') { + if (cc != '0' && cc != '1' && cc != '2' && cc != '3' && cc != '4' && cc != '5' && cc != '6' && cc != '7' && cc != '8' && cc != '9' && + cc != 'A' && cc != 'B' && cc != 'C' && cc != 'D' && cc != 'E' && cc != 'F' && cc != 'a' && cc != 'b' && cc != 'c' && cc != 'd' && + cc != 'e' && cc != 'f') { flag = false; break; } @@ -834,9 +850,16 @@ public static String toGeometry(Object value) throws Exception { if (!strVal.startsWith("0x") && !strVal.startsWith("0X")) { return (String) value; } - return new WKBReader(GEOMETRY_FACTORY).read(hex2bytes(strVal.substring(2))).toText(); + return new WKTReader().read((String) value).toText(); } else if (value instanceof byte[]) { - return new WKBReader(GEOMETRY_FACTORY).read((byte[]) value).toText(); + // mysql add 4 byte in header of geometry + byte[] bytes = (byte[]) value; + if (bytes.length > 4) { + byte[] dst = new byte[bytes.length - 4]; + System.arraycopy(bytes, 4, dst, 0, bytes.length - 4); + return new WKBReader().read(dst).toText(); + } + return new WKBReader().read(bytes).toText(); } else { throw new UnsupportedOperationException("class " + value.getClass() + ", value '" + value + "' , " + "parse to geometry failed."); @@ -844,8 +867,9 @@ public static String toGeometry(Object value) throws Exception { } public static byte[] hex2bytes(String hexStr) { - if (hexStr == null) + if (hexStr == null) { return null; + } if (org.apache.commons.lang3.StringUtils.isBlank(hexStr)) { return new byte[0]; } @@ -860,64 +884,47 @@ public static byte[] hex2bytes(String hexStr) { int index = i * 2; char c1 = hexStr.charAt(index); char c2 = hexStr.charAt(index + 1); - ret[i] = (byte) ((byte) c1 << 4); - ret[i] = (byte) (ret[i] | (byte) (c2)); + ret[i] = (byte) (toByte(c1) << 4); + ret[i] = (byte) (ret[i] | toByte(c2)); } return ret; } private static byte toByte(char src) { - switch (Character.toUpperCase(src)) { - case '0': - return 0; - case '1': - return 1; - case '2': - return 2; - case '3': - return 3; - case '4': - return 4; - case '5': - return 5; - case '6': - return 6; - case '7': - return 7; - case '8': - return 8; - case '9': - return 9; - case 'A': - return 10; - case 'B': - return 11; - case 'C': - return 12; - case 'D': - return 13; - case 'E': - return 14; - case 'F': - return 15; - } - throw new IllegalStateException("0-F"); - } - - public static String toGisWKT(Object value) throws Exception { - if (value == null) { - return null; - } - if (value instanceof String) { - String strVal = (String) value; - if (!strVal.startsWith("0x") && !strVal.startsWith("0X")) { - return (String) value; - } - return WKB_READER.read(hex2bytes(strVal.substring(2))).toText(); - } else if (value instanceof byte[]) { - return WKB_READER.read((byte[]) value).toText(); - } else { - throw new UnsupportedOperationException("class " + value.getClass() + ", value '" + value + "' , parse to gis failed."); - } + switch (Character.toUpperCase(src)) { + case '0': + return 0; + case '1': + return 1; + case '2': + return 2; + case '3': + return 3; + case '4': + return 4; + case '5': + return 5; + case '6': + return 6; + case '7': + return 7; + case '8': + return 8; + case '9': + return 9; + case 'A': + return 10; + case 'B': + return 11; + case 'C': + return 12; + case 'D': + return 13; + case 'E': + return 14; + case 'F': + return 15; + } + throw new IllegalStateException("0-F"); } } diff --git a/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/sink/connector/CanalSinkFullConnector.java b/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/sink/connector/CanalSinkFullConnector.java index da6245440b..0508b291b9 100644 --- a/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/sink/connector/CanalSinkFullConnector.java +++ b/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/sink/connector/CanalSinkFullConnector.java @@ -1,7 +1,5 @@ package org.apache.eventmesh.connector.canal.sink.connector; -import com.alibaba.druid.pool.DruidPooledConnection; -import lombok.extern.slf4j.Slf4j; import org.apache.eventmesh.common.config.connector.Config; import org.apache.eventmesh.common.config.connector.rdb.canal.CanalSinkFullConfig; import org.apache.eventmesh.common.config.connector.rdb.canal.mysql.Constants; @@ -32,7 +30,6 @@ import java.util.concurrent.locks.LockSupport; import com.alibaba.druid.pool.DruidPooledConnection; -import com.mysql.cj.exceptions.MysqlErrorNumbers; import lombok.extern.slf4j.Slf4j; @@ -41,6 +38,7 @@ public class CanalSinkFullConnector implements Sink, ConnectorCreateService sinkRecords) { return; } ConnectRecord record = sinkRecords.get(0); - List> data = (List>)record.getData(); + List> data = (List>) record.getData(); if (data == null || data.isEmpty()) { if (log.isDebugEnabled()) { log.debug("[{}] got rows data is none", this.getClass()); @@ -117,7 +116,7 @@ public void put(List sinkRecords) { return; } - MySQLTableDef tableDefinition = (MySQLTableDef)tableMgr.getTable(offset.getPosition().getSchema(), offset.getPosition().getTableName()); + MySQLTableDef tableDefinition = (MySQLTableDef) tableMgr.getTable(offset.getPosition().getSchema(), offset.getPosition().getTableName()); if (tableDefinition == null) { log.warn("target schema [{}] table [{}] is not exists", offset.getPosition().getSchema(), offset.getPosition().getTableName()); return; @@ -125,24 +124,53 @@ public void put(List sinkRecords) { List cols = new ArrayList<>(tableDefinition.getColumnDefinitions().values()); String sql = generateInsertPrepareSql(offset.getPosition().getSchema(), offset.getPosition().getTableName(), cols); - - try(DruidPooledConnection connection = DatabaseConnection.sinkDataSource.getConnection();PreparedStatement statement = - connection.prepareStatement(sql)) { + DruidPooledConnection connection = null; + PreparedStatement statement = null; + try { + connection = DatabaseConnection.sinkDataSource.getConnection(); + statement = + connection.prepareStatement(sql); for (Map col : data) { setPrepareParams(statement, col, cols); + log.info("insert sql {}", statement.toString()); statement.addBatch(); } statement.executeBatch(); } catch (SQLException e) { - log.warn("sink connector write fail", e); + log.warn("full sink process schema [{}] table [{}] connector write fail", tableDefinition.getSchemaName(),tableDefinition.getTableName(), + e); LockSupport.parkNanos(3000 * 1000L); } catch (Exception e) { - log.error("", e); - // todo rollback + log.error("full sink process schema [{}] table [{}] catch unknown exception", tableDefinition.getSchemaName(), + tableDefinition.getTableName(), e); + try { + if (connection != null && !connection.isClosed()) { + connection.rollback(); + } + } catch (SQLException rollback) { + log.warn("full sink process schema [{}] table [{}] rollback fail", tableDefinition.getSchemaName(), + tableDefinition.getTableName(), e); + } + } finally { + if (statement != null) { + try { + statement.close(); + } catch (SQLException e) { + log.info("close prepare statement fail", e); + } + } + + if (connection != null) { + try { + connection.close(); + } catch (SQLException e) { + log.info("close db connection fail", e); + } + } } } - private void setPrepareParams(PreparedStatement preparedStatement, Map col, List columnDefs) throws Exception { + private void setPrepareParams(PreparedStatement preparedStatement, Map col, List columnDefs) throws Exception { for (int i = 0; i < columnDefs.size(); i++) { writeColumn(preparedStatement, i + 1, columnDefs.get(i), col.get(columnDefs.get(i).getName())); } @@ -305,23 +333,18 @@ public void writeColumn(PreparedStatement ps, int index, MySQLColumnDef colType, case GEOMETRY: case GEOMETRY_COLLECTION: case GEOM_COLLECTION: - String geoValue = SqlUtils.toGisWKT(value); - if (geoValue == null) { - ps.setNull(index, Types.VARCHAR); - return; - } - if (geoValue.length() >= 5 && StringUtils.startsWithIgnoreCase(geoValue.substring(0, 5), "SRID=")) { - geoValue = geoValue.substring(geoValue.indexOf(59) + 1); - } - ps.setString(index, geoValue); - return; case POINT: case LINESTRING: case POLYGON: case MULTIPOINT: case MULTILINESTRING: case MULTIPOLYGON: - ps.setNull(index, MysqlErrorNumbers.ER_INVALID_GROUP_FUNC_USE); + String geoValue = SqlUtils.toGeometry(value); + if (geoValue == null) { + ps.setNull(index, Types.VARCHAR); + return; + } + ps.setString(index, geoValue); return; default: throw new UnsupportedOperationException("columnType '" + colType + "' Unsupported."); @@ -348,7 +371,7 @@ private String generateInsertPrepareSql(String schema, String table, List columnValues = new LinkedHashMap<>(); for (Map.Entry col : - tableDefinition.getColumnDefinitions().entrySet()) { + tableDefinition.getColumnDefinitions().entrySet()) { columnValues.put(col.getKey(), readColumn(resultSet, col.getKey(), - col.getValue().getType())); + col.getValue().getType())); } lastCol = columnValues; rows.add(lastCol); @@ -109,24 +109,25 @@ public void start(AtomicBoolean flag) { if (lastCol == null || checkIsScanFinish(lastCol)) { log.info("full scan db [{}] table [{}] finish", tableDefinition.getSchemaName(), - tableDefinition.getTableName()); + tableDefinition.getTableName()); commitConnectRecord(rows); return; } refreshPosition(lastCol); } catch (InterruptedException ignore) { log.info("full scan db [{}] table [{}] interrupted", tableDefinition.getSchemaName(), - tableDefinition.getTableName()); + tableDefinition.getTableName()); Thread.currentThread().interrupt(); return; } } catch (SQLException e) { - log.error("catch SQLException fail", e); + log.error("full source process schema [{}] table [{}] catch SQLException fail",tableDefinition.getSchemaName(), + tableDefinition.getTableName(), e); LockSupport.parkNanos(3000 * 1000L); } catch (Exception e) { - log.error("process schema [{}] table [{}] catch unknown exception", tableDefinition.getSchemaName(), - tableDefinition.getTableName(), e); - LockSupport.parkNanos(3000 * 1000L); + log.error("full source process schema [{}] table [{}] catch unknown exception", tableDefinition.getSchemaName(), + tableDefinition.getTableName(), e); + return; } if (isFirstSelect) { isFirstSelect = false; @@ -156,7 +157,7 @@ private boolean checkIsScanFinish(Map lastCol) { if (lastPrimaryValue instanceof Number) { BigDecimal last = new BigDecimal(String.valueOf(lastPrimaryValue)); BigDecimal max = - new BigDecimal(String.valueOf(maxPrimaryValue)); + new BigDecimal(String.valueOf(maxPrimaryValue)); return last.compareTo(max) > 0; } if (lastPrimaryValue instanceof Comparable) { @@ -166,17 +167,15 @@ private boolean checkIsScanFinish(Map lastCol) { } public Object readColumn(ResultSet rs, String col, CanalMySQLType colType) throws Exception { + if (col == null || rs.wasNull()) { + return null; + } switch (colType) { case TINYINT: case SMALLINT: case MEDIUMINT: case INT: - Long uLong; - if (rs.wasNull()) { - return null; - } else { - uLong = rs.getLong(col); - } + Long uLong = rs.getLong(col); if (uLong.compareTo((long) Integer.MAX_VALUE) > 0) { return uLong; } @@ -203,9 +202,6 @@ public Object readColumn(ResultSet rs, String col, CanalMySQLType colType) throw case TIMESTAMP: return rs.getObject(col, LocalDateTime.class); case YEAR: - if (rs.wasNull()) { - return null; - } return rs.getInt(col); case CHAR: case VARCHAR: @@ -226,11 +222,6 @@ public Object readColumn(ResultSet rs, String col, CanalMySQLType colType) throw case LONGBLOB: return rs.getBytes(col); case GEOMETRY: - String geo = rs.getString(col); - if (col == null) { - return null; - } - return SqlUtils.toGeometry("0x" + geo); case GEOMETRY_COLLECTION: case GEOM_COLLECTION: case POINT: @@ -239,14 +230,17 @@ public Object readColumn(ResultSet rs, String col, CanalMySQLType colType) throw case MULTIPOINT: case MULTILINESTRING: case MULTIPOLYGON: - return null; + byte[] geo = rs.getBytes(col); + if (geo == null) { + return null; + } + return SqlUtils.toGeometry(geo); default: return rs.getObject(col); } } - private void refreshPosition(Map lastCol) { Map nextPosition = new LinkedHashMap<>(); for (Map.Entry entry : position.getCurPrimaryKeyCols().entrySet()) { @@ -335,7 +329,6 @@ private void setPrepareStatementValue(PreparedStatement statement) throws SQLExc } - private void generateQueryColumnsSql(StringBuilder builder, Collection rdbColDefs) { if (rdbColDefs == null || rdbColDefs.isEmpty()) { builder.append("*"); @@ -373,15 +366,15 @@ private String generateScanSql(boolean isFirst) { private void buildWhereSql(StringBuilder builder, boolean isEquals) { builder.append(" where ") - .append(Constants.MySQLQuot) - .append(choosePrimaryKey.get()) - .append(Constants.MySQLQuot); + .append(Constants.MySQLQuot) + .append(choosePrimaryKey.get()) + .append(Constants.MySQLQuot); if (isEquals) { builder.append(" >= ? "); } else { builder.append(" > ? "); } builder.append(" order by ").append(Constants.MySQLQuot).append(choosePrimaryKey.get()).append(Constants.MySQLQuot) - .append(" asc "); + .append(" asc "); } } From 7d546302322c62631252a529e97594fd3ad9166b Mon Sep 17 00:00:00 2001 From: sodaRyCN <757083350@qq.com> Date: Tue, 23 Jul 2024 11:11:45 +0800 Subject: [PATCH 12/15] fix commit execute --- eventmesh-connectors/eventmesh-connector-canal/build.gradle | 2 +- .../connector/canal/sink/connector/CanalSinkFullConnector.java | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/eventmesh-connectors/eventmesh-connector-canal/build.gradle b/eventmesh-connectors/eventmesh-connector-canal/build.gradle index 94d85ee22b..134af8ed3e 100644 --- a/eventmesh-connectors/eventmesh-connector-canal/build.gradle +++ b/eventmesh-connectors/eventmesh-connector-canal/build.gradle @@ -23,7 +23,7 @@ List canal = [ dependencies { api project(":eventmesh-openconnect:eventmesh-openconnect-java") - implementation "org.locationtech.jts" + implementation "org.locationtech.jts:jts-core" implementation project(":eventmesh-common") implementation canal implementation "com.alibaba:druid" diff --git a/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/sink/connector/CanalSinkFullConnector.java b/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/sink/connector/CanalSinkFullConnector.java index 0508b291b9..027bc7e15b 100644 --- a/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/sink/connector/CanalSinkFullConnector.java +++ b/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/sink/connector/CanalSinkFullConnector.java @@ -136,6 +136,7 @@ public void put(List sinkRecords) { statement.addBatch(); } statement.executeBatch(); + connection.commit(); } catch (SQLException e) { log.warn("full sink process schema [{}] table [{}] connector write fail", tableDefinition.getSchemaName(),tableDefinition.getTableName(), e); From 0d7a7184ffe7ee9cc225b11a4d9ee53ee50b9506 Mon Sep 17 00:00:00 2001 From: sodaRyCN <757083350@qq.com> Date: Tue, 23 Jul 2024 17:05:47 +0800 Subject: [PATCH 13/15] fix checkstyle and license --- .../eventmesh/admin/server/AdminServer.java | 9 +++-- .../admin/server/web/GrpcServer.java | 11 +++--- .../eventmesh/common/AbstractComponent.java | 34 ++++++++++++++----- .../config/connector/rdb/JdbcConfig.java | 17 ++++++++++ .../connector/rdb/canal/CanalMySQLType.java | 23 +++++++++++-- .../rdb/canal/CanalSinkFullConfig.java | 17 ++++++++++ .../rdb/canal/CanalSourceFullConfig.java | 22 ++++++++++-- .../rdb/canal/JobRdbFullPosition.java | 21 ++++++++++-- .../rdb/canal/RdbColumnDefinition.java | 21 ++++++++++-- .../connector/rdb/canal/RdbDBDefinition.java | 17 ++++++++++ .../rdb/canal/RdbTableDefinition.java | 17 ++++++++++ .../connector/rdb/canal/mysql/Constants.java | 17 ++++++++++ .../rdb/canal/mysql/MySQLColumnDef.java | 22 ++++++++++-- .../rdb/canal/mysql/MySQLTableDef.java | 17 ++++++++++ .../offset/canal/CanalFullRecordOffset.java | 23 +++++++++++-- .../connector/CanalSinkFullConnector.java | 17 ++++++++++ .../source/connector/CanalFullProducer.java | 17 ++++++++++ .../connector/CanalSourceFullConnector.java | 19 ++++++++++- .../source/position/CanalFullPositionMgr.java | 26 +++++++++----- .../canal/source/table/RdbSimpleTable.java | 21 +++++++----- .../canal/source/table/RdbTableMgr.java | 19 ++++++++++- 21 files changed, 361 insertions(+), 46 deletions(-) diff --git a/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/AdminServer.java b/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/AdminServer.java index ae2736aaaa..a2e4cc7063 100644 --- a/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/AdminServer.java +++ b/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/AdminServer.java @@ -17,8 +17,6 @@ package org.apache.eventmesh.admin.server; -import lombok.extern.slf4j.Slf4j; -import org.apache.commons.lang3.StringUtils; import org.apache.eventmesh.common.Constants; import org.apache.eventmesh.common.config.CommonConfiguration; import org.apache.eventmesh.common.config.ConfigService; @@ -30,11 +28,16 @@ import org.apache.eventmesh.registry.RegisterServerInfo; import org.apache.eventmesh.registry.RegistryFactory; import org.apache.eventmesh.registry.RegistryService; + +import org.apache.commons.lang3.StringUtils; + +import javax.annotation.PostConstruct; + import org.springframework.boot.context.event.ApplicationReadyEvent; import org.springframework.context.ApplicationListener; import org.springframework.stereotype.Service; -import javax.annotation.PostConstruct; +import lombok.extern.slf4j.Slf4j; @Service @Slf4j diff --git a/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/GrpcServer.java b/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/GrpcServer.java index 3a4bbe3d71..572e07a21d 100644 --- a/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/GrpcServer.java +++ b/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/GrpcServer.java @@ -17,14 +17,17 @@ package org.apache.eventmesh.admin.server.web; -import io.grpc.Server; -import io.grpc.netty.shaded.io.grpc.netty.NettyServerBuilder; -import lombok.extern.slf4j.Slf4j; import org.apache.eventmesh.admin.server.AdminServerProperties; + +import java.util.concurrent.TimeUnit; + import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Controller; -import java.util.concurrent.TimeUnit; +import io.grpc.Server; +import io.grpc.netty.shaded.io.grpc.netty.NettyServerBuilder; + +import lombok.extern.slf4j.Slf4j; @Controller @Slf4j diff --git a/eventmesh-common/src/main/java/org/apache/eventmesh/common/AbstractComponent.java b/eventmesh-common/src/main/java/org/apache/eventmesh/common/AbstractComponent.java index bc4e9ad404..375b6cb1d3 100644 --- a/eventmesh-common/src/main/java/org/apache/eventmesh/common/AbstractComponent.java +++ b/eventmesh-common/src/main/java/org/apache/eventmesh/common/AbstractComponent.java @@ -1,30 +1,45 @@ -package org.apache.eventmesh.common; +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ -import lombok.extern.slf4j.Slf4j; +package org.apache.eventmesh.common; import java.util.concurrent.atomic.AtomicBoolean; -/** - * @Description: - */ +import lombok.extern.slf4j.Slf4j; + @Slf4j public abstract class AbstractComponent implements ComponentLifeCycle { private final AtomicBoolean started = new AtomicBoolean(false); private final AtomicBoolean stopped = new AtomicBoolean(false); + @Override public void start() throws Exception { - if (!started.compareAndSet(false, true)){ + if (!started.compareAndSet(false, true)) { log.info("component [{}] has started", this.getClass()); return; } log.info("component [{}] will start", this.getClass()); - startup(); + run(); log.info("component [{}] started successfully", this.getClass()); } @Override public void stop() throws Exception { - if (!stopped.compareAndSet(false, true)){ + if (!stopped.compareAndSet(false, true)) { log.info("component [{}] has stopped", this.getClass()); return; } @@ -33,6 +48,7 @@ public void stop() throws Exception { log.info("component [{}] stopped successfully", this.getClass()); } - protected abstract void startup() throws Exception; + protected abstract void run() throws Exception; + protected abstract void shutdown() throws Exception; } diff --git a/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/rdb/JdbcConfig.java b/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/rdb/JdbcConfig.java index 70c18a221a..fc784fc187 100644 --- a/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/rdb/JdbcConfig.java +++ b/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/rdb/JdbcConfig.java @@ -1,3 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package org.apache.eventmesh.common.config.connector.rdb; import org.apache.eventmesh.common.config.connector.rdb.canal.RdbDBDefinition; diff --git a/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/rdb/canal/CanalMySQLType.java b/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/rdb/canal/CanalMySQLType.java index e14ea78c82..b5107ccbf3 100644 --- a/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/rdb/canal/CanalMySQLType.java +++ b/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/rdb/canal/CanalMySQLType.java @@ -1,3 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package org.apache.eventmesh.common.config.connector.rdb.canal; import java.util.HashMap; @@ -48,16 +65,18 @@ public enum CanalMySQLType { private final String codeKey; private final MysqlType mysqlType; - + CanalMySQLType(String codeKey) { this.codeKey = codeKey; this.mysqlType = MysqlType.getByName(codeKey); } + private static final Map TYPES = new HashMap<>(); + static { CanalMySQLType[] values = values(); for (CanalMySQLType tableType : values) { - TYPES.put(tableType.codeKey, tableType); + TYPES.put(tableType.codeKey, tableType); } } diff --git a/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/rdb/canal/CanalSinkFullConfig.java b/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/rdb/canal/CanalSinkFullConfig.java index 33161e07ad..c2b881df6c 100644 --- a/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/rdb/canal/CanalSinkFullConfig.java +++ b/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/rdb/canal/CanalSinkFullConfig.java @@ -1,3 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package org.apache.eventmesh.common.config.connector.rdb.canal; import org.apache.eventmesh.common.config.connector.SinkConfig; diff --git a/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/rdb/canal/CanalSourceFullConfig.java b/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/rdb/canal/CanalSourceFullConfig.java index 6508e49fdb..a2ab8ba31d 100644 --- a/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/rdb/canal/CanalSourceFullConfig.java +++ b/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/rdb/canal/CanalSourceFullConfig.java @@ -1,12 +1,30 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package org.apache.eventmesh.common.config.connector.rdb.canal; -import lombok.Data; -import lombok.EqualsAndHashCode; import org.apache.eventmesh.common.config.connector.SourceConfig; import org.apache.eventmesh.common.remote.offset.RecordPosition; import java.util.List; +import lombok.Data; +import lombok.EqualsAndHashCode; + @Data @EqualsAndHashCode(callSuper = true) public class CanalSourceFullConfig extends SourceConfig { diff --git a/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/rdb/canal/JobRdbFullPosition.java b/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/rdb/canal/JobRdbFullPosition.java index 5f0f5326ed..08f88e1d24 100644 --- a/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/rdb/canal/JobRdbFullPosition.java +++ b/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/rdb/canal/JobRdbFullPosition.java @@ -1,10 +1,27 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package org.apache.eventmesh.common.config.connector.rdb.canal; +import java.math.BigDecimal; + import lombok.Data; import lombok.ToString; -import java.math.BigDecimal; - @Data @ToString public class JobRdbFullPosition { diff --git a/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/rdb/canal/RdbColumnDefinition.java b/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/rdb/canal/RdbColumnDefinition.java index 2fe4d7a2f9..94c0135c3e 100644 --- a/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/rdb/canal/RdbColumnDefinition.java +++ b/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/rdb/canal/RdbColumnDefinition.java @@ -1,9 +1,26 @@ -package org.apache.eventmesh.common.config.connector.rdb.canal; +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ -import lombok.Data; +package org.apache.eventmesh.common.config.connector.rdb.canal; import java.sql.JDBCType; +import lombok.Data; + @Data public class RdbColumnDefinition { protected String name; diff --git a/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/rdb/canal/RdbDBDefinition.java b/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/rdb/canal/RdbDBDefinition.java index 93ab696ad5..ab3ed336f8 100644 --- a/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/rdb/canal/RdbDBDefinition.java +++ b/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/rdb/canal/RdbDBDefinition.java @@ -1,3 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package org.apache.eventmesh.common.config.connector.rdb.canal; import java.util.Set; diff --git a/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/rdb/canal/RdbTableDefinition.java b/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/rdb/canal/RdbTableDefinition.java index 5001a8bcb8..c281035578 100644 --- a/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/rdb/canal/RdbTableDefinition.java +++ b/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/rdb/canal/RdbTableDefinition.java @@ -1,3 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package org.apache.eventmesh.common.config.connector.rdb.canal; import lombok.Data; diff --git a/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/rdb/canal/mysql/Constants.java b/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/rdb/canal/mysql/Constants.java index 3025be245b..8c51c7255b 100644 --- a/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/rdb/canal/mysql/Constants.java +++ b/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/rdb/canal/mysql/Constants.java @@ -1,3 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package org.apache.eventmesh.common.config.connector.rdb.canal.mysql; public class Constants { diff --git a/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/rdb/canal/mysql/MySQLColumnDef.java b/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/rdb/canal/mysql/MySQLColumnDef.java index 8f1fcbd741..cdc9adf33f 100644 --- a/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/rdb/canal/mysql/MySQLColumnDef.java +++ b/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/rdb/canal/mysql/MySQLColumnDef.java @@ -1,10 +1,28 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package org.apache.eventmesh.common.config.connector.rdb.canal.mysql; -import lombok.Data; -import lombok.EqualsAndHashCode; import org.apache.eventmesh.common.config.connector.rdb.canal.CanalMySQLType; import org.apache.eventmesh.common.config.connector.rdb.canal.RdbColumnDefinition; +import lombok.Data; +import lombok.EqualsAndHashCode; + @Data @EqualsAndHashCode(callSuper = true) public class MySQLColumnDef extends RdbColumnDefinition { diff --git a/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/rdb/canal/mysql/MySQLTableDef.java b/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/rdb/canal/mysql/MySQLTableDef.java index 0aa9a0f0d7..cdd3652378 100644 --- a/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/rdb/canal/mysql/MySQLTableDef.java +++ b/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/rdb/canal/mysql/MySQLTableDef.java @@ -1,3 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package org.apache.eventmesh.common.config.connector.rdb.canal.mysql; import org.apache.eventmesh.common.config.connector.rdb.canal.RdbTableDefinition; diff --git a/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/offset/canal/CanalFullRecordOffset.java b/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/offset/canal/CanalFullRecordOffset.java index f1f3a7c132..a0a077b7f5 100644 --- a/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/offset/canal/CanalFullRecordOffset.java +++ b/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/offset/canal/CanalFullRecordOffset.java @@ -1,16 +1,35 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package org.apache.eventmesh.common.remote.offset.canal; +import org.apache.eventmesh.common.config.connector.rdb.canal.JobRdbFullPosition; +import org.apache.eventmesh.common.remote.offset.RecordOffset; + import lombok.Data; import lombok.EqualsAndHashCode; import lombok.ToString; -import org.apache.eventmesh.common.config.connector.rdb.canal.JobRdbFullPosition; -import org.apache.eventmesh.common.remote.offset.RecordOffset; @Data @EqualsAndHashCode(callSuper = true) @ToString public class CanalFullRecordOffset extends RecordOffset { private JobRdbFullPosition position; + @Override public Class getRecordOffsetClass() { return CanalFullRecordOffset.class; diff --git a/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/sink/connector/CanalSinkFullConnector.java b/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/sink/connector/CanalSinkFullConnector.java index 027bc7e15b..f4577e09fc 100644 --- a/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/sink/connector/CanalSinkFullConnector.java +++ b/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/sink/connector/CanalSinkFullConnector.java @@ -1,3 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package org.apache.eventmesh.connector.canal.sink.connector; import org.apache.eventmesh.common.config.connector.Config; diff --git a/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/source/connector/CanalFullProducer.java b/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/source/connector/CanalFullProducer.java index 31b3574869..1b20b138d1 100644 --- a/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/source/connector/CanalFullProducer.java +++ b/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/source/connector/CanalFullProducer.java @@ -1,3 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package org.apache.eventmesh.connector.canal.source.connector; import org.apache.eventmesh.common.config.connector.rdb.canal.CanalMySQLType; diff --git a/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/source/connector/CanalSourceFullConnector.java b/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/source/connector/CanalSourceFullConnector.java index b24a8b36e1..df3c7571c2 100644 --- a/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/source/connector/CanalSourceFullConnector.java +++ b/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/source/connector/CanalSourceFullConnector.java @@ -1,3 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package org.apache.eventmesh.connector.canal.source.connector; import org.apache.eventmesh.common.AbstractComponent; @@ -41,7 +58,7 @@ public class CanalSourceFullConnector extends AbstractComponent implements Sourc private final AtomicBoolean flag = new AtomicBoolean(true); @Override - protected void startup() throws Exception { + protected void run() throws Exception { this.tableMgr.start(); this.positionMgr.start(); if (positionMgr.isFinished()) { diff --git a/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/source/position/CanalFullPositionMgr.java b/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/source/position/CanalFullPositionMgr.java index c95c50b5ce..7e77053406 100644 --- a/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/source/position/CanalFullPositionMgr.java +++ b/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/source/position/CanalFullPositionMgr.java @@ -1,3 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package org.apache.eventmesh.connector.canal.source.position; import org.apache.eventmesh.common.AbstractComponent; @@ -33,7 +50,6 @@ public class CanalFullPositionMgr extends AbstractComponent { private final CanalSourceFullConfig config; -// private ScheduledThreadPoolExecutor executor; private final Map positions = new LinkedHashMap<>(); private final RdbTableMgr tableMgr; @@ -43,17 +59,11 @@ public CanalFullPositionMgr(CanalSourceFullConfig config, RdbTableMgr tableMgr) } @Override - protected void startup() throws Exception { + protected void run() throws Exception { if (config == null || config.getConnectorConfig() == null || config.getConnectorConfig().getDatabases() == null) { log.info("config or database is null"); return; } -// executor = new ScheduledThreadPoolExecutor(1, r -> { -// Thread thread = new Thread(r); -// thread.setDaemon(true); -// thread.setName("task-full-position-timer"); -// return thread; -// }, new ScheduledThreadPoolExecutor.DiscardOldestPolicy()); prepareRecordPosition(); initPositions(); } diff --git a/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/source/table/RdbSimpleTable.java b/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/source/table/RdbSimpleTable.java index cd8fa4c93b..1ec2a4c4a5 100644 --- a/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/source/table/RdbSimpleTable.java +++ b/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/source/table/RdbSimpleTable.java @@ -1,19 +1,24 @@ package org.apache.eventmesh.connector.canal.source.table; -import lombok.AllArgsConstructor; -import lombok.Data; import org.apache.eventmesh.common.config.connector.rdb.canal.RdbTableDefinition; import java.util.Objects; +import lombok.Data; + @Data public class RdbSimpleTable extends RdbTableDefinition { - public RdbSimpleTable(String schema, String tableName) { - super(); - this.schema = schema; + public RdbSimpleTable(String database, String schema, String tableName) { + this.schemaName = schema; this.tableName = tableName; + this.database = database; } - private String schema; + + public RdbSimpleTable(String schema, String tableName) { + this(null, schema, tableName); + } + + private final String database; @Override public boolean equals(Object o) { @@ -21,11 +26,11 @@ public boolean equals(Object o) { if (o == null || getClass() != o.getClass()) return false; if (!super.equals(o)) return false; RdbSimpleTable that = (RdbSimpleTable) o; - return Objects.equals(schema, that.schema); + return Objects.equals(database, that.database); } @Override public int hashCode() { - return Objects.hash(super.hashCode(), schema); + return Objects.hash(super.hashCode(), database); } } diff --git a/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/source/table/RdbTableMgr.java b/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/source/table/RdbTableMgr.java index 470de5f39a..158e879404 100644 --- a/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/source/table/RdbTableMgr.java +++ b/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/source/table/RdbTableMgr.java @@ -1,3 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package org.apache.eventmesh.connector.canal.source.table; import org.apache.eventmesh.common.AbstractComponent; @@ -49,7 +66,7 @@ public RdbTableDefinition getTable(RdbSimpleTable table) { } @Override - protected void startup() { + protected void run() { if (config != null && config.getDatabases() != null) { for (RdbDBDefinition db : config.getDatabases()) { if (db.getTables() == null) { From 540e5bd14cc763de20c0a07880c51df6a72d9206 Mon Sep 17 00:00:00 2001 From: sodaRyCN <757083350@qq.com> Date: Tue, 23 Jul 2024 17:54:54 +0800 Subject: [PATCH 14/15] fix checkstyle and license --- .../connector/canal/DatabaseConnection.java | 12 ++--- .../eventmesh/connector/canal/SqlUtils.java | 21 ++++----- .../connector/CanalSinkFullConnector.java | 2 +- .../source/connector/CanalFullProducer.java | 2 +- .../source/position/CanalFullPositionMgr.java | 46 ++++++++----------- .../source/position/TableFullPosition.java | 27 +++++++++-- .../canal/source/table/RdbSimpleTable.java | 29 ++++++++++-- .../canal/source/table/RdbTableMgr.java | 30 +++++------- 8 files changed, 95 insertions(+), 74 deletions(-) diff --git a/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/DatabaseConnection.java b/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/DatabaseConnection.java index 6fbda08d83..0310e5434c 100644 --- a/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/DatabaseConnection.java +++ b/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/DatabaseConnection.java @@ -36,10 +36,10 @@ public class DatabaseConnection { public static SinkConnectorConfig sinkConfig; - public static DruidDataSource createDruidDataSource(String url, String UserName, String passWord) { + public static DruidDataSource createDruidDataSource(String url, String userName, String passWord) { DruidDataSource dataSource = new DruidDataSource(); dataSource.setUrl(url); - dataSource.setUsername(UserName); + dataSource.setUsername(userName); dataSource.setPassword(passWord); dataSource.setInitialSize(5); dataSource.setMinIdle(5); @@ -58,14 +58,14 @@ public static DruidDataSource createDruidDataSource(String url, String UserName, public static void initSourceConnection() { sourceDataSource = createDruidDataSource(sourceConfig.getUrl(), - sourceConfig.getUserName(), - sourceConfig.getPassWord()); + sourceConfig.getUserName(), + sourceConfig.getPassWord()); } public static void initSinkConnection() { sinkDataSource = createDruidDataSource(sinkConfig.getUrl(), - sinkConfig.getUserName(), - sinkConfig.getPassWord()); + sinkConfig.getUserName(), + sinkConfig.getPassWord()); } diff --git a/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/SqlUtils.java b/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/SqlUtils.java index 05b6b1836f..9e8403092e 100644 --- a/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/SqlUtils.java +++ b/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/SqlUtils.java @@ -146,8 +146,7 @@ public static void setInClauseParameters(PreparedStatement preparedStatement, Li setInClauseParameters(preparedStatement, 0, params); } - public static void setInClauseParameters(PreparedStatement preparedStatement, int paramIndexStart, - List params) throws SQLException { + public static void setInClauseParameters(PreparedStatement preparedStatement, int paramIndexStart, List params) throws SQLException { for (int i = 0; i < params.size(); i++) { preparedStatement.setString(paramIndexStart + i, params.get(i)); } @@ -261,8 +260,7 @@ private static String getResultSetValue(ResultSet rs, int index, Class requir } else if (float.class.equals(requiredType) || Float.class.equals(requiredType)) { value = rs.getFloat(index); wasNullCheck = true; - } else if (double.class.equals(requiredType) || Double.class.equals(requiredType) - || Number.class.equals(requiredType)) { + } else if (double.class.equals(requiredType) || Double.class.equals(requiredType) || Number.class.equals(requiredType)) { value = rs.getDouble(index); wasNullCheck = true; } else if (Time.class.equals(requiredType)) { @@ -326,16 +324,14 @@ private static String getResultSetValue(ResultSet rs, int index) throws SQLExcep * Check whether the given SQL type is numeric. */ public static boolean isNumeric(int sqlType) { - return (Types.BIT == sqlType) || (Types.BIGINT == sqlType) || (Types.DECIMAL == sqlType) - || (Types.DOUBLE == sqlType) || (Types.FLOAT == sqlType) || (Types.INTEGER == sqlType) - || (Types.NUMERIC == sqlType) || (Types.REAL == sqlType) || (Types.SMALLINT == sqlType) - || (Types.TINYINT == sqlType); + return (Types.BIT == sqlType) || (Types.BIGINT == sqlType) || (Types.DECIMAL == sqlType) || (Types.DOUBLE == sqlType) || + (Types.FLOAT == sqlType) || (Types.INTEGER == sqlType) || (Types.NUMERIC == sqlType) || (Types.REAL == sqlType) || + (Types.SMALLINT == sqlType) || (Types.TINYINT == sqlType); } public static boolean isTextType(int sqlType) { - return sqlType == Types.CHAR || sqlType == Types.VARCHAR || sqlType == Types.CLOB || sqlType == Types.LONGVARCHAR - || sqlType == Types.NCHAR || sqlType == Types.NVARCHAR || sqlType == Types.NCLOB - || sqlType == Types.LONGNVARCHAR; + return sqlType == Types.CHAR || sqlType == Types.VARCHAR || sqlType == Types.CLOB || sqlType == Types.LONGVARCHAR || sqlType == Types.NCHAR || + sqlType == Types.NVARCHAR || sqlType == Types.NCLOB || sqlType == Types.LONGNVARCHAR; } public static JDBCType toJDBCType(String connectorDataType) { @@ -861,8 +857,7 @@ public static String toGeometry(Object value) throws Exception { } return new WKBReader().read(bytes).toText(); } else { - throw new UnsupportedOperationException("class " + value.getClass() + ", value '" + value + "' , " + - "parse to geometry failed."); + throw new UnsupportedOperationException("class " + value.getClass() + ", value '" + value + "' , " + "parse to geometry failed."); } } diff --git a/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/sink/connector/CanalSinkFullConnector.java b/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/sink/connector/CanalSinkFullConnector.java index f4577e09fc..d7d07afcf3 100644 --- a/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/sink/connector/CanalSinkFullConnector.java +++ b/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/sink/connector/CanalSinkFullConnector.java @@ -155,7 +155,7 @@ public void put(List sinkRecords) { statement.executeBatch(); connection.commit(); } catch (SQLException e) { - log.warn("full sink process schema [{}] table [{}] connector write fail", tableDefinition.getSchemaName(),tableDefinition.getTableName(), + log.warn("full sink process schema [{}] table [{}] connector write fail", tableDefinition.getSchemaName(), tableDefinition.getTableName(), e); LockSupport.parkNanos(3000 * 1000L); } catch (Exception e) { diff --git a/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/source/connector/CanalFullProducer.java b/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/source/connector/CanalFullProducer.java index 1b20b138d1..83146de733 100644 --- a/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/source/connector/CanalFullProducer.java +++ b/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/source/connector/CanalFullProducer.java @@ -138,7 +138,7 @@ public void start(AtomicBoolean flag) { return; } } catch (SQLException e) { - log.error("full source process schema [{}] table [{}] catch SQLException fail",tableDefinition.getSchemaName(), + log.error("full source process schema [{}] table [{}] catch SQLException fail", tableDefinition.getSchemaName(), tableDefinition.getTableName(), e); LockSupport.parkNanos(3000 * 1000L); } catch (Exception e) { diff --git a/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/source/position/CanalFullPositionMgr.java b/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/source/position/CanalFullPositionMgr.java index 7e77053406..c5bec2cc56 100644 --- a/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/source/position/CanalFullPositionMgr.java +++ b/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/source/position/CanalFullPositionMgr.java @@ -99,29 +99,26 @@ private void initPositions() { RdbSimpleTable simpleTable = new RdbSimpleTable(database.getSchemaName(), table.getTableName()); RdbTableDefinition tableDefinition; if ((tableDefinition = tableMgr.getTable(simpleTable)) == null) { - log.error("db [{}] table [{}] definition is null", database.getSchemaName(), - table.getTableName()); + log.error("db [{}] table [{}] definition is null", database.getSchemaName(), table.getTableName()); continue; } - log.info("init position of data [{}] table [{}]", database.getSchemaName(), - table.getTableName()); + log.info("init position of data [{}] table [{}]", database.getSchemaName(), table.getTableName()); JobRdbFullPosition recordPosition = positions.get(simpleTable); if (recordPosition == null || !recordPosition.isFinished()) { - positions.put(simpleTable, fetchTableInfo(DatabaseConnection.sourceDataSource, (MySQLTableDef) tableDefinition, - recordPosition)); + positions.put(simpleTable, + fetchTableInfo(DatabaseConnection.sourceDataSource, (MySQLTableDef) tableDefinition, recordPosition)); } } catch (Exception e) { - log.error("process schema [{}] table [{}] position fail", database.getSchemaName(), - table.getTableName(), e); + log.error("process schema [{}] table [{}] position fail", database.getSchemaName(), table.getTableName(), e); } } } } - private JobRdbFullPosition fetchTableInfo(DataSource dataSource, MySQLTableDef tableDefinition, - JobRdbFullPosition recordPosition) throws SQLException { + private JobRdbFullPosition fetchTableInfo(DataSource dataSource, MySQLTableDef tableDefinition, JobRdbFullPosition recordPosition) + throws SQLException { TableFullPosition position = new TableFullPosition(); Map preMinPrimaryKeys = new LinkedHashMap<>(); Map preMaxPrimaryKeys = new LinkedHashMap<>(); @@ -138,8 +135,7 @@ private JobRdbFullPosition fetchTableInfo(DataSource dataSource, MySQLTableDef t JobRdbFullPosition jobRdbFullPosition = new JobRdbFullPosition(); if (recordPosition != null) { if (StringUtils.isNotBlank(recordPosition.getPrimaryKeyRecords())) { - TableFullPosition record = JsonUtils.parseObject(recordPosition.getPrimaryKeyRecords(), - TableFullPosition.class); + TableFullPosition record = JsonUtils.parseObject(recordPosition.getPrimaryKeyRecords(), TableFullPosition.class); if (record != null && record.getCurPrimaryKeyCols() != null && !record.getCurPrimaryKeyCols().isEmpty()) { position.setCurPrimaryKeyCols(record.getCurPrimaryKeyCols()); } @@ -155,11 +151,9 @@ private JobRdbFullPosition fetchTableInfo(DataSource dataSource, MySQLTableDef t private long queryCurTableRowCount(DataSource datasource, MySQLTableDef tableDefinition) throws SQLException { - String sql = - "select `AVG_ROW_LENGTH`,`DATA_LENGTH` from information_schema.TABLES where `TABLE_SCHEMA`='" + tableDefinition.getSchemaName() + - "' and `TABLE_NAME`='" + tableDefinition.getTableName() + "'"; - try (Statement statement = datasource.getConnection().createStatement(); ResultSet resultSet = - statement.executeQuery(sql)) { + String sql = "select `AVG_ROW_LENGTH`,`DATA_LENGTH` from information_schema.TABLES where `TABLE_SCHEMA`='" + tableDefinition.getSchemaName() + + "' and `TABLE_NAME`='" + tableDefinition.getTableName() + "'"; + try (Statement statement = datasource.getConnection().createStatement(); ResultSet resultSet = statement.executeQuery(sql)) { long result = 0L; if (resultSet.next()) { long avgRowLength = resultSet.getLong("AVG_ROW_LENGTH"); @@ -198,13 +192,12 @@ private void setValue2Statement(PreparedStatement ps, Map preMap } } - private Object fetchMinPrimaryKey(DataSource dataSource, MySQLTableDef tableDefinition, - Map prePrimary, String curPrimaryKeyCol) throws SQLException { + private Object fetchMinPrimaryKey(DataSource dataSource, MySQLTableDef tableDefinition, Map prePrimary, String curPrimaryKeyCol) + throws SQLException { StringBuilder builder = new StringBuilder(); builder.append("SELECT MIN(").append(Constants.MySQLQuot).append(curPrimaryKeyCol).append(Constants.MySQLQuot) - .append(") min_primary_key FROM") - .append(Constants.MySQLQuot).append(tableDefinition.getSchemaName()).append(Constants.MySQLQuot).append(".").append(Constants.MySQLQuot) - .append(tableDefinition.getTableName()).append(Constants.MySQLQuot); + .append(") min_primary_key FROM").append(Constants.MySQLQuot).append(tableDefinition.getSchemaName()).append(Constants.MySQLQuot) + .append(".").append(Constants.MySQLQuot).append(tableDefinition.getTableName()).append(Constants.MySQLQuot); appendPrePrimaryKey(prePrimary, builder); String sql = builder.toString(); log.info("fetch min primary sql [{}]", sql); @@ -224,13 +217,12 @@ private Object fetchMinPrimaryKey(DataSource dataSource, MySQLTableDef tableDefi return null; } - private Object fetchMaxPrimaryKey(DataSource dataSource, MySQLTableDef tableDefinition, - Map prePrimary, String curPrimaryKeyCol) throws SQLException { + private Object fetchMaxPrimaryKey(DataSource dataSource, MySQLTableDef tableDefinition, Map prePrimary, String curPrimaryKeyCol) + throws SQLException { StringBuilder builder = new StringBuilder(); builder.append("SELECT MAX(").append(Constants.MySQLQuot).append(curPrimaryKeyCol).append(Constants.MySQLQuot) - .append(") max_primary_key FROM") - .append(Constants.MySQLQuot).append(tableDefinition.getSchemaName()).append(Constants.MySQLQuot).append(".").append(Constants.MySQLQuot) - .append(tableDefinition.getTableName()).append(Constants.MySQLQuot); + .append(") max_primary_key FROM").append(Constants.MySQLQuot).append(tableDefinition.getSchemaName()).append(Constants.MySQLQuot) + .append(".").append(Constants.MySQLQuot).append(tableDefinition.getTableName()).append(Constants.MySQLQuot); appendPrePrimaryKey(prePrimary, builder); String sql = builder.toString(); log.info("fetch max primary sql [{}]", sql); diff --git a/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/source/position/TableFullPosition.java b/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/source/position/TableFullPosition.java index b4b30e7a24..b1a8024ec5 100644 --- a/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/source/position/TableFullPosition.java +++ b/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/source/position/TableFullPosition.java @@ -1,13 +1,30 @@ -package org.apache.eventmesh.connector.canal.source.position; +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ -import lombok.Data; +package org.apache.eventmesh.connector.canal.source.position; import java.util.LinkedHashMap; import java.util.Map; +import lombok.Data; + @Data public class TableFullPosition { - private Map curPrimaryKeyCols = new LinkedHashMap<>(); - private Map minPrimaryKeyCols = new LinkedHashMap<>(); - private Map maxPrimaryKeyCols = new LinkedHashMap<>(); + private Map curPrimaryKeyCols = new LinkedHashMap<>(); + private Map minPrimaryKeyCols = new LinkedHashMap<>(); + private Map maxPrimaryKeyCols = new LinkedHashMap<>(); } diff --git a/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/source/table/RdbSimpleTable.java b/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/source/table/RdbSimpleTable.java index 1ec2a4c4a5..5b9c35fff3 100644 --- a/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/source/table/RdbSimpleTable.java +++ b/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/source/table/RdbSimpleTable.java @@ -1,3 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package org.apache.eventmesh.connector.canal.source.table; import org.apache.eventmesh.common.config.connector.rdb.canal.RdbTableDefinition; @@ -22,9 +39,15 @@ public RdbSimpleTable(String schema, String tableName) { @Override public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - if (!super.equals(o)) return false; + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + if (!super.equals(o)) { + return false; + } RdbSimpleTable that = (RdbSimpleTable) o; return Objects.equals(database, that.database); } diff --git a/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/source/table/RdbTableMgr.java b/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/source/table/RdbTableMgr.java index 158e879404..2e99d06096 100644 --- a/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/source/table/RdbTableMgr.java +++ b/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/source/table/RdbTableMgr.java @@ -79,18 +79,15 @@ protected void run() { mysqlTable.setSchemaName(db.getSchemaName()); mysqlTable.setTableName(table.getTableName()); List tables = Collections.singletonList(table.getTableName()); - Map> primaryKeys = queryTablePrimaryKey(db.getSchemaName(), - tables); + Map> primaryKeys = queryTablePrimaryKey(db.getSchemaName(), tables); Map> columns = queryColumns(db.getSchemaName(), tables); if (primaryKeys == null || primaryKeys.isEmpty() || primaryKeys.get(table.getTableName()) == null) { - log.warn("init db [{}] table [{}] info, and primary keys are empty", db.getSchemaName(), - table.getTableName()); + log.warn("init db [{}] table [{}] info, and primary keys are empty", db.getSchemaName(), table.getTableName()); } else { mysqlTable.setPrimaryKeys(new HashSet<>(primaryKeys.get(table.getTableName()))); } if (columns == null || columns.isEmpty() || columns.get(table.getTableName()) == null) { - log.warn("init db [{}] table [{}] info, and columns are empty", db.getSchemaName(), - table.getTableName()); + log.warn("init db [{}] table [{}] info, and columns are empty", db.getSchemaName(), table.getTableName()); } else { LinkedHashMap cols = new LinkedHashMap<>(); columns.get(table.getTableName()).forEach(x -> cols.put(x.getName(), x)); @@ -99,8 +96,7 @@ protected void run() { this.tables.put(new RdbSimpleTable(db.getSchemaName(), table.getTableName()), mysqlTable); } catch (Exception e) { - log.error("init rdb table schema [{}] table [{}] fail", db.getSchemaName(), - table.getTableName(), e); + log.error("init rdb table schema [{}] table [{}] fail", db.getSchemaName(), table.getTableName(), e); } } @@ -108,16 +104,15 @@ protected void run() { } } - private Map> queryTablePrimaryKey(String schema, - List tables) throws SQLException { + private Map> queryTablePrimaryKey(String schema, List tables) throws SQLException { Map> primaryKeys = new LinkedHashMap<>(); String prepareTables = SqlUtils.genPrepareSqlOfInClause(tables.size()); - String sql = "select L.TABLE_NAME,L.COLUMN_NAME,R.CONSTRAINT_TYPE from " + - "INFORMATION_SCHEMA.KEY_COLUMN_USAGE L left join INFORMATION_SCHEMA.TABLE_CONSTRAINTS R on L" + - ".TABLE_SCHEMA = R.TABLE_SCHEMA and L.TABLE_NAME = R.TABLE_NAME and L.CONSTRAINT_CATALOG = R" + - ".CONSTRAINT_CATALOG and L.CONSTRAINT_SCHEMA = R.CONSTRAINT_SCHEMA and L.CONSTRAINT_NAME = R" + - ".CONSTRAINT_NAME where L.TABLE_SCHEMA = ? and L.TABLE_NAME in " + prepareTables + " and R" + - ".CONSTRAINT_TYPE IN ('PRIMARY KEY') order by L.ORDINAL_POSITION asc"; + String sql = "select L.TABLE_NAME,L.COLUMN_NAME,R.CONSTRAINT_TYPE from " + + "INFORMATION_SCHEMA.KEY_COLUMN_USAGE L left join INFORMATION_SCHEMA.TABLE_CONSTRAINTS R on L" + + ".TABLE_SCHEMA = R.TABLE_SCHEMA and L.TABLE_NAME = R.TABLE_NAME and L.CONSTRAINT_CATALOG = R" + + ".CONSTRAINT_CATALOG and L.CONSTRAINT_SCHEMA = R.CONSTRAINT_SCHEMA and L.CONSTRAINT_NAME = R" + + ".CONSTRAINT_NAME where L.TABLE_SCHEMA = ? and L.TABLE_NAME in " + prepareTables + " and R" + + ".CONSTRAINT_TYPE IN ('PRIMARY KEY') order by L.ORDINAL_POSITION asc"; try (PreparedStatement statement = dataSource.getConnection().prepareStatement(sql)) { statement.setString(1, schema); SqlUtils.setInClauseParameters(statement, 2, tables); @@ -146,8 +141,7 @@ private Map> queryColumns(String schema, List> cols = new LinkedHashMap<>(); try (PreparedStatement statement = dataSource.getConnection().prepareStatement(sql)) { statement.setString(1, schema); From 23b3585d8d7def2ac6b5774b726c6af7a1e6891a Mon Sep 17 00:00:00 2001 From: sodaRyCN <757083350@qq.com> Date: Tue, 23 Jul 2024 19:44:04 +0800 Subject: [PATCH 15/15] fix checkstyle and license --- .../eventmesh/connector/canal/SqlUtils.java | 204 +++++++++--------- .../connector/CanalSinkFullConnector.java | 12 +- .../source/connector/CanalFullProducer.java | 20 +- .../source/position/CanalFullPositionMgr.java | 6 +- .../canal/source/table/RdbTableMgr.java | 12 +- 5 files changed, 125 insertions(+), 129 deletions(-) diff --git a/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/SqlUtils.java b/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/SqlUtils.java index 9e8403092e..1008ad1cf3 100644 --- a/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/SqlUtils.java +++ b/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/SqlUtils.java @@ -324,14 +324,14 @@ private static String getResultSetValue(ResultSet rs, int index) throws SQLExcep * Check whether the given SQL type is numeric. */ public static boolean isNumeric(int sqlType) { - return (Types.BIT == sqlType) || (Types.BIGINT == sqlType) || (Types.DECIMAL == sqlType) || (Types.DOUBLE == sqlType) || - (Types.FLOAT == sqlType) || (Types.INTEGER == sqlType) || (Types.NUMERIC == sqlType) || (Types.REAL == sqlType) || - (Types.SMALLINT == sqlType) || (Types.TINYINT == sqlType); + return (Types.BIT == sqlType) || (Types.BIGINT == sqlType) || (Types.DECIMAL == sqlType) || (Types.DOUBLE == sqlType) + || (Types.FLOAT == sqlType) || (Types.INTEGER == sqlType) || (Types.NUMERIC == sqlType) || (Types.REAL == sqlType) + || (Types.SMALLINT == sqlType) || (Types.TINYINT == sqlType); } public static boolean isTextType(int sqlType) { - return sqlType == Types.CHAR || sqlType == Types.VARCHAR || sqlType == Types.CLOB || sqlType == Types.LONGVARCHAR || sqlType == Types.NCHAR || - sqlType == Types.NVARCHAR || sqlType == Types.NCLOB || sqlType == Types.LONGNVARCHAR; + return sqlType == Types.CHAR || sqlType == Types.VARCHAR || sqlType == Types.CLOB || sqlType == Types.LONGVARCHAR || sqlType == Types.NCHAR + || sqlType == Types.NVARCHAR || sqlType == Types.NCLOB || sqlType == Types.LONGNVARCHAR; } public static JDBCType toJDBCType(String connectorDataType) { @@ -473,119 +473,37 @@ public static String removeZone(String datetime) { } if (len >= 7) { char checkCharAt1 = datetime.charAt(len - 2); - char checkCharAt2 = datetime.charAt(len - 3); - char checkCharAt3 = datetime.charAt(len - 6); - char checkCharAt4 = datetime.charAt(len - 5); - char checkCharAt5 = len >= 9 ? datetime.charAt(len - 9) : ' '; - char checkCharAt6 = datetime.charAt(len - 7); - boolean matchTest1 = (checkCharAt1 == '+' || checkCharAt1 == '-') && len >= 10; - boolean matchTest2 = (checkCharAt2 == '+' || checkCharAt2 == '-') && len >= 11; - boolean matchTest3 = (checkCharAt3 == '+' || checkCharAt3 == '-') && checkCharAt2 == ':'; - boolean matchTest4 = (checkCharAt4 == '+' || checkCharAt4 == '-') && checkCharAt2 == ':'; - boolean matchTest5 = (checkCharAt5 == '+' || checkCharAt5 == '-') && checkCharAt2 == ':' && checkCharAt3 == ':'; - boolean matchTest6 = checkCharAt6 == '+' || checkCharAt6 == '-'; - boolean matchTest7 = checkCharAt4 == '+' || checkCharAt4 == '-'; - if (matchTest1) { + if ((checkCharAt1 == '+' || checkCharAt1 == '-') && len >= 10) { return datetime.substring(0, len - 2).trim(); } - if (matchTest2) { + char checkCharAt2 = datetime.charAt(len - 3); + if ((checkCharAt2 == '+' || checkCharAt2 == '-') && len >= 11) { return datetime.substring(0, len - 3).trim(); } - if (matchTest3) { + char checkCharAt3 = datetime.charAt(len - 6); + if ((checkCharAt3 == '+' || checkCharAt3 == '-') && checkCharAt2 == ':') { return datetime.substring(0, len - 6).trim(); } - if (matchTest4) { + char checkCharAt4 = datetime.charAt(len - 5); + if ((checkCharAt4 == '+' || checkCharAt4 == '-') && checkCharAt2 == ':') { return datetime.substring(0, len - 5).trim(); } - if (matchTest5) { + char checkCharAt5 = len >= 9 ? datetime.charAt(len - 9) : ' '; + if ((checkCharAt5 == '+' || checkCharAt5 == '-') && checkCharAt2 == ':' && checkCharAt3 == ':') { return datetime.substring(0, len - 9).trim(); } - if (matchTest6) { + char checkCharAt6 = datetime.charAt(len - 7); + if (checkCharAt6 == '+' || checkCharAt6 == '-') { return datetime.substring(0, len - 7).trim(); } - if (matchTest7) { + if (checkCharAt4 == '+' || checkCharAt4 == '-') { return datetime.substring(0, len - 5).trim(); } } return datetime; } - private static LocalDateTime toLocalDateTime(String value) { - if (value.trim().length() >= 4) { - String dateStr2 = removeZone(value); - int len = dateStr2.length(); - if (len == 4) { - return LocalDateTime.of(Integer.parseInt(dateStr2), 1, 1, 0, 0, 0, 0); - } - if (dateStr2.charAt(4) == '-') { - switch (len) { - case 7: - String[] dataParts = dateStr2.split("-"); - return LocalDateTime.of(Integer.parseInt(dataParts[0]), Integer.parseInt(dataParts[1]), 1, 0, 0, 0, 0); - case 8: - case 9: - case 11: - case 12: - case 14: - case 15: - case 17: - case 18: - default: - String[] dataTime = dateStr2.split(" "); - String[] dataParts2 = dataTime[0].split("-"); - String[] timeParts = dataTime[1].split(":"); - String[] secondParts = timeParts[2].split("\\."); - secondParts[1] = StringUtils.rightPad(secondParts[1], 9, Constants.CJ_MINOR_VERSION); - return LocalDateTime.of(Integer.parseInt(dataParts2[0]), Integer.parseInt(dataParts2[1]), Integer.parseInt(dataParts2[2]), - Integer.parseInt(timeParts[0]), Integer.parseInt(timeParts[1]), Integer.parseInt(secondParts[0]), - Integer.parseInt(secondParts[1])); - case 10: - String[] dataParts3 = dateStr2.split("-"); - return LocalDateTime.of(Integer.parseInt(dataParts3[0]), Integer.parseInt(dataParts3[1]), Integer.parseInt(dataParts3[2]), 0, - 0, 0, 0); - case 13: - String[] dataTime2 = dateStr2.split(" "); - String[] dataParts4 = dataTime2[0].split("-"); - return LocalDateTime.of(Integer.parseInt(dataParts4[0]), Integer.parseInt(dataParts4[1]), Integer.parseInt(dataParts4[2]), - Integer.parseInt(dataTime2[1]), 0, 0, 0); - case 16: - String[] dataTime3 = dateStr2.split(" "); - String[] dataParts5 = dataTime3[0].split("-"); - String[] timeParts2 = dataTime3[1].split(":"); - return LocalDateTime.of(Integer.parseInt(dataParts5[0]), Integer.parseInt(dataParts5[1]), Integer.parseInt(dataParts5[2]), - Integer.parseInt(timeParts2[0]), Integer.parseInt(timeParts2[1]), 0, 0); - case 19: - String[] dataTime4 = dateStr2.split(" "); - String[] dataParts6 = dataTime4[0].split("-"); - String[] timeParts3 = dataTime4[1].split(":"); - return LocalDateTime.of(Integer.parseInt(dataParts6[0]), Integer.parseInt(dataParts6[1]), Integer.parseInt(dataParts6[2]), - Integer.parseInt(timeParts3[0]), Integer.parseInt(timeParts3[1]), Integer.parseInt(timeParts3[2]), 0); - } - } else if (dateStr2.charAt(2) == ':') { - switch (len) { - case 5: - String[] timeParts4 = dateStr2.split(":"); - return LocalDateTime.of(0, 1, 1, Integer.parseInt(timeParts4[0]), Integer.parseInt(timeParts4[1]), 0, 0); - case 8: - String[] timeParts5 = dateStr2.split(":"); - return LocalDateTime.of(0, 1, 1, Integer.parseInt(timeParts5[0]), Integer.parseInt(timeParts5[1]), - Integer.parseInt(timeParts5[2]), 0); - default: - String[] timeParts6 = dateStr2.split(":"); - String[] secondParts2 = timeParts6[2].split("\\."); - secondParts2[1] = StringUtils.rightPad(secondParts2[1], 9, Constants.CJ_MINOR_VERSION); - return LocalDateTime.of(0, 1, 1, Integer.parseInt(timeParts6[0]), Integer.parseInt(timeParts6[1]), - Integer.parseInt(secondParts2[0]), Integer.parseInt(secondParts2[1])); - } - } else { - throw new UnsupportedOperationException(value.getClass() + ", value '" + value + "' , parse to local date time failed."); - } - } else if (StringUtils.isNumeric(value)) { - return LocalDateTime.of(Integer.parseInt(value), 1, 1, 0, 0, 0, 0); - } else { - throw new DateTimeException(value + " format error."); - } - } + public static String bytes2hex(byte[] b) { if (b == null) { @@ -760,6 +678,83 @@ public static Integer toInt(Object value) { } } + private static LocalDateTime toLocalDateTime(String value) { + if (value.trim().length() >= 4) { + String dateStr2 = removeZone(value); + int len = dateStr2.length(); + if (len == 4) { + return LocalDateTime.of(Integer.parseInt(dateStr2), 1, 1, 0, 0, 0, 0); + } + if (dateStr2.charAt(4) == '-') { + switch (len) { + case 7: + String[] dataParts = dateStr2.split("-"); + return LocalDateTime.of(Integer.parseInt(dataParts[0]), Integer.parseInt(dataParts[1]), 1, 0, 0, 0, 0); + case 8: + case 9: + case 11: + case 12: + case 14: + case 15: + case 17: + case 18: + default: + String[] dataTime = dateStr2.split(" "); + String[] dataParts2 = dataTime[0].split("-"); + String[] timeParts = dataTime[1].split(":"); + String[] secondParts = timeParts[2].split("\\."); + secondParts[1] = StringUtils.rightPad(secondParts[1], 9, Constants.CJ_MINOR_VERSION); + return LocalDateTime.of(Integer.parseInt(dataParts2[0]), Integer.parseInt(dataParts2[1]), Integer.parseInt(dataParts2[2]), + Integer.parseInt(timeParts[0]), Integer.parseInt(timeParts[1]), Integer.parseInt(secondParts[0]), + Integer.parseInt(secondParts[1])); + case 10: + String[] dataParts3 = dateStr2.split("-"); + return LocalDateTime.of(Integer.parseInt(dataParts3[0]), Integer.parseInt(dataParts3[1]), Integer.parseInt(dataParts3[2]), 0, + 0, 0, 0); + case 13: + String[] dataTime2 = dateStr2.split(" "); + String[] dataParts4 = dataTime2[0].split("-"); + return LocalDateTime.of(Integer.parseInt(dataParts4[0]), Integer.parseInt(dataParts4[1]), Integer.parseInt(dataParts4[2]), + Integer.parseInt(dataTime2[1]), 0, 0, 0); + case 16: + String[] dataTime3 = dateStr2.split(" "); + String[] dataParts5 = dataTime3[0].split("-"); + String[] timeParts2 = dataTime3[1].split(":"); + return LocalDateTime.of(Integer.parseInt(dataParts5[0]), Integer.parseInt(dataParts5[1]), Integer.parseInt(dataParts5[2]), + Integer.parseInt(timeParts2[0]), Integer.parseInt(timeParts2[1]), 0, 0); + case 19: + String[] dataTime4 = dateStr2.split(" "); + String[] dataParts6 = dataTime4[0].split("-"); + String[] timeParts3 = dataTime4[1].split(":"); + return LocalDateTime.of(Integer.parseInt(dataParts6[0]), Integer.parseInt(dataParts6[1]), Integer.parseInt(dataParts6[2]), + Integer.parseInt(timeParts3[0]), Integer.parseInt(timeParts3[1]), Integer.parseInt(timeParts3[2]), 0); + } + } else if (dateStr2.charAt(2) == ':') { + switch (len) { + case 5: + String[] timeParts4 = dateStr2.split(":"); + return LocalDateTime.of(0, 1, 1, Integer.parseInt(timeParts4[0]), Integer.parseInt(timeParts4[1]), 0, 0); + case 8: + String[] timeParts5 = dateStr2.split(":"); + return LocalDateTime.of(0, 1, 1, Integer.parseInt(timeParts5[0]), Integer.parseInt(timeParts5[1]), + Integer.parseInt(timeParts5[2]), 0); + default: + String[] timeParts6 = dateStr2.split(":"); + String[] secondParts2 = timeParts6[2].split("\\."); + secondParts2[1] = StringUtils.rightPad(secondParts2[1], 9, Constants.CJ_MINOR_VERSION); + return LocalDateTime.of(0, 1, 1, Integer.parseInt(timeParts6[0]), Integer.parseInt(timeParts6[1]), + Integer.parseInt(secondParts2[0]), Integer.parseInt(secondParts2[1])); + } + } else { + throw new UnsupportedOperationException(value.getClass() + ", value '" + value + "' , parse to local date time failed."); + } + } else if (StringUtils.isNumeric(value)) { + return LocalDateTime.of(Integer.parseInt(value), 1, 1, 0, 0, 0, 0); + } else { + throw new DateTimeException(value + " format error."); + } + } + public static LocalDateTime toLocalDateTime(Object value) { if (value == null || StringUtils.isBlank(value.toString())) { return null; @@ -806,9 +801,9 @@ public static boolean isHexNumber(String str) { while (true) { if (i < str.length()) { char cc = str.charAt(i); - if (cc != '0' && cc != '1' && cc != '2' && cc != '3' && cc != '4' && cc != '5' && cc != '6' && cc != '7' && cc != '8' && cc != '9' && - cc != 'A' && cc != 'B' && cc != 'C' && cc != 'D' && cc != 'E' && cc != 'F' && cc != 'a' && cc != 'b' && cc != 'c' && cc != 'd' && - cc != 'e' && cc != 'f') { + if (cc != '0' && cc != '1' && cc != '2' && cc != '3' && cc != '4' && cc != '5' && cc != '6' && cc != '7' && cc != '8' && cc != '9' + && cc != 'A' && cc != 'B' && cc != 'C' && cc != 'D' && cc != 'E' && cc != 'F' && cc != 'a' && cc != 'b' && cc != 'c' && cc != 'd' + && cc != 'e' && cc != 'f') { flag = false; break; } @@ -919,7 +914,8 @@ private static byte toByte(char src) { return 14; case 'F': return 15; + default: + throw new IllegalStateException("0-F"); } - throw new IllegalStateException("0-F"); } } diff --git a/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/sink/connector/CanalSinkFullConnector.java b/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/sink/connector/CanalSinkFullConnector.java index d7d07afcf3..36c03b156c 100644 --- a/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/sink/connector/CanalSinkFullConnector.java +++ b/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/sink/connector/CanalSinkFullConnector.java @@ -314,20 +314,20 @@ public void writeColumn(PreparedStatement ps, int index, MySQLColumnDef colType, return; case BIT: if (value instanceof Boolean) { - byte[] bArr = new byte[1]; - bArr[0] = (byte) (Boolean.TRUE.equals(value) ? 1 : 0); - ps.setBytes(index, bArr); + byte[] arrayBoolean = new byte[1]; + arrayBoolean[0] = (byte) (Boolean.TRUE.equals(value) ? 1 : 0); + ps.setBytes(index, arrayBoolean); return; } else if (value instanceof Number) { ps.setBytes(index, SqlUtils.numberToBinaryArray((Number) value)); return; } else if ((value instanceof byte[]) || value.toString().startsWith("0x") || value.toString().startsWith("0X")) { - byte[] bArr = SqlUtils.toBytes(value); - if (bArr == null || bArr.length == 0) { + byte[] arrayBoolean = SqlUtils.toBytes(value); + if (arrayBoolean == null || arrayBoolean.length == 0) { ps.setNull(index, Types.BIT); return; } else { - ps.setBytes(index, bArr); + ps.setBytes(index, arrayBoolean); return; } } else { diff --git a/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/source/connector/CanalFullProducer.java b/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/source/connector/CanalFullProducer.java index 83146de733..062bbb93a8 100644 --- a/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/source/connector/CanalFullProducer.java +++ b/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/source/connector/CanalFullProducer.java @@ -156,14 +156,14 @@ private void commitConnectRecord(List> rows) throws Interrup if (rows == null || rows.isEmpty()) { return; } - ArrayList records = new ArrayList<>(); - CanalFullRecordOffset offset = new CanalFullRecordOffset(); JobRdbFullPosition jobRdbFullPosition = new JobRdbFullPosition(); jobRdbFullPosition.setPrimaryKeyRecords(JsonUtils.toJSONString(position)); jobRdbFullPosition.setTableName(tableDefinition.getTableName()); jobRdbFullPosition.setSchema(tableDefinition.getSchemaName()); + CanalFullRecordOffset offset = new CanalFullRecordOffset(); offset.setPosition(jobRdbFullPosition); CanalFullRecordPartition partition = new CanalFullRecordPartition(); + ArrayList records = new ArrayList<>(); records.add(new ConnectRecord(partition, offset, System.currentTimeMillis(), rows)); queue.put(records); } @@ -192,21 +192,21 @@ public Object readColumn(ResultSet rs, String col, CanalMySQLType colType) throw case SMALLINT: case MEDIUMINT: case INT: - Long uLong = rs.getLong(col); - if (uLong.compareTo((long) Integer.MAX_VALUE) > 0) { - return uLong; + Long valueLong = rs.getLong(col); + if (valueLong.compareTo((long) Integer.MAX_VALUE) > 0) { + return valueLong; } - return uLong.intValue(); + return valueLong.intValue(); case BIGINT: String v = rs.getString(col); if (v == null) { return null; } - BigDecimal uBigInt = new BigDecimal(v); - if (uBigInt.compareTo(BigDecimal.valueOf(Long.MAX_VALUE)) > 0) { - return uBigInt; + BigDecimal valueBigInt = new BigDecimal(v); + if (valueBigInt.compareTo(BigDecimal.valueOf(Long.MAX_VALUE)) > 0) { + return valueBigInt; } - return uBigInt.longValue(); + return valueBigInt.longValue(); case FLOAT: case DOUBLE: case DECIMAL: diff --git a/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/source/position/CanalFullPositionMgr.java b/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/source/position/CanalFullPositionMgr.java index c5bec2cc56..a9d47b4604 100644 --- a/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/source/position/CanalFullPositionMgr.java +++ b/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/source/position/CanalFullPositionMgr.java @@ -131,7 +131,6 @@ private JobRdbFullPosition fetchTableInfo(DataSource dataSource, MySQLTableDef t position.getMinPrimaryKeyCols().put(pk, min); position.getMaxPrimaryKeyCols().put(pk, max); } - long rowCount = queryCurTableRowCount(dataSource, tableDefinition); JobRdbFullPosition jobRdbFullPosition = new JobRdbFullPosition(); if (recordPosition != null) { if (StringUtils.isNotBlank(recordPosition.getPrimaryKeyRecords())) { @@ -142,6 +141,7 @@ private JobRdbFullPosition fetchTableInfo(DataSource dataSource, MySQLTableDef t } jobRdbFullPosition.setPercent(recordPosition.getPercent()); } + long rowCount = queryCurTableRowCount(dataSource, tableDefinition); jobRdbFullPosition.setSchema(tableDefinition.getSchemaName()); jobRdbFullPosition.setTableName(tableDefinition.getTableName()); jobRdbFullPosition.setMaxCount(rowCount); @@ -151,8 +151,8 @@ private JobRdbFullPosition fetchTableInfo(DataSource dataSource, MySQLTableDef t private long queryCurTableRowCount(DataSource datasource, MySQLTableDef tableDefinition) throws SQLException { - String sql = "select `AVG_ROW_LENGTH`,`DATA_LENGTH` from information_schema.TABLES where `TABLE_SCHEMA`='" + tableDefinition.getSchemaName() + - "' and `TABLE_NAME`='" + tableDefinition.getTableName() + "'"; + String sql = "select `AVG_ROW_LENGTH`,`DATA_LENGTH` from information_schema.TABLES where `TABLE_SCHEMA`='" + tableDefinition.getSchemaName() + + "' and `TABLE_NAME`='" + tableDefinition.getTableName() + "'"; try (Statement statement = datasource.getConnection().createStatement(); ResultSet resultSet = statement.executeQuery(sql)) { long result = 0L; if (resultSet.next()) { diff --git a/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/source/table/RdbTableMgr.java b/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/source/table/RdbTableMgr.java index 2e99d06096..1aebcf6364 100644 --- a/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/source/table/RdbTableMgr.java +++ b/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/source/table/RdbTableMgr.java @@ -138,10 +138,10 @@ private Map> queryTablePrimaryKey(String schema, List> queryColumns(String schema, List tables) throws SQLException { String prepareTables = SqlUtils.genPrepareSqlOfInClause(tables.size()); - String sql = "select TABLE_SCHEMA,TABLE_NAME,COLUMN_NAME,IS_NULLABLE,DATA_TYPE,CHARACTER_MAXIMUM_LENGTH," + - "CHARACTER_OCTET_LENGTH,NUMERIC_SCALE,NUMERIC_PRECISION,DATETIME_PRECISION,CHARACTER_SET_NAME," + - "COLLATION_NAME,COLUMN_TYPE,COLUMN_DEFAULT,COLUMN_COMMENT,ORDINAL_POSITION,EXTRA from " + - "INFORMATION_SCHEMA.COLUMNS where TABLE_SCHEMA = ? and TABLE_NAME in " + prepareTables + " order by " + "ORDINAL_POSITION asc"; + String sql = "select TABLE_SCHEMA,TABLE_NAME,COLUMN_NAME,IS_NULLABLE,DATA_TYPE,CHARACTER_MAXIMUM_LENGTH," + + "CHARACTER_OCTET_LENGTH,NUMERIC_SCALE,NUMERIC_PRECISION,DATETIME_PRECISION,CHARACTER_SET_NAME," + + "COLLATION_NAME,COLUMN_TYPE,COLUMN_DEFAULT,COLUMN_COMMENT,ORDINAL_POSITION,EXTRA from " + + "INFORMATION_SCHEMA.COLUMNS where TABLE_SCHEMA = ? and TABLE_NAME in " + prepareTables + " order by " + "ORDINAL_POSITION asc"; Map> cols = new LinkedHashMap<>(); try (PreparedStatement statement = dataSource.getConnection().prepareStatement(sql)) { statement.setString(1, schema); @@ -151,14 +151,14 @@ private Map> queryColumns(String schema, List { if (v == null) { v = new LinkedList<>();