Skip to content

Commit d66a15f

Browse files
lulu2panpanagapple
lulu2panpan
and
agapple
authored
migrate sql parser from druid to polardbx-parser (#5097)
* migrate parser from druid to polardbx-parser * migrate parser from druid to polardbx-parser(with version 5.4.19) * fix bug : full ddl sqls should not assign to extra field but data field * fix bug : full ddl sqls should not assign to extra field but data field --------- Co-authored-by: agapple <[email protected]>
1 parent aa05d8e commit d66a15f

File tree

12 files changed

+99
-74
lines changed

12 files changed

+99
-74
lines changed

‎common/pom.xml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -87,6 +87,11 @@
8787
<groupId>com.google.code.findbugs</groupId>
8888
<artifactId>jsr305</artifactId>
8989
</dependency>
90+
<dependency>
91+
<groupId>com.alibaba</groupId>
92+
<artifactId>druid</artifactId>
93+
<optional>true</optional>
94+
</dependency>
9095
<dependency>
9196
<groupId>org.springframework</groupId>
9297
<artifactId>spring-test</artifactId>

‎common/src/main/java/com/alibaba/otter/canal/common/utils/CommonUtils.java‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,7 @@
11
package com.alibaba.otter.canal.common.utils;
22

3+
import com.alibaba.druid.filter.config.ConfigTools;
4+
35
import java.io.File;
46

57
/**
@@ -50,4 +52,8 @@ public static boolean deleteDir(File dirFile) {
5052

5153
return dirFile.delete();
5254
}
55+
56+
public static String decryptDruidPassword(String publicKeyText, String cipherText) throws Exception {
57+
return ConfigTools.decrypt(publicKeyText, cipherText);
58+
}
5359
}

‎parse/pom.xml‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -46,8 +46,8 @@
4646
<version>${project.version}</version>
4747
</dependency>
4848
<dependency>
49-
<groupId>com.alibaba</groupId>
50-
<artifactId>druid</artifactId>
49+
<groupId>com.alibaba.polardbx</groupId>
50+
<artifactId>polardbx-parser</artifactId>
5151
</dependency>
5252
<dependency>
5353
<groupId>mysql</groupId>

‎parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/dbsync/TableMetaCache.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -258,7 +258,7 @@ private String getFullName(String schema, String table) {
258258
.append('`')
259259
.append('.')
260260
.append('`')
261-
.append(table)
261+
.append(StringUtils.replace(table,"`","``"))
262262
.append('`')
263263
.toString();
264264
}

‎parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/ddl/DruidDdlParser.java‎

Lines changed: 31 additions & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -4,39 +4,38 @@
44
import java.util.Arrays;
55
import java.util.List;
66

7+
import com.alibaba.polardbx.druid.sql.SQLUtils;
8+
import com.alibaba.polardbx.druid.sql.ast.SQLExpr;
9+
import com.alibaba.polardbx.druid.sql.ast.SQLStatement;
10+
import com.alibaba.polardbx.druid.sql.ast.expr.SQLIdentifierExpr;
11+
import com.alibaba.polardbx.druid.sql.ast.expr.SQLPropertyExpr;
12+
import com.alibaba.polardbx.druid.sql.ast.statement.SQLAlterTableAddConstraint;
13+
import com.alibaba.polardbx.druid.sql.ast.statement.SQLAlterTableAddIndex;
14+
import com.alibaba.polardbx.druid.sql.ast.statement.SQLAlterTableDropConstraint;
15+
import com.alibaba.polardbx.druid.sql.ast.statement.SQLAlterTableDropIndex;
16+
import com.alibaba.polardbx.druid.sql.ast.statement.SQLAlterTableDropKey;
17+
import com.alibaba.polardbx.druid.sql.ast.statement.SQLAlterTableItem;
18+
import com.alibaba.polardbx.druid.sql.ast.statement.SQLAlterTableRename;
19+
import com.alibaba.polardbx.druid.sql.ast.statement.SQLAlterTableStatement;
20+
import com.alibaba.polardbx.druid.sql.ast.statement.SQLConstraint;
21+
import com.alibaba.polardbx.druid.sql.ast.statement.SQLCreateDatabaseStatement;
22+
import com.alibaba.polardbx.druid.sql.ast.statement.SQLCreateIndexStatement;
23+
import com.alibaba.polardbx.druid.sql.ast.statement.SQLCreateTableStatement;
24+
import com.alibaba.polardbx.druid.sql.ast.statement.SQLDeleteStatement;
25+
import com.alibaba.polardbx.druid.sql.ast.statement.SQLDropDatabaseStatement;
26+
import com.alibaba.polardbx.druid.sql.ast.statement.SQLDropIndexStatement;
27+
import com.alibaba.polardbx.druid.sql.ast.statement.SQLDropTableStatement;
28+
import com.alibaba.polardbx.druid.sql.ast.statement.SQLExprTableSource;
29+
import com.alibaba.polardbx.druid.sql.ast.statement.SQLInsertStatement;
30+
import com.alibaba.polardbx.druid.sql.ast.statement.SQLTableSource;
31+
import com.alibaba.polardbx.druid.sql.ast.statement.SQLTruncateStatement;
32+
import com.alibaba.polardbx.druid.sql.ast.statement.SQLUnique;
33+
import com.alibaba.polardbx.druid.sql.ast.statement.SQLUpdateStatement;
34+
import com.alibaba.polardbx.druid.sql.dialect.mysql.ast.statement.MySqlRenameTableStatement;
35+
import com.alibaba.polardbx.druid.sql.parser.ParserException;
36+
import com.alibaba.polardbx.druid.util.JdbcConstants;
737
import org.apache.commons.lang.StringUtils;
838

9-
import com.alibaba.druid.sql.SQLUtils;
10-
import com.alibaba.druid.sql.ast.SQLExpr;
11-
import com.alibaba.druid.sql.ast.SQLStatement;
12-
import com.alibaba.druid.sql.ast.expr.SQLIdentifierExpr;
13-
import com.alibaba.druid.sql.ast.expr.SQLPropertyExpr;
14-
import com.alibaba.druid.sql.ast.statement.SQLAlterTableAddConstraint;
15-
import com.alibaba.druid.sql.ast.statement.SQLAlterTableAddIndex;
16-
import com.alibaba.druid.sql.ast.statement.SQLAlterTableDropConstraint;
17-
import com.alibaba.druid.sql.ast.statement.SQLAlterTableDropIndex;
18-
import com.alibaba.druid.sql.ast.statement.SQLAlterTableDropKey;
19-
import com.alibaba.druid.sql.ast.statement.SQLAlterTableItem;
20-
import com.alibaba.druid.sql.ast.statement.SQLAlterTableRename;
21-
import com.alibaba.druid.sql.ast.statement.SQLAlterTableStatement;
22-
import com.alibaba.druid.sql.ast.statement.SQLConstraint;
23-
import com.alibaba.druid.sql.ast.statement.SQLCreateDatabaseStatement;
24-
import com.alibaba.druid.sql.ast.statement.SQLCreateIndexStatement;
25-
import com.alibaba.druid.sql.ast.statement.SQLCreateTableStatement;
26-
import com.alibaba.druid.sql.ast.statement.SQLDeleteStatement;
27-
import com.alibaba.druid.sql.ast.statement.SQLDropDatabaseStatement;
28-
import com.alibaba.druid.sql.ast.statement.SQLDropIndexStatement;
29-
import com.alibaba.druid.sql.ast.statement.SQLDropTableStatement;
30-
import com.alibaba.druid.sql.ast.statement.SQLExprTableSource;
31-
import com.alibaba.druid.sql.ast.statement.SQLInsertStatement;
32-
import com.alibaba.druid.sql.ast.statement.SQLTableSource;
33-
import com.alibaba.druid.sql.ast.statement.SQLTruncateStatement;
34-
import com.alibaba.druid.sql.ast.statement.SQLUnique;
35-
import com.alibaba.druid.sql.ast.statement.SQLUpdateStatement;
36-
import com.alibaba.druid.sql.dialect.mysql.ast.statement.MySqlRenameTableStatement;
37-
import com.alibaba.druid.sql.dialect.mysql.ast.statement.MySqlRenameTableStatement.Item;
38-
import com.alibaba.druid.sql.parser.ParserException;
39-
import com.alibaba.druid.util.JdbcConstants;
4039
import com.alibaba.otter.canal.protocol.CanalEntry.EventType;
4140

4241
/**
@@ -141,7 +140,7 @@ public static List<DdlResult> parse(String queryString, String schmeaName) {
141140
}
142141
} else if (statement instanceof MySqlRenameTableStatement) {
143142
MySqlRenameTableStatement rename = (MySqlRenameTableStatement) statement;
144-
for (Item item : rename.getItems()) {
143+
for (MySqlRenameTableStatement.Item item : rename.getItems()) {
145144
DdlResult ddlResult = new DdlResult();
146145
processName(ddlResult, schmeaName, item.getName(), true);
147146
processName(ddlResult, schmeaName, item.getTo(), false);

‎parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/DatabaseTableMeta.java‎

Lines changed: 6 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -13,14 +13,15 @@
1313
import java.util.concurrent.locks.ReadWriteLock;
1414
import java.util.concurrent.locks.ReentrantReadWriteLock;
1515
import java.util.regex.Pattern;
16-
16+
import com.alibaba.otter.canal.parse.driver.mysql.packets.server.FieldPacket;
17+
import com.alibaba.polardbx.druid.sql.repository.Schema;
18+
import org.apache.commons.beanutils.BeanUtils;
1719
import org.apache.commons.lang.ObjectUtils;
1820
import org.apache.commons.lang.StringUtils;
1921
import org.slf4j.Logger;
2022
import org.slf4j.LoggerFactory;
2123
import org.slf4j.MDC;
2224

23-
import com.alibaba.druid.sql.repository.Schema;
2425
import com.alibaba.fastjson2.JSON;
2526
import com.alibaba.fastjson2.JSONObject;
2627
import com.alibaba.otter.canal.filter.CanalEventFilter;
@@ -227,7 +228,7 @@ private boolean dumpTableMeta(MysqlConnection connection, final CanalEventFilter
227228

228229
StringBuilder sql = new StringBuilder();
229230
for (String table : tables) {
230-
sql.append("show create table `" + schema + "`.`" + table + "`;");
231+
sql.append("show create table `" + schema + "`.`" + StringUtils.replace(table,"`","``") + "`;");
231232
}
232233

233234
List<ResultSetPacket> packets = connection.queryMulti(sql.toString());
@@ -363,7 +364,7 @@ private boolean applySnapshotToDB(EntryPosition position, boolean init) {
363364
snapshotDO.setBinlogOffest(position.getPosition());
364365
snapshotDO.setBinlogMasterId(String.valueOf(position.getServerId()));
365366
snapshotDO.setBinlogTimestamp(position.getTimestamp());
366-
snapshotDO.setExtra(JSON.toJSONString(schemaDdls));
367+
snapshotDO.setData(JSON.toJSONString(schemaDdls));
367368
try {
368369
metaSnapshotDAO.insert(snapshotDO);
369370
} catch (Throwable e) {
@@ -508,7 +509,7 @@ private String structureSchema(String schema) {
508509

509510
private String getFullName(String schema, String table) {
510511
StringBuilder builder = new StringBuilder();
511-
return builder.append(structureSchema(schema)).append('.').append('`').append(table).append('`').toString();
512+
return builder.append(structureSchema(schema)).append('.').append('`').append(StringUtils.replace(table,"`","``")).append('`').toString();
512513
}
513514

514515
public static boolean compareTableMeta(TableMeta source, TableMeta target) {

‎parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/MemoryTableMeta.java‎

Lines changed: 31 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -3,27 +3,38 @@
33
import java.util.*;
44
import java.util.concurrent.ConcurrentHashMap;
55

6+
import com.alibaba.polardbx.druid.DbType;
7+
import com.alibaba.polardbx.druid.sql.SQLUtils;
8+
import com.alibaba.polardbx.druid.sql.ast.SQLDataType;
9+
import com.alibaba.polardbx.druid.sql.ast.SQLDataTypeImpl;
10+
import com.alibaba.polardbx.druid.sql.ast.SQLExpr;
11+
import com.alibaba.polardbx.druid.sql.ast.SQLStatement;
12+
import com.alibaba.polardbx.druid.sql.ast.expr.SQLCharExpr;
13+
import com.alibaba.polardbx.druid.sql.ast.expr.SQLIdentifierExpr;
14+
import com.alibaba.polardbx.druid.sql.ast.expr.SQLMethodInvokeExpr;
15+
import com.alibaba.polardbx.druid.sql.ast.expr.SQLNullExpr;
16+
import com.alibaba.polardbx.druid.sql.ast.expr.SQLPropertyExpr;
17+
import com.alibaba.polardbx.druid.sql.ast.statement.SQLColumnConstraint;
18+
import com.alibaba.polardbx.druid.sql.ast.statement.SQLColumnDefinition;
19+
import com.alibaba.polardbx.druid.sql.ast.statement.SQLColumnPrimaryKey;
20+
import com.alibaba.polardbx.druid.sql.ast.statement.SQLColumnUniqueKey;
21+
import com.alibaba.polardbx.druid.sql.ast.statement.SQLCreateTableStatement;
22+
import com.alibaba.polardbx.druid.sql.ast.statement.SQLNotNullConstraint;
23+
import com.alibaba.polardbx.druid.sql.ast.statement.SQLNullConstraint;
24+
import com.alibaba.polardbx.druid.sql.ast.statement.SQLSelectOrderByItem;
25+
import com.alibaba.polardbx.druid.sql.ast.statement.SQLTableElement;
26+
import com.alibaba.polardbx.druid.sql.dialect.mysql.ast.MySqlPrimaryKey;
27+
import com.alibaba.polardbx.druid.sql.dialect.mysql.ast.MySqlUnique;
28+
import com.alibaba.polardbx.druid.sql.dialect.mysql.ast.expr.MySqlOrderingExpr;
29+
import com.alibaba.polardbx.druid.sql.repository.Schema;
30+
import com.alibaba.polardbx.druid.sql.repository.SchemaObject;
31+
import com.alibaba.polardbx.druid.sql.repository.SchemaRepository;
32+
import com.alibaba.polardbx.druid.sql.visitor.SQLASTOutputVisitor;
33+
import com.alibaba.polardbx.druid.sql.visitor.VisitorFeature;
34+
import com.alibaba.polardbx.druid.util.JdbcConstants;
635
import org.apache.commons.lang.StringUtils;
736
import org.slf4j.Logger;
837
import org.slf4j.LoggerFactory;
9-
10-
import com.alibaba.druid.DbType;
11-
import com.alibaba.druid.sql.SQLUtils;
12-
import com.alibaba.druid.sql.ast.SQLDataType;
13-
import com.alibaba.druid.sql.ast.SQLDataTypeImpl;
14-
import com.alibaba.druid.sql.ast.SQLExpr;
15-
import com.alibaba.druid.sql.ast.SQLStatement;
16-
import com.alibaba.druid.sql.ast.expr.*;
17-
import com.alibaba.druid.sql.ast.statement.*;
18-
import com.alibaba.druid.sql.dialect.mysql.ast.MySqlPrimaryKey;
19-
import com.alibaba.druid.sql.dialect.mysql.ast.MySqlUnique;
20-
import com.alibaba.druid.sql.dialect.mysql.ast.expr.MySqlOrderingExpr;
21-
import com.alibaba.druid.sql.repository.Schema;
22-
import com.alibaba.druid.sql.repository.SchemaObject;
23-
import com.alibaba.druid.sql.repository.SchemaRepository;
24-
import com.alibaba.druid.sql.visitor.SQLASTOutputVisitor;
25-
import com.alibaba.druid.sql.visitor.VisitorFeature;
26-
import com.alibaba.druid.util.JdbcConstants;
2738
import com.alibaba.otter.canal.parse.inbound.TableMeta;
2839
import com.alibaba.otter.canal.parse.inbound.TableMeta.FieldMeta;
2940
import com.alibaba.otter.canal.parse.inbound.mysql.ddl.DruidDdlParser;
@@ -39,9 +50,10 @@ public class MemoryTableMeta implements TableMetaTSDB {
3950

4051
private Logger logger = LoggerFactory.getLogger(MemoryTableMeta.class);
4152
private Map<List<String>, TableMeta> tableMetas = new ConcurrentHashMap<>();
42-
private SchemaRepository repository = new SchemaRepository(JdbcConstants.MYSQL);
53+
private SchemaRepository repository;
4354

4455
public MemoryTableMeta(){
56+
repository = new SchemaRepository(JdbcConstants.MYSQL);
4557
}
4658

4759
@Override

‎parse/src/main/java/com/alibaba/otter/canal/parse/support/AuthenticationInfo.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@
22

33
import java.net.InetSocketAddress;
44

5-
import com.alibaba.druid.filter.config.ConfigTools;
5+
import com.alibaba.otter.canal.common.utils.CommonUtils;
66
import org.apache.commons.lang.builder.ToStringBuilder;
77
import org.apache.commons.lang.builder.ToStringStyle;
88

@@ -25,7 +25,7 @@ public class AuthenticationInfo {
2525

2626
public void initPwd() throws Exception{
2727
if (enableDruid) {
28-
this.password = ConfigTools.decrypt(pwdPublicKey, password);
28+
this.password = CommonUtils.decryptDruidPassword(pwdPublicKey, password);
2929
}
3030
}
3131

‎parse/src/test/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/FastsqlSchemaTest.java‎

Lines changed: 8 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -3,14 +3,14 @@
33
import org.junit.Assert;
44
import org.junit.Test;
55

6-
import com.alibaba.druid.DbType;
7-
import com.alibaba.druid.sql.SQLUtils;
8-
import com.alibaba.druid.sql.repository.Schema;
9-
import com.alibaba.druid.sql.repository.SchemaObject;
10-
import com.alibaba.druid.sql.repository.SchemaRepository;
11-
import com.alibaba.druid.sql.visitor.SQLASTOutputVisitor;
12-
import com.alibaba.druid.sql.visitor.VisitorFeature;
13-
import com.alibaba.druid.util.JdbcConstants;
6+
import com.alibaba.polardbx.druid.DbType;
7+
import com.alibaba.polardbx.druid.sql.SQLUtils;
8+
import com.alibaba.polardbx.druid.sql.repository.Schema;
9+
import com.alibaba.polardbx.druid.sql.repository.SchemaObject;
10+
import com.alibaba.polardbx.druid.sql.repository.SchemaRepository;
11+
import com.alibaba.polardbx.druid.sql.visitor.SQLASTOutputVisitor;
12+
import com.alibaba.polardbx.druid.sql.visitor.VisitorFeature;
13+
import com.alibaba.polardbx.druid.util.JdbcConstants;
1414

1515
public class FastsqlSchemaTest {
1616

‎parse/src/test/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/MemoryTableMeta_DDL_Test.java‎

Lines changed: 1 addition & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -6,9 +6,6 @@
66
import java.util.ArrayList;
77
import java.util.List;
88

9-
import com.alibaba.druid.sql.repository.SchemaObject;
10-
import com.alibaba.druid.sql.repository.SchemaRepository;
11-
import com.alibaba.druid.util.JdbcConstants;
129
import org.apache.commons.io.IOUtils;
1310
import org.apache.commons.lang.StringUtils;
1411
import org.junit.Assert;
@@ -17,7 +14,7 @@
1714
import org.springframework.test.context.ContextConfiguration;
1815
import org.springframework.test.context.junit4.SpringJUnit4ClassRunner;
1916

20-
import com.alibaba.druid.sql.repository.Schema;
17+
import com.alibaba.polardbx.druid.sql.repository.Schema;
2118
import com.alibaba.otter.canal.parse.inbound.TableMeta;
2219

2320
/**

0 commit comments

Comments
 (0)