diff --git a/connectors-common/connector-core/src/main/java/io/tapdata/kit/DbKit.java b/connectors-common/connector-core/src/main/java/io/tapdata/kit/DbKit.java index 1e35ad34a..ef2835124 100644 --- a/connectors-common/connector-core/src/main/java/io/tapdata/kit/DbKit.java +++ b/connectors-common/connector-core/src/main/java/io/tapdata/kit/DbKit.java @@ -157,7 +157,11 @@ public static boolean ignoreCreateIndex(TapIndex exists, TapIndex created) { } public static String buildIndexName(String table) { - return "TAPIDX_" + table.substring(Math.max(table.length() - 10, 0)) + UUID.randomUUID().toString().replaceAll("-", "").substring(20); + return "IDX_" + table.substring(Math.max(table.length() - 10, 0)) + UUID.randomUUID().toString().replaceAll("-", "").substring(20); + } + + public static String buildForeignKeyName(String table) { + return "FK_" + table.substring(Math.max(table.length() - 10, 0)) + UUID.randomUUID().toString().replaceAll("-", "").substring(20); } public static List> splitToPieces(List data, int eachPieceSize) { diff --git a/connectors-common/connector-core/src/main/java/io/tapdata/kit/StringKit.java b/connectors-common/connector-core/src/main/java/io/tapdata/kit/StringKit.java index d252c039e..dbe9c7fec 100644 --- a/connectors-common/connector-core/src/main/java/io/tapdata/kit/StringKit.java +++ b/connectors-common/connector-core/src/main/java/io/tapdata/kit/StringKit.java @@ -6,6 +6,7 @@ import java.util.Collection; import java.util.Iterator; import java.util.List; +import java.util.regex.Matcher; import java.util.regex.Pattern; import java.util.stream.Collectors; @@ -190,7 +191,7 @@ public static String removeHeadTail(String str, String remove, Boolean upperCase if (EmptyKit.isBlank(str) || EmptyKit.isBlank(remove)) { return str; } - if (str.startsWith(remove) && str.endsWith(remove) && str.length() > 2 * remove.length()) { + if (str.startsWith(remove) && str.endsWith(remove) && str.length() >= 2 * remove.length()) { return str.substring(remove.length(), str.length() - remove.length()); } if (EmptyKit.isNull(upperCase)) { @@ -342,4 +343,27 @@ public static String trimTailBlank(Object str) { if (null == str) return null; return ("_" + str).trim().substring(1); } + + public static String escape(String name, String escapes) { + String res = name; + for (int i = 0; i < escapes.length(); i++) { + char escape = escapes.charAt(i); + res = escape(res, escape); + } + return res; + } + + public static String escape(String name, char escape) { + return name.replace(escape + "", "" + escape + escape); + } + + private static final Pattern REGEX_SPECIAL_CHARS = Pattern.compile("[\\\\^$|*+?.,()\\[\\]{}]"); + + public static String escapeRegex(String input) { + if (input == null || input.isEmpty()) { + return input; + } + Matcher matcher = REGEX_SPECIAL_CHARS.matcher(input); + return matcher.replaceAll("\\\\$0"); + } } diff --git a/connectors-common/debezium-bucket/debezium-connector-mysql/src/main/java/io/debezium/connector/mysql/MySqlSnapshotChangeEventSource.java b/connectors-common/debezium-bucket/debezium-connector-mysql/src/main/java/io/debezium/connector/mysql/MySqlSnapshotChangeEventSource.java index fcf847373..ba071b2fb 100644 --- a/connectors-common/debezium-bucket/debezium-connector-mysql/src/main/java/io/debezium/connector/mysql/MySqlSnapshotChangeEventSource.java +++ b/connectors-common/debezium-bucket/debezium-connector-mysql/src/main/java/io/debezium/connector/mysql/MySqlSnapshotChangeEventSource.java @@ -583,7 +583,7 @@ private void tableUnlock() throws SQLException { } private String quote(String dbOrTableName) { - return "`" + dbOrTableName + "`"; + return "`" + dbOrTableName.replace("`", "``") + "`"; } private String quote(TableId id) { diff --git a/connectors-common/debezium-bucket/debezium-connector-postgres/src/main/java/io/debezium/connector/postgresql/PostgresChangeRecordEmitter.java b/connectors-common/debezium-bucket/debezium-connector-postgres/src/main/java/io/debezium/connector/postgresql/PostgresChangeRecordEmitter.java index d786c5388..e2310d6d2 100644 --- a/connectors-common/debezium-bucket/debezium-connector-postgres/src/main/java/io/debezium/connector/postgresql/PostgresChangeRecordEmitter.java +++ b/connectors-common/debezium-bucket/debezium-connector-postgres/src/main/java/io/debezium/connector/postgresql/PostgresChangeRecordEmitter.java @@ -63,7 +63,7 @@ public PostgresChangeRecordEmitter(OffsetContext offset, Clock clock, PostgresCo this.connectorConfig = connectorConfig; this.connection = connection; - this.tableId = PostgresSchema.parse(message.getTable()); + this.tableId = message.getTableId(); this.unchangedToastColumnMarkerMissing = !connectorConfig.plugin().hasUnchangedToastColumnMarker(); this.nullToastedValuesMissingFromOld = !connectorConfig.plugin().sendsNullToastedValuesInOld(); Objects.requireNonNull(tableId); diff --git a/connectors-common/debezium-bucket/debezium-connector-postgres/src/main/java/io/debezium/connector/postgresql/PostgresSchema.java b/connectors-common/debezium-bucket/debezium-connector-postgres/src/main/java/io/debezium/connector/postgresql/PostgresSchema.java index 57efed0d0..3376e9c6a 100644 --- a/connectors-common/debezium-bucket/debezium-connector-postgres/src/main/java/io/debezium/connector/postgresql/PostgresSchema.java +++ b/connectors-common/debezium-bucket/debezium-connector-postgres/src/main/java/io/debezium/connector/postgresql/PostgresSchema.java @@ -213,7 +213,7 @@ private void refreshToastableColumnsMap(PostgresConnection connection, TableId t tableIdToToastableColumns.put(tableId, Collections.unmodifiableList(toastableColumns)); } - protected static TableId parse(String table) { + public static TableId parse(String table) { TableId tableId = TableId.parse(table, false); if (tableId == null) { return null; diff --git a/connectors-common/debezium-bucket/debezium-connector-postgres/src/main/java/io/debezium/connector/postgresql/PostgresStreamingChangeEventSource.java b/connectors-common/debezium-bucket/debezium-connector-postgres/src/main/java/io/debezium/connector/postgresql/PostgresStreamingChangeEventSource.java index c192e411e..38d7a6dda 100644 --- a/connectors-common/debezium-bucket/debezium-connector-postgres/src/main/java/io/debezium/connector/postgresql/PostgresStreamingChangeEventSource.java +++ b/connectors-common/debezium-bucket/debezium-connector-postgres/src/main/java/io/debezium/connector/postgresql/PostgresStreamingChangeEventSource.java @@ -5,23 +5,8 @@ */ package io.debezium.connector.postgresql; -import java.sql.SQLException; -import java.util.Map; -import java.util.Objects; -import java.util.concurrent.Executors; -import java.util.concurrent.atomic.AtomicReference; - -import org.apache.kafka.connect.errors.ConnectException; -import org.postgresql.core.BaseConnection; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import io.debezium.connector.postgresql.connection.Lsn; -import io.debezium.connector.postgresql.connection.PostgresConnection; -import io.debezium.connector.postgresql.connection.ReplicationConnection; +import io.debezium.connector.postgresql.connection.*; import io.debezium.connector.postgresql.connection.ReplicationMessage.Operation; -import io.debezium.connector.postgresql.connection.ReplicationStream; -import io.debezium.connector.postgresql.connection.WalPositionLocator; import io.debezium.connector.postgresql.spi.Snapshotter; import io.debezium.heartbeat.Heartbeat; import io.debezium.pipeline.ErrorHandler; @@ -30,9 +15,18 @@ import io.debezium.relational.TableId; import io.debezium.util.Clock; import io.debezium.util.DelayStrategy; +import org.apache.kafka.connect.errors.ConnectException; +import org.postgresql.core.BaseConnection; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.sql.SQLException; +import java.util.Map; +import java.util.Objects; +import java.util.concurrent.Executors; +import java.util.concurrent.atomic.AtomicReference; /** - * * @author Horia Chiorean (hchiorea@redhat.com), Jiri Pechanec */ public class PostgresStreamingChangeEventSource implements StreamingChangeEventSource { @@ -104,8 +98,7 @@ public void execute(ChangeEventSourceContext context) throws InterruptedExceptio LOGGER.info("Retrieved latest position from stored offset '{}'", lsn); walPosition = new WalPositionLocator(offsetContext.lastCommitLsn(), lsn); replicationStream.compareAndSet(null, replicationConnection.startStreaming(lsn, walPosition)); - } - else { + } else { LOGGER.info("No previous LSN found in Kafka, streaming from the latest xlogpos or flushed LSN..."); walPosition = new WalPositionLocator(); replicationStream.compareAndSet(null, replicationConnection.startStreaming(walPosition)); @@ -133,8 +126,7 @@ public void execute(ChangeEventSourceContext context) throws InterruptedExceptio if (!isInPreSnapshotCatchUpStreaming()) { connection.commit(); } - } - catch (Exception e) { + } catch (Exception e) { LOGGER.info("Commit failed while preparing for reconnect", e); } walPosition.enableFiltering(); @@ -145,11 +137,9 @@ public void execute(ChangeEventSourceContext context) throws InterruptedExceptio stream.startKeepAlive(Executors.newSingleThreadExecutor()); } processMessages(context, stream); - } - catch (Throwable e) { + } catch (Throwable e) { errorHandler.setProducerThrowable(e); - } - finally { + } finally { if (replicationConnection != null) { LOGGER.debug("stopping streaming..."); // stop the keep alive thread, this also shuts down the @@ -166,8 +156,7 @@ public void execute(ChangeEventSourceContext context) throws InterruptedExceptio connection.commit(); } replicationConnection.close(); - } - catch (Exception e) { + } catch (Exception e) { LOGGER.debug("Exception while closing the connection", e); } replicationStream.set(null); @@ -205,8 +194,7 @@ private void processMessages(ChangeEventSourceContext context, final Replication taskContext.getSlotXmin(connection)); if (message.getOperation() == Operation.BEGIN) { dispatcher.dispatchTransactionStartedEvent(Long.toString(message.getTransactionId()), offsetContext); - } - else if (message.getOperation() == Operation.COMMIT) { + } else if (message.getOperation() == Operation.COMMIT) { commitMessage(lsn); dispatcher.dispatchTransactionCommittedEvent(offsetContext); } @@ -216,7 +204,7 @@ else if (message.getOperation() == Operation.COMMIT) { else { TableId tableId = null; if (message.getOperation() != Operation.NOOP) { - tableId = PostgresSchema.parse(message.getTable()); + tableId = message.getTableId(); Objects.requireNonNull(tableId); } @@ -239,8 +227,7 @@ else if (message.getOperation() == Operation.COMMIT) { if (receivedMessage) { noMessageIterations = 0; - } - else { + } else { if (offsetContext.hasCompletelyProcessedPosition()) { dispatcher.dispatchHeartbeatEvent(offsetContext); } @@ -275,8 +262,7 @@ private void searchWalPosition(ChangeEventSourceContext context, final Replicati if (receivedMessage) { noMessageIterations = 0; - } - else { + } else { noMessageIterations++; if (noMessageIterations >= THROTTLE_NO_MESSAGE_BEFORE_PAUSE) { noMessageIterations = 0; @@ -307,23 +293,21 @@ private void commitMessage(final Lsn lsn) throws SQLException, InterruptedExcept * The purpose of this method is to detect this situation and log a warning * every {@link #GROWING_WAL_WARNING_LOG_INTERVAL} filtered events. * - * @param dispatched - * Whether an event was sent to the broker or not + * @param dispatched Whether an event was sent to the broker or not */ private void maybeWarnAboutGrowingWalBacklog(boolean dispatched) { if (dispatched) { numberOfEventsSinceLastEventSentOrWalGrowingWarning = 0; - } - else { + } else { numberOfEventsSinceLastEventSentOrWalGrowingWarning++; } if (numberOfEventsSinceLastEventSentOrWalGrowingWarning > GROWING_WAL_WARNING_LOG_INTERVAL && !dispatcher.heartbeatsEnabled()) { LOGGER.warn("Received {} events which were all filtered out, so no offset could be committed. " - + "This prevents the replication slot from acknowledging the processed WAL offsets, " - + "causing a growing backlog of non-removeable WAL segments on the database server. " - + "Consider to either adjust your filter configuration or enable heartbeat events " - + "(via the {} option) to avoid this situation.", + + "This prevents the replication slot from acknowledging the processed WAL offsets, " + + "causing a growing backlog of non-removeable WAL segments on the database server. " + + "Consider to either adjust your filter configuration or enable heartbeat events " + + "(via the {} option) to avoid this situation.", numberOfEventsSinceLastEventSentOrWalGrowingWarning, Heartbeat.HEARTBEAT_INTERVAL_PROPERTY_NAME); numberOfEventsSinceLastEventSentOrWalGrowingWarning = 0; @@ -344,12 +328,10 @@ public void commitOffset(Map offset) { } // tell the server the point up to which we've processed data, so it can be free to recycle WAL segments replicationStream.flushLsn(lsn); - } - else { + } else { LOGGER.debug("Streaming has already stopped, ignoring commit callback..."); } - } - catch (SQLException e) { + } catch (SQLException e) { throw new ConnectException(e); } } @@ -358,7 +340,7 @@ public void commitOffset(Map offset) { * Returns whether the current streaming phase is running a catch up streaming * phase that runs before a snapshot. This is useful for transaction * management. - * + *

* During pre-snapshot catch up streaming, we open the snapshot transaction * early and hold the transaction open throughout the pre snapshot catch up * streaming phase so that we know where to stop streaming and can start the diff --git a/connectors-common/debezium-bucket/debezium-connector-postgres/src/main/java/io/debezium/connector/postgresql/connection/ReplicationMessage.java b/connectors-common/debezium-bucket/debezium-connector-postgres/src/main/java/io/debezium/connector/postgresql/connection/ReplicationMessage.java index fc6cb6a47..bd1cc153c 100644 --- a/connectors-common/debezium-bucket/debezium-connector-postgres/src/main/java/io/debezium/connector/postgresql/connection/ReplicationMessage.java +++ b/connectors-common/debezium-bucket/debezium-connector-postgres/src/main/java/io/debezium/connector/postgresql/connection/ReplicationMessage.java @@ -12,6 +12,8 @@ import java.time.OffsetTime; import java.util.List; +import io.debezium.connector.postgresql.PostgresSchema; +import io.debezium.relational.TableId; import org.postgresql.geometric.PGbox; import org.postgresql.geometric.PGcircle; import org.postgresql.geometric.PGline; @@ -156,6 +158,10 @@ public interface ColumnValue { */ public String getTable(); + default TableId getTableId() { + return PostgresSchema.parse(getTable()); + } + /** * @return Set of original values of table columns, null for INSERT */ diff --git a/connectors-common/debezium-bucket/debezium-connector-postgres/src/main/java/io/debezium/connector/postgresql/connection/pgoutput/PgOutputMessageDecoder.java b/connectors-common/debezium-bucket/debezium-connector-postgres/src/main/java/io/debezium/connector/postgresql/connection/pgoutput/PgOutputMessageDecoder.java index dd9a95fef..f0c17a06b 100644 --- a/connectors-common/debezium-bucket/debezium-connector-postgres/src/main/java/io/debezium/connector/postgresql/connection/pgoutput/PgOutputMessageDecoder.java +++ b/connectors-common/debezium-bucket/debezium-connector-postgres/src/main/java/io/debezium/connector/postgresql/connection/pgoutput/PgOutputMessageDecoder.java @@ -5,6 +5,7 @@ */ package io.debezium.connector.postgresql.connection.pgoutput; +import java.io.ByteArrayOutputStream; import java.nio.ByteBuffer; import java.nio.charset.Charset; import java.sql.DatabaseMetaData; @@ -250,8 +251,8 @@ private void handleCommitMessage(ByteBuffer buffer, ReplicationMessageProcessor */ private void handleRelationMessage(ByteBuffer buffer, TypeRegistry typeRegistry) throws SQLException { int relationId = buffer.getInt(); - String schemaName = readString(buffer); - String tableName = readString(buffer); + String schemaName = readStringV2(buffer); + String tableName = readStringV2(buffer); int replicaIdentityId = buffer.get(); short columnCount = buffer.getShort(); @@ -276,7 +277,7 @@ private void handleRelationMessage(ByteBuffer buffer, TypeRegistry typeRegistry) Set columnNames = new HashSet<>(); for (short i = 0; i < columnCount; ++i) { byte flags = buffer.get(); - String columnName = Strings.unquoteIdentifierPart(readString(buffer)); + String columnName = Strings.unquoteIdentifierPart(readStringV2(buffer)); int columnType = buffer.getInt(); int attypmod = buffer.getInt(); @@ -379,7 +380,7 @@ private void decodeInsert(ByteBuffer buffer, TypeRegistry typeRegistry, Replicat List columns = resolveColumnsFromStreamTupleData(buffer, typeRegistry, table); processor.process(new PgOutputReplicationMessage( Operation.INSERT, - table.id().toDoubleQuotedString(), + table.id(), commitTimestamp, transactionId, null, @@ -426,7 +427,7 @@ private void decodeUpdate(ByteBuffer buffer, TypeRegistry typeRegistry, Replicat List columns = resolveColumnsFromStreamTupleData(buffer, typeRegistry, table); processor.process(new PgOutputReplicationMessage( Operation.UPDATE, - table.id().toDoubleQuotedString(), + table.id(), commitTimestamp, transactionId, oldColumns, @@ -459,7 +460,7 @@ private void decodeDelete(ByteBuffer buffer, TypeRegistry typeRegistry, Replicat List columns = resolveColumnsFromStreamTupleData(buffer, typeRegistry, table); processor.process(new PgOutputReplicationMessage( Operation.DELETE, - table.id().toDoubleQuotedString(), + table.id(), commitTimestamp, transactionId, columns, @@ -512,7 +513,7 @@ private void decodeTruncate(ByteBuffer buffer, TypeRegistry typeRegistry, Replic boolean lastTableInTruncate = (i + 1) == noOfResolvedTables; processor.process(new PgOutputTruncateReplicationMessage( Operation.TRUNCATE, - table.id().toDoubleQuotedString(), + table.id(), commitTimestamp, transactionId, lastTableInTruncate)); @@ -596,6 +597,15 @@ private static String readString(ByteBuffer buffer) { return sb.toString(); } + private static String readStringV2(ByteBuffer buffer) { + ByteArrayOutputStream buf = new ByteArrayOutputStream(); + byte b = 0; + while ((b = buffer.get()) != 0) { + buf.write(b); + } + return buf.toString(); + } + /** * Reads the replication stream where the column stream specifies a length followed by the value. * diff --git a/connectors-common/debezium-bucket/debezium-connector-postgres/src/main/java/io/debezium/connector/postgresql/connection/pgoutput/PgOutputReplicationMessage.java b/connectors-common/debezium-bucket/debezium-connector-postgres/src/main/java/io/debezium/connector/postgresql/connection/pgoutput/PgOutputReplicationMessage.java index 71dd2924c..391e68dc1 100644 --- a/connectors-common/debezium-bucket/debezium-connector-postgres/src/main/java/io/debezium/connector/postgresql/connection/pgoutput/PgOutputReplicationMessage.java +++ b/connectors-common/debezium-bucket/debezium-connector-postgres/src/main/java/io/debezium/connector/postgresql/connection/pgoutput/PgOutputReplicationMessage.java @@ -13,6 +13,7 @@ import io.debezium.connector.postgresql.TypeRegistry; import io.debezium.connector.postgresql.connection.ReplicationMessage; import io.debezium.connector.postgresql.connection.ReplicationMessageColumnValueResolver; +import io.debezium.relational.TableId; /** * @author Gunnar Morling @@ -23,15 +24,15 @@ public class PgOutputReplicationMessage implements ReplicationMessage { private Operation op; private Instant commitTimestamp; private long transactionId; - private String table; + private TableId tableId; private List oldColumns; private List newColumns; - public PgOutputReplicationMessage(Operation op, String table, Instant commitTimestamp, long transactionId, List oldColumns, List newColumns) { + public PgOutputReplicationMessage(Operation op, TableId tableId, Instant commitTimestamp, long transactionId, List oldColumns, List newColumns) { this.op = op; this.commitTimestamp = commitTimestamp; this.transactionId = transactionId; - this.table = table; + this.tableId = tableId; this.oldColumns = oldColumns; this.newColumns = newColumns; } @@ -53,7 +54,12 @@ public long getTransactionId() { @Override public String getTable() { - return table; + return tableId.toDoubleQuotedString(); + } + + @Override + public TableId getTableId() { + return tableId; } @Override diff --git a/connectors-common/debezium-bucket/debezium-connector-postgres/src/main/java/io/debezium/connector/postgresql/connection/pgoutput/PgOutputTruncateReplicationMessage.java b/connectors-common/debezium-bucket/debezium-connector-postgres/src/main/java/io/debezium/connector/postgresql/connection/pgoutput/PgOutputTruncateReplicationMessage.java index 25980afe4..33312cbde 100644 --- a/connectors-common/debezium-bucket/debezium-connector-postgres/src/main/java/io/debezium/connector/postgresql/connection/pgoutput/PgOutputTruncateReplicationMessage.java +++ b/connectors-common/debezium-bucket/debezium-connector-postgres/src/main/java/io/debezium/connector/postgresql/connection/pgoutput/PgOutputTruncateReplicationMessage.java @@ -6,15 +6,17 @@ package io.debezium.connector.postgresql.connection.pgoutput; +import io.debezium.relational.TableId; + import java.time.Instant; public class PgOutputTruncateReplicationMessage extends PgOutputReplicationMessage { private final boolean lastTableInTruncate; - public PgOutputTruncateReplicationMessage(Operation op, String table, Instant commitTimestamp, long transactionId, + public PgOutputTruncateReplicationMessage(Operation op, TableId tableId, Instant commitTimestamp, long transactionId, boolean lastTableInTruncate) { - super(op, table, commitTimestamp, transactionId, null, null); + super(op, tableId, commitTimestamp, transactionId, null, null); this.lastTableInTruncate = lastTableInTruncate; } diff --git a/connectors-common/debezium-bucket/debezium-connector-postgres/src/main/java/io/debezium/connector/postgresql/connection/wal2json/Wal2JsonReplicationMessage.java b/connectors-common/debezium-bucket/debezium-connector-postgres/src/main/java/io/debezium/connector/postgresql/connection/wal2json/Wal2JsonReplicationMessage.java index a6d8e7213..391ee5f31 100644 --- a/connectors-common/debezium-bucket/debezium-connector-postgres/src/main/java/io/debezium/connector/postgresql/connection/wal2json/Wal2JsonReplicationMessage.java +++ b/connectors-common/debezium-bucket/debezium-connector-postgres/src/main/java/io/debezium/connector/postgresql/connection/wal2json/Wal2JsonReplicationMessage.java @@ -11,6 +11,7 @@ import java.util.List; import java.util.regex.Matcher; +import io.debezium.relational.TableId; import org.apache.kafka.connect.data.Field; import org.apache.kafka.connect.errors.ConnectException; import org.slf4j.Logger; @@ -82,6 +83,11 @@ public String getTable() { return "\"" + rawMessage.getString("schema") + "\".\"" + rawMessage.getString("table") + "\""; } + @Override + public TableId getTableId() { + return new TableId(null, rawMessage.getString("schema"), rawMessage.getString("table")); + } + @Override public List getOldTupleList() { final Document oldkeys = rawMessage.getDocument("oldkeys"); diff --git a/connectors-common/debezium-bucket/debezium-core/src/main/java/io/debezium/relational/ddl/AbstractDdlParser.java b/connectors-common/debezium-bucket/debezium-core/src/main/java/io/debezium/relational/ddl/AbstractDdlParser.java index 8fa886eda..3147fdecb 100644 --- a/connectors-common/debezium-bucket/debezium-core/src/main/java/io/debezium/relational/ddl/AbstractDdlParser.java +++ b/connectors-common/debezium-bucket/debezium-core/src/main/java/io/debezium/relational/ddl/AbstractDdlParser.java @@ -274,7 +274,12 @@ protected String removeLineFeeds(String input) { * @return string without quotes */ public static String withoutQuotes(String possiblyQuoted) { - return isQuoted(possiblyQuoted) ? possiblyQuoted.substring(1, possiblyQuoted.length() - 1) : possiblyQuoted; + if (isQuoted(possiblyQuoted)) { + String quote = possiblyQuoted.substring(0, 1); + return possiblyQuoted.substring(1, possiblyQuoted.length() - 1).replace(quote + quote, quote); + } else { + return possiblyQuoted; + } } /** diff --git a/connectors-common/file-connector-core/pom.xml b/connectors-common/file-connector-core/pom.xml index 1a5a7c8d3..265dad85c 100644 --- a/connectors-common/file-connector-core/pom.xml +++ b/connectors-common/file-connector-core/pom.xml @@ -13,8 +13,8 @@ jar - 1.4.1-SNAPSHOT - 1.4.1-SNAPSHOT + 1.4.4-SNAPSHOT + 1.4.4-SNAPSHOT 8 diff --git a/connectors-common/hive-core/pom.xml b/connectors-common/hive-core/pom.xml index 6c70db825..6878fd68a 100644 --- a/connectors-common/hive-core/pom.xml +++ b/connectors-common/hive-core/pom.xml @@ -18,7 +18,7 @@ 8 1.0-SNAPSHOT - 1.4.1-SNAPSHOT + 1.4.4-SNAPSHOT diff --git a/connectors-common/hive-core/src/main/java/io/tapdata/connector/hive/HiveConnector.java b/connectors-common/hive-core/src/main/java/io/tapdata/connector/hive/HiveConnector.java index b0d2938de..0d42e0ad2 100644 --- a/connectors-common/hive-core/src/main/java/io/tapdata/connector/hive/HiveConnector.java +++ b/connectors-common/hive-core/src/main/java/io/tapdata/connector/hive/HiveConnector.java @@ -55,8 +55,8 @@ public void registerCapabilities(ConnectorFunctions connectorFunctions, TapCodec return "null"; }); codecRegistry.registerFromTapValue(TapBinaryValue.class, "string", tapValue -> { - if (tapValue != null && tapValue.getValue() != null) - return new String(Base64.encodeBase64(tapValue.getValue())); + if (tapValue != null && tapValue.getValue() != null && tapValue.getValue().getValue() != null) + return new String(Base64.encodeBase64(tapValue.getValue().getValue())); return null; }); codecRegistry.registerFromTapValue(TapTimeValue.class, "string", tapTimeValue -> formatTapDateTime(tapTimeValue.getValue(), "HH:mm:ss.SS")); diff --git a/connectors-common/js-connector-core-plus/pom.xml b/connectors-common/js-connector-core-plus/pom.xml index 6ad065862..da4766d8c 100644 --- a/connectors-common/js-connector-core-plus/pom.xml +++ b/connectors-common/js-connector-core-plus/pom.xml @@ -14,7 +14,7 @@ 8 - 1.4.1-SNAPSHOT + 1.4.4-SNAPSHOT diff --git a/connectors-common/js-connector-core/pom.xml b/connectors-common/js-connector-core/pom.xml index e18aa5b1e..b9d74983c 100644 --- a/connectors-common/js-connector-core/pom.xml +++ b/connectors-common/js-connector-core/pom.xml @@ -14,7 +14,7 @@ 8 - 1.4.1-SNAPSHOT + 1.4.4-SNAPSHOT diff --git a/connectors-common/mysql-core/pom.xml b/connectors-common/mysql-core/pom.xml index 44c2f4ec0..d395f811c 100644 --- a/connectors-common/mysql-core/pom.xml +++ b/connectors-common/mysql-core/pom.xml @@ -33,7 +33,7 @@ 8 8.0.33 1.5.4.Final - 1.4.1-SNAPSHOT + 1.4.4-SNAPSHOT 1.0-SNAPSHOT diff --git a/connectors-common/mysql-core/src/main/java/io/tapdata/connector/mysql/MysqlJdbcContextV2.java b/connectors-common/mysql-core/src/main/java/io/tapdata/connector/mysql/MysqlJdbcContextV2.java index da671c6be..c070cb2e7 100644 --- a/connectors-common/mysql-core/src/main/java/io/tapdata/connector/mysql/MysqlJdbcContextV2.java +++ b/connectors-common/mysql-core/src/main/java/io/tapdata/connector/mysql/MysqlJdbcContextV2.java @@ -58,19 +58,25 @@ public TimeZone queryTimeZone() throws SQLException { @Override protected String queryAllTablesSql(String schema, List tableNames) { String tableSql = EmptyKit.isNotEmpty(tableNames) ? "AND TABLE_NAME IN (" + StringKit.joinString(tableNames, "'", ",") + ")" : ""; - return String.format(MYSQL_ALL_TABLE, schema, tableSql); + return String.format(MYSQL_ALL_TABLE, StringKit.escape(schema, "'"), tableSql); } @Override protected String queryAllColumnsSql(String schema, List tableNames) { String tableSql = EmptyKit.isNotEmpty(tableNames) ? "AND TABLE_NAME IN (" + StringKit.joinString(tableNames, "'", ",") + ")" : ""; - return String.format(MYSQL_ALL_COLUMN, schema, tableSql); + return String.format(MYSQL_ALL_COLUMN, StringKit.escape(schema, "'"), tableSql); } @Override protected String queryAllIndexesSql(String schema, List tableNames) { String tableSql = EmptyKit.isNotEmpty(tableNames) ? "AND TABLE_NAME IN (" + StringKit.joinString(tableNames, "'", ",") + ")" : ""; - return String.format(MYSQL_ALL_INDEX, schema, tableSql); + return String.format(MYSQL_ALL_INDEX, StringKit.escape(schema, "'"), tableSql); + } + + @Override + protected String queryAllForeignKeysSql(String schema, List tableNames) { + String tableSql = EmptyKit.isNotEmpty(tableNames) ? "AND k.TABLE_NAME IN (" + StringKit.joinString(tableNames, "'", ",") + ")" : ""; + return String.format(MYSQL_ALL_FOREIGN_KEY, StringKit.escape(schema, "'"), tableSql); } public DataMap getTableInfo(String tableName) { @@ -79,7 +85,7 @@ public DataMap getTableInfo(String tableName) { list.add("TABLE_ROWS"); list.add("DATA_LENGTH"); try { - query(String.format(GET_TABLE_INFO_SQL, getConfig().getDatabase(), tableName), resultSet -> { + query(String.format(GET_TABLE_INFO_SQL, StringKit.escape(getConfig().getDatabase(), "'"), tableName), resultSet -> { while (resultSet.next()) { dataMap.putAll(DbKit.getRowFromResultSet(resultSet, list)); } @@ -229,7 +235,9 @@ public void queryWithStream(String sql, ResultSetConsumer resultSetConsumer) thr " COLUMN_NAME `columnName`,\n" + " COLUMN_TYPE `dataType`,\n" + " IS_NULLABLE `nullable`,\n" + - " COLUMN_COMMENT `columnComment`\n" + + " COLUMN_COMMENT `columnComment`,\n" + + " COLUMN_DEFAULT `columnDefault`,\n" + + " IF(EXTRA = 'auto_increment', 1, 0) `autoInc`" + "FROM INFORMATION_SCHEMA.COLUMNS\n" + "WHERE TABLE_SCHEMA = '%s' %s\n" + "ORDER BY ORDINAL_POSITION"; @@ -259,6 +267,24 @@ public void queryWithStream(String sql, ResultSetConsumer resultSetConsumer) thr "\tINDEX_NAME,\n" + "\tSEQ_IN_INDEX"; + private final static String MYSQL_ALL_FOREIGN_KEY = + "SELECT\n" + + " k.CONSTRAINT_NAME `constraintName`,\n" + + " k.TABLE_NAME `tableName`,\n" + + " k.REFERENCED_TABLE_NAME `referencesTableName`,\n" + + " c.DELETE_RULE `onDelete`,\n" + + " c.UPDATE_RULE `onUpdate`,\n" + + " k.COLUMN_NAME `fk`,\n" + + " k.REFERENCED_COLUMN_NAME `rfk`\n" + + "FROM\n" + + " information_schema.KEY_COLUMN_USAGE k\n" + + "JOIN information_schema.REFERENTIAL_CONSTRAINTS c\n" + + "ON c.CONSTRAINT_SCHEMA=k.TABLE_SCHEMA\n" + + "AND c.CONSTRAINT_NAME=k.CONSTRAINT_NAME\n" + + "WHERE\n" + + " k.REFERENCED_TABLE_NAME IS NOT NULL\n" + + " AND k.CONSTRAINT_SCHEMA = '%s' %s"; + private final static String MYSQL_VERSION = "SELECT VERSION()"; private final static String MYSQL_CURRENT_TIME = "SELECT NOW();"; private final static String MYSQL_TIMESTAMP = "SELECT REPLACE(unix_timestamp(NOW(3)),'.','') AS currentTimeMillis"; diff --git a/connectors-common/mysql-core/src/main/java/io/tapdata/connector/mysql/MysqlMaker.java b/connectors-common/mysql-core/src/main/java/io/tapdata/connector/mysql/MysqlMaker.java index 086cf9c05..08d567dc3 100644 --- a/connectors-common/mysql-core/src/main/java/io/tapdata/connector/mysql/MysqlMaker.java +++ b/connectors-common/mysql-core/src/main/java/io/tapdata/connector/mysql/MysqlMaker.java @@ -1,5 +1,6 @@ package io.tapdata.connector.mysql; +import io.tapdata.connector.mysql.bean.MysqlColumn; import io.tapdata.connector.mysql.entity.MysqlSnapshotOffset; import io.tapdata.connector.mysql.util.MysqlUtil; import io.tapdata.connector.tencent.db.mysql.MysqlJdbcContext; @@ -9,11 +10,11 @@ import io.tapdata.entity.schema.TapIndex; import io.tapdata.entity.schema.TapIndexField; import io.tapdata.entity.schema.TapTable; -import io.tapdata.entity.schema.type.TapNumber; -import io.tapdata.entity.simplify.TapSimplify; import io.tapdata.entity.schema.value.DateTime; +import io.tapdata.entity.simplify.TapSimplify; import io.tapdata.entity.utils.DataMap; import io.tapdata.kit.EmptyKit; +import io.tapdata.kit.StringKit; import io.tapdata.pdk.apis.context.TapConnectorContext; import io.tapdata.pdk.apis.entity.Projection; import io.tapdata.pdk.apis.entity.QueryOperator; @@ -43,6 +44,8 @@ public class MysqlMaker implements SqlMaker { private static final String MYSQL_ADD_INDEX = "ALTER TABLE `%s`.`%s` ADD %s %s (%s)"; private boolean hasAutoIncrement; protected static final int DEFAULT_CONSTRAINT_NAME_MAX_LENGTH = 30; + protected Boolean createAutoInc = false; + protected Boolean applyDefault = false; @Override public String[] createTable(TapConnectorContext tapConnectorContext, TapCreateTableEvent tapCreateTableEvent, String version) throws Throwable { @@ -71,7 +74,7 @@ public String[] createTable(TapConnectorContext tapConnectorContext, TapCreateTa tablePropertiesSql += " COMMENT='" + tapTable.getComment().replace("'", "''").replaceAll("\\\\", "\\\\\\\\") + "'"; } - String sql = String.format(CREATE_TABLE_TEMPLATE, database, tapTable.getId(), fieldSql, tablePropertiesSql); + String sql = String.format(CREATE_TABLE_TEMPLATE, StringKit.escape(database, '`'), StringKit.escape(tapTable.getId(), '`'), fieldSql, tablePropertiesSql); return new String[]{sql}; } @@ -122,11 +125,11 @@ public String selectSql(TapConnectorContext tapConnectorContext, TapTable tapTab String database = connectionConfig.getString("database"); String tableId = tapTable.getId(); String sql; - Projection projection = tapAdvanceFilter.getProjection(); + Projection projection = tapAdvanceFilter.getProjection(); if (EmptyKit.isNull(projection) || (EmptyKit.isEmpty(projection.getIncludeFields()) && EmptyKit.isEmpty(projection.getExcludeFields()))) { sql = String.format(MysqlJdbcContext.SELECT_TABLE, database, tableId); } else if (EmptyKit.isNotEmpty(tapAdvanceFilter.getProjection().getIncludeFields())) { - sql = String.format(MysqlJdbcContext.SELECT_SOME_FROM_TABLE, String.join("`,`", tapAdvanceFilter.getProjection().getIncludeFields()), database, tableId); + sql = String.format(MysqlJdbcContext.SELECT_SOME_FROM_TABLE, String.join("`,`", tapAdvanceFilter.getProjection().getIncludeFields()), database, tableId); } else { sql = String.format(MysqlJdbcContext.SELECT_SOME_FROM_TABLE, tapTable.getNameFieldMap().keySet().stream() .filter(tapField -> !tapAdvanceFilter.getProjection().getExcludeFields().contains(tapField)).collect(Collectors.joining("`,`")), database, tableId); @@ -156,7 +159,7 @@ public String selectSql(TapConnectorContext tapConnectorContext, TapTable tapTab String opStr = queryOperatorEnum.getOpStr(); if (value instanceof Number) { whereList.add(String.format(MysqlJdbcContext.FIELD_TEMPLATE, key) + opStr + value); - }else if (value instanceof DateTime) { + } else if (value instanceof DateTime) { whereList.add(String.format(MysqlJdbcContext.FIELD_TEMPLATE, key) + opStr + "'" + dateTimeToStr((DateTime) value, "yyyy-MM-dd HH:mm:ss") + "'"); } else { whereList.add(String.format(MysqlJdbcContext.FIELD_TEMPLATE, key) + opStr + "'" + value + "'"); @@ -206,7 +209,7 @@ public String selectSql(TapConnectorContext tapConnectorContext, TapTable tapTab QueryOperator leftBoundary = tapPartitionFilter.getLeftBoundary(); QueryOperator rightBoundary = tapPartitionFilter.getRightBoundary(); List queryOperators = TapSimplify.list(leftBoundary, rightBoundary); - queryOperators.forEach(o->{ + queryOperators.forEach(o -> { String queryOperatorSql = getQueryOperatorSql(o); if (StringUtils.isNotBlank(queryOperatorSql)) { whereList.add(queryOperatorSql); @@ -214,7 +217,7 @@ public String selectSql(TapConnectorContext tapConnectorContext, TapTable tapTab }); DataMap match = tapPartitionFilter.getMatch(); if (MapUtils.isNotEmpty(match)) { - match.forEach((k,v)->{ + match.forEach((k, v) -> { String valueStr = MysqlUtil.object2String(v); whereList.add(String.format("`%s`<=>%s", k, valueStr)); }); @@ -281,11 +284,14 @@ public String createIndex(TapConnectorContext tapConnectorContext, TapTable tapT protected String createTableAppendField(TapField tapField) { String datatype = tapField.getDataType(); - String fieldSql = " `" + tapField.getName() + "`" + " " + datatype; + if (createAutoInc && Boolean.TRUE.equals(tapField.getAutoInc()) && tapField.getPrimaryKeyPos() == 1 && datatype.startsWith("decimal")) { + datatype = "bigint"; + } + String fieldSql = " `" + StringKit.escape(tapField.getName(), '`') + "`" + " " + datatype; // auto increment // mysql a table can only create one auto-increment column, and must be the primary key - if (null != tapField.getAutoInc() && tapField.getAutoInc()) { + if (createAutoInc && null != tapField.getAutoInc() && tapField.getAutoInc()) { if (tapField.getPrimaryKeyPos() == 1) { fieldSql += " AUTO_INCREMENT"; } else { @@ -301,16 +307,22 @@ protected String createTableAppendField(TapField tapField) { } // default value - String defaultValue = tapField.getDefaultValue() == null ? "" : tapField.getDefaultValue().toString(); - if (StringUtils.isNotBlank(defaultValue)) { - if (defaultValue.contains("'")) { - defaultValue = StringUtils.replace(defaultValue, "'", "''"); - } - if (tapField.getTapType() instanceof TapNumber) { - defaultValue = defaultValue.trim(); + if (applyDefault && EmptyKit.isNotNull(tapField.getDefaultValue())) { + StringBuilder builder = new StringBuilder(); + builder.append("DEFAULT").append(' '); + if (EmptyKit.isNotNull(tapField.getDefaultFunction())) { + String function = MysqlColumn.MysqlDefaultFunction.valueOf(tapField.getDefaultFunction().toString()).getFunction(); + if (function.endsWith("()")) { + builder.append("(").append(function).append(") "); + } else { + builder.append(function).append(' '); + } + } else if (tapField.getDefaultValue() instanceof Number) { + builder.append(tapField.getDefaultValue()).append(' '); + } else { + builder.append("'").append(StringKit.escape(tapField.getDefaultValue().toString(), "'")).append("' "); } - fieldSql += " DEFAULT '" + defaultValue + "'"; - + fieldSql += " " + builder; } // comment @@ -338,7 +350,7 @@ protected String createTableAppendPrimaryKey(TapTable tapTable) { // pk fields Collection primaryKeys = tapTable.primaryKeys(); - String pkFieldString = "`" + String.join("`,`", primaryKeys) + "`"; + String pkFieldString = "`" + primaryKeys.stream().map(v -> StringKit.escape(v, '`')).collect(Collectors.joining("`,`")) + "`"; pkSql += pkFieldString + ")"; return pkSql; @@ -355,6 +367,14 @@ protected String getConstraintName(String constraintName) { return constraintName; } + public void setCreateAutoInc(Boolean createAutoInc) { + this.createAutoInc = createAutoInc; + } + + public void setApplyDefault(Boolean applyDefault) { + this.applyDefault = applyDefault; + } + private enum QueryOperatorEnum { GT(QueryOperator.GT, ">"), GTE(QueryOperator.GTE, ">="), LE(QueryOperator.LT, "<"), LTE(QueryOperator.LTE, "<="), ; diff --git a/connectors-common/mysql-core/src/main/java/io/tapdata/connector/mysql/MysqlReader.java b/connectors-common/mysql-core/src/main/java/io/tapdata/connector/mysql/MysqlReader.java index 41d21451a..82a173197 100644 --- a/connectors-common/mysql-core/src/main/java/io/tapdata/connector/mysql/MysqlReader.java +++ b/connectors-common/mysql-core/src/main/java/io/tapdata/connector/mysql/MysqlReader.java @@ -41,6 +41,7 @@ import io.tapdata.entity.utils.cache.KVReadOnlyMap; import io.tapdata.kit.EmptyKit; import io.tapdata.kit.ErrorKit; +import io.tapdata.kit.StringKit; import io.tapdata.pdk.apis.consumer.StreamReadConsumer; import io.tapdata.pdk.apis.context.TapConnectorContext; import io.tapdata.pdk.apis.entity.TapAdvanceFilter; @@ -54,7 +55,6 @@ import java.io.Closeable; import java.io.IOException; -import java.net.URLDecoder; import java.nio.ByteBuffer; import java.sql.ResultSetMetaData; import java.time.*; @@ -66,7 +66,6 @@ import java.util.function.Consumer; import java.util.function.Predicate; import java.util.function.Supplier; -import java.util.logging.LogManager; import java.util.stream.Collectors; import static io.tapdata.connector.mysql.util.MysqlUtil.randomServerId; @@ -298,16 +297,16 @@ public void readBinlog(TapConnectorContext tapConnectorContext, List tab // .with("converters", "time") // .with("time.type", "io.tapdata.connector.mysql.converters.TimeConverter") // .with("time.schema.name", "io.debezium.mysql.type.Time") - .with("enable.time.adjuster",false) + .with("enable.time.adjuster", false) .with("snapshot.locking.mode", "none"); // if (EmptyKit.isNotBlank(mysqlConfig.getTimezone())) { // builder.with("database.serverTimezone", mysqlJdbcContext.queryTimeZone()); // } - List dbTableNames = tables.stream().map(t -> mysqlConfig.getDatabase() + "." + t).collect(Collectors.toList()); + List dbTableNames = tables.stream().map(t -> StringKit.escapeRegex(mysqlConfig.getDatabase()) + "." + StringKit.escapeRegex(t)).collect(Collectors.toList()); if (mysqlConfig.getDoubleActive()) { - dbTableNames.add(mysqlConfig.getDatabase() + "._tap_double_active"); + dbTableNames.add(StringKit.escapeRegex(mysqlConfig.getDatabase()) + "._tap_double_active"); } - builder.with(MySqlConnectorConfig.DATABASE_INCLUDE_LIST, mysqlConfig.getDatabase()); + builder.with(MySqlConnectorConfig.DATABASE_INCLUDE_LIST, StringKit.escapeRegex(mysqlConfig.getDatabase())); builder.with(MySqlConnectorConfig.TABLE_INCLUDE_LIST, String.join(",", dbTableNames)); builder.with(EmbeddedEngine.OFFSET_STORAGE, "io.tapdata.connector.mysql.PdkPersistenceOffsetBackingStore"); if (StringUtils.isNotBlank(offsetStr)) { @@ -937,5 +936,4 @@ private Set dateFields(TapTable tapTable) { return dateTypeSet; } - } diff --git a/connectors-common/mysql-core/src/main/java/io/tapdata/connector/mysql/bean/MysqlColumn.java b/connectors-common/mysql-core/src/main/java/io/tapdata/connector/mysql/bean/MysqlColumn.java index 0a5870885..57f0d9e15 100644 --- a/connectors-common/mysql-core/src/main/java/io/tapdata/connector/mysql/bean/MysqlColumn.java +++ b/connectors-common/mysql-core/src/main/java/io/tapdata/connector/mysql/bean/MysqlColumn.java @@ -5,6 +5,9 @@ import io.tapdata.entity.utils.DataMap; import io.tapdata.kit.EmptyKit; +import java.util.HashMap; +import java.util.Map; + /** * @author jarad @@ -12,19 +15,31 @@ public class MysqlColumn extends CommonColumn { private String version; + private Long seedValue; + private Long incrementValue; + private Long autoIncCacheValue; public MysqlColumn(DataMap dataMap) { this.columnName = dataMap.getString("columnName"); this.dataType = dataMap.getString("dataType"); this.nullable = dataMap.getString("nullable"); this.remarks = dataMap.getString("columnComment"); - this.columnDefaultValue = null; + this.columnDefaultValue = dataMap.getString("columnDefault"); + this.autoInc = dataMap.getString("autoInc"); } @Override public TapField getTapField() { - return new TapField(this.columnName, this.dataType).nullable(this.isNullable()). - defaultValue(columnDefaultValue).comment(this.remarks); + TapField field = new TapField(this.columnName, this.dataType) + .nullable(this.isNullable()).autoInc(isAutoInc()) + .defaultValue(columnDefaultValue).comment(this.remarks); + if (isAutoInc()) { + field.autoIncStartValue(seedValue) + .autoIncrementValue(incrementValue) + .autoIncCacheValue(autoIncCacheValue); + } + generateDefaultValue(field); + return field; } public MysqlColumn withVersion(String version) { @@ -32,6 +47,21 @@ public MysqlColumn withVersion(String version) { return this; } + public MysqlColumn withSeedValue(Long seedValue) { + this.seedValue = seedValue; + return this; + } + + public MysqlColumn withIncrementValue(Long incrementValue) { + this.incrementValue = incrementValue; + return this; + } + + public MysqlColumn withAutoIncCacheValue(Long autoIncCacheValue) { + this.autoIncCacheValue = autoIncCacheValue; + return this; + } + @Override protected Boolean isNullable() { if (EmptyKit.isNotNull(version) && "5.6".compareTo(version) > 0) { @@ -40,4 +70,37 @@ protected Boolean isNullable() { return "YES".equals(this.nullable); } + protected String parseDefaultFunction(String defaultValue) { + return MysqlDefaultFunction.parseFunction(columnDefaultValue); + } + + public enum MysqlDefaultFunction { + _CURRENT_TIMESTAMP("CURRENT_TIMESTAMP"), + _GENERATE_UUID("uuid()"); + + private final String function; + private static final Map map = new HashMap<>(); + + static { + for (MysqlDefaultFunction value : MysqlDefaultFunction.values()) { + map.put(value.function, value.name()); + } + } + + MysqlDefaultFunction(String function) { + this.function = function; + } + + public static String parseFunction(String key) { + if (map.containsKey(key)) { + return map.get(key); + } + return null; + } + + public String getFunction() { + return function; + } + } + } diff --git a/connectors-common/mysql-core/src/main/java/io/tapdata/connector/mysql/config/MysqlConfig.java b/connectors-common/mysql-core/src/main/java/io/tapdata/connector/mysql/config/MysqlConfig.java index 6a8238f98..5fb3c10b9 100644 --- a/connectors-common/mysql-core/src/main/java/io/tapdata/connector/mysql/config/MysqlConfig.java +++ b/connectors-common/mysql-core/src/main/java/io/tapdata/connector/mysql/config/MysqlConfig.java @@ -7,6 +7,7 @@ import org.apache.commons.lang3.StringUtils; import java.io.IOException; +import java.net.URLEncoder; import java.util.*; public class MysqlConfig extends CommonDbConfig { @@ -72,7 +73,7 @@ public String getDatabaseUrl() { if (additionalString.startsWith("?")) { additionalString = additionalString.substring(1); } - StringBuilder sbURL = new StringBuilder("jdbc:").append(getDbType()).append("://").append(getHost()).append(":").append(getPort()).append("/").append(getDatabase()); + StringBuilder sbURL = new StringBuilder("jdbc:").append(getDbType()).append("://").append(getHost()).append(":").append(getPort()).append("/").append(URLEncoder.encode(getDatabase())); Map properties = new HashMap<>(); if (StringUtils.isNotBlank(additionalString)) { diff --git a/connectors-common/mysql-core/src/main/java/io/tapdata/connector/mysql/dml/MysqlRecordWriter.java b/connectors-common/mysql-core/src/main/java/io/tapdata/connector/mysql/dml/MysqlRecordWriter.java index 7a7aab36d..c40fa9bb9 100644 --- a/connectors-common/mysql-core/src/main/java/io/tapdata/connector/mysql/dml/MysqlRecordWriter.java +++ b/connectors-common/mysql-core/src/main/java/io/tapdata/connector/mysql/dml/MysqlRecordWriter.java @@ -34,4 +34,8 @@ public MysqlRecordWriter(JdbcContext jdbcContext, Connection connection, TapTabl deleteRecorder = new MysqlWriteRecorder(connection, tapTable, jdbcContext.getConfig().getDatabase()); } + protected String getCloseConstraintCheckSql() { + return "SET FOREIGN_KEY_CHECKS=0"; + } + } diff --git a/connectors-common/mysql-core/src/main/java/io/tapdata/connector/mysql/dml/MysqlWriteRecorder.java b/connectors-common/mysql-core/src/main/java/io/tapdata/connector/mysql/dml/MysqlWriteRecorder.java index e7b087a37..0d93e8128 100644 --- a/connectors-common/mysql-core/src/main/java/io/tapdata/connector/mysql/dml/MysqlWriteRecorder.java +++ b/connectors-common/mysql-core/src/main/java/io/tapdata/connector/mysql/dml/MysqlWriteRecorder.java @@ -30,17 +30,17 @@ protected void largeInsert(Map after) { protected String getLargeInsertSql() { if (WritePolicyEnum.UPDATE_ON_EXISTS == insertPolicy) { - return "INSERT INTO " + escapeChar + schema + escapeChar + "." + escapeChar + tapTable.getId() + escapeChar + " (" - + allColumn.stream().map(k -> escapeChar + k + escapeChar).collect(Collectors.joining(", ")) + ") VALUES " + return "INSERT INTO " + getSchemaAndTable() + " (" + + allColumn.stream().map(this::quoteAndEscape).collect(Collectors.joining(", ")) + ") VALUES " + String.join(", ", largeSqlValues) + " ON DUPLICATE KEY UPDATE " - + allColumn.stream().map(k -> escapeChar + k + escapeChar + "=values(" + escapeChar + k + escapeChar + ")").collect(Collectors.joining(", ")); + + allColumn.stream().map(k -> quoteAndEscape(k) + "=values(" + quoteAndEscape(k) + ")").collect(Collectors.joining(", ")); } else if (WritePolicyEnum.IGNORE_ON_EXISTS == insertPolicy) { - return "INSERT IGNORE INTO " + escapeChar + schema + escapeChar + "." + escapeChar + tapTable.getId() + escapeChar + " (" - + allColumn.stream().map(k -> escapeChar + k + escapeChar).collect(Collectors.joining(", ")) + ") VALUES " + return "INSERT IGNORE INTO " + getSchemaAndTable() + " (" + + allColumn.stream().map(this::quoteAndEscape).collect(Collectors.joining(", ")) + ") VALUES " + String.join(", ", largeSqlValues); } else { - return "INSERT INTO " + escapeChar + schema + escapeChar + "." + escapeChar + tapTable.getId() + escapeChar + " (" - + allColumn.stream().map(k -> escapeChar + k + escapeChar).collect(Collectors.joining(", ")) + ") VALUES " + return "INSERT INTO " + getSchemaAndTable() + " (" + + allColumn.stream().map(this::quoteAndEscape).collect(Collectors.joining(", ")) + ") VALUES " + String.join(", ", largeSqlValues); } } @@ -58,10 +58,10 @@ protected void upsert(Map after, WriteListResult } protected String getUpsertSql() { - return "INSERT INTO " + escapeChar + schema + escapeChar + "." + escapeChar + tapTable.getId() + escapeChar + " (" - + allColumn.stream().map(k -> escapeChar + k + escapeChar).collect(Collectors.joining(", ")) + ") " + + return "INSERT INTO " + getSchemaAndTable() + " (" + + allColumn.stream().map(this::quoteAndEscape).collect(Collectors.joining(", ")) + ") " + "VALUES(" + StringKit.copyString("?", allColumn.size(), ",") + ") ON DUPLICATE KEY UPDATE " - + allColumn.stream().map(k -> escapeChar + k + escapeChar + "=values(" + escapeChar + k + escapeChar + ")").collect(Collectors.joining(", ")); + + allColumn.stream().map(k -> quoteAndEscape(k) + "=values(" + quoteAndEscape(k) + ")").collect(Collectors.joining(", ")); } protected void insertIgnore(Map after, WriteListResult listResult) throws SQLException { @@ -76,8 +76,8 @@ protected void insertIgnore(Map after, WriteListResult escapeChar + k + escapeChar).collect(Collectors.joining(", ")) + ") " + + return "INSERT IGNORE INTO " + getSchemaAndTable() + " (" + + allColumn.stream().map(this::quoteAndEscape).collect(Collectors.joining(", ")) + ") " + "VALUES(" + StringKit.copyString("?", allColumn.size(), ",") + ")"; } @@ -109,10 +109,10 @@ protected void insertUpdate(Map after, Map befor } protected String getInsertUpdateSql(Map after, Map before) { - return "INSERT INTO " + escapeChar + schema + escapeChar + "." + escapeChar + tapTable.getId() + escapeChar + " (" - + allColumn.stream().map(k -> escapeChar + k + escapeChar).collect(Collectors.joining(", ")) + ") " + + return "INSERT INTO " + getSchemaAndTable() + " (" + + allColumn.stream().map(this::quoteAndEscape).collect(Collectors.joining(", ")) + ") " + "VALUES(" + StringKit.copyString("?", allColumn.size(), ",") + ") ON DUPLICATE KEY UPDATE " - + after.keySet().stream().map(k -> escapeChar + k + escapeChar + "=values(" + escapeChar + k + escapeChar + ")").collect(Collectors.joining(", ")); + + after.keySet().stream().map(k -> quoteAndEscape(k) + "=values(" + quoteAndEscape(k) + ")").collect(Collectors.joining(", ")); } protected Object filterValue(Object value, String dataType) { diff --git a/connectors-common/pom.xml b/connectors-common/pom.xml index f79b3aa87..d40d3651c 100644 --- a/connectors-common/pom.xml +++ b/connectors-common/pom.xml @@ -28,8 +28,8 @@ ${project.artifactId}-v${project.version} 8 1.10-SNAPSHOT - 1.4.1-SNAPSHOT - 1.4.1-SNAPSHOT + 1.4.4-SNAPSHOT + 1.4.4-SNAPSHOT 1.0-SNAPSHOT 5.8.1 1.8.1 diff --git a/connectors-common/postgres-core/pom.xml b/connectors-common/postgres-core/pom.xml index f92747592..e4d2fc6c4 100644 --- a/connectors-common/postgres-core/pom.xml +++ b/connectors-common/postgres-core/pom.xml @@ -21,7 +21,7 @@ 1.0-SNAPSHOT 1.5.4.Final 1.0-SNAPSHOT - 1.4.3-SNAPSHOT + 1.4.4-SNAPSHOT diff --git a/connectors-common/postgres-core/src/main/java/io/tapdata/connector/postgres/PostgresJdbcContext.java b/connectors-common/postgres-core/src/main/java/io/tapdata/connector/postgres/PostgresJdbcContext.java index c05ed4d8a..14b6f948a 100644 --- a/connectors-common/postgres-core/src/main/java/io/tapdata/connector/postgres/PostgresJdbcContext.java +++ b/connectors-common/postgres-core/src/main/java/io/tapdata/connector/postgres/PostgresJdbcContext.java @@ -59,6 +59,7 @@ public TimeZone queryTimeZone() throws SQLException { return TimeZone.getTimeZone(ZoneId.of(decimalFormat.format(timeOffset.get()) + ":00")); } } + @Override public Long queryTimestamp() throws SQLException { AtomicReference currentTime = new AtomicReference<>(); @@ -70,21 +71,29 @@ public Long queryTimestamp() throws SQLException { protected String queryAllTablesSql(String schema, List tableNames) { String tableSql = EmptyKit.isNotEmpty(tableNames) ? "AND t.table_name IN (" + StringKit.joinString(tableNames, "'", ",") + ")" : ""; if (Integer.parseInt(postgresVersion) < 100000) { - return String.format(PG_ALL_TABLE_LOWER_VERSION, getConfig().getDatabase(), schema, tableSql); + return String.format(PG_ALL_TABLE_LOWER_VERSION, StringKit.escape(getConfig().getDatabase(), "'"), StringKit.escape(schema, "'"), tableSql); } - return String.format(PG_ALL_TABLE, getConfig().getDatabase(), schema, tableSql); + return String.format(PG_ALL_TABLE, StringKit.escape(getConfig().getDatabase(), "'"), StringKit.escape(schema, "'"), tableSql); } @Override protected String queryAllColumnsSql(String schema, List tableNames) { String tableSql = EmptyKit.isNotEmpty(tableNames) ? "AND table_name IN (" + StringKit.joinString(tableNames, "'", ",") + ")" : ""; - return String.format(PG_ALL_COLUMN, schema, getConfig().getDatabase(), schema, tableSql); + if (Integer.parseInt(postgresVersion) < 100000) { + return String.format(PG_ALL_COLUMN_LOWER_VERSION, StringKit.escape(schema, "'"), StringKit.escape(getConfig().getDatabase(), "'"), StringKit.escape(schema, "'"), tableSql); + } + return String.format(PG_ALL_COLUMN, StringKit.escape(schema, "'"), StringKit.escape(getConfig().getDatabase(), "'"), StringKit.escape(schema, "'"), tableSql); } @Override protected String queryAllIndexesSql(String schema, List tableNames) { String tableSql = EmptyKit.isNotEmpty(tableNames) ? "AND table_name IN (" + StringKit.joinString(tableNames, "'", ",") + ")" : ""; - return String.format(PG_ALL_INDEX, getConfig().getDatabase(), schema, tableSql); + return String.format(PG_ALL_INDEX, StringKit.escape(getConfig().getDatabase(), "'"), StringKit.escape(schema, "'"), tableSql); + } + + @Override + protected String queryAllForeignKeysSql(String schema, List tableNames) { + return String.format(PG_ALL_FOREIGN_KEY, StringKit.escape(getConfig().getSchema(), "'"), EmptyKit.isEmpty(tableNames) ? "" : " and pc.relname in (" + StringKit.joinString(tableNames, "'", ",") + ")"); } public DataMap getTableInfo(String tableName) { @@ -93,7 +102,7 @@ public DataMap getTableInfo(String tableName) { list.add("size"); list.add("rowcount"); try { - query(String.format(TABLE_INFO_SQL, getConfig().getSchema(), tableName, getConfig().getDatabase(), getConfig().getSchema()), resultSet -> { + query(String.format(TABLE_INFO_SQL, StringKit.escape(getConfig().getSchema(), "'"), StringKit.escape(tableName, "'"), StringKit.escape(getConfig().getDatabase(), "'"), StringKit.escape(getConfig().getSchema(), "'")), resultSet -> { while (resultSet.next()) { dataMap.putAll(DbKit.getRowFromResultSet(resultSet, list)); } @@ -146,6 +155,45 @@ public DataMap getTableInfo(String tableName) { " t.table_name"; protected final static String PG_ALL_COLUMN = + "SELECT\n" + + " col.table_name \"tableName\",\n" + + " col.column_name \"columnName\",\n" + + " col.data_type \"pureDataType\",\n" + + " col.column_default \"columnDefault\",\n" + + " col.is_nullable \"nullable\",\n" + + " col.is_identity \"autoInc\",\n" + + " col.identity_start AS \"seedValue\",\n" + + " col.identity_increment AS \"incrementValue\",\n" + + " seq.sequence_name AS \"sequenceName\",\n" + + " seq.start_value AS \"seedValue2\",\n" + + " seq.increment AS \"incrementValue2\",\n" + + " (SELECT seqcache\n" + + " FROM pg_sequence\n" + + " WHERE seqrelid = pg_get_serial_sequence('\"'||replace(col.table_schema, '\"', '\"\"')||'\".\"'||replace(col.table_name,'\"','\"\"')||'\"', col.column_name)::regclass) \"cacheValue\"," + + " (SELECT max(d.description)\n" + + " FROM pg_catalog.pg_class c,\n" + + " pg_description d\n" + + " WHERE c.relname = col.table_name\n" + + " AND d.objoid = c.oid\n" + + " AND d.objsubid = col.ordinal_position) AS \"columnComment\",\n" + + " (SELECT pg_catalog.format_type(a.atttypid, a.atttypmod)\n" + + " FROM pg_catalog.pg_attribute a\n" + + " WHERE a.attnum > 0\n" + + " AND a.attname = col.column_name\n" + + " AND NOT a.attisdropped\n" + + " AND a.attrelid =\n" + + " (SELECT max(cl.oid)\n" + + " FROM pg_catalog.pg_class cl\n" + + " WHERE cl.relname = col.table_name and cl.relnamespace=(select oid from pg_namespace where nspname='%s'))) AS \"dataType\"\n" + + "FROM information_schema.columns col\n" + + "LEFT JOIN information_schema.sequences seq\n" + + " ON (seq.sequence_catalog=col.table_catalog and sequence_schema=col.table_schema and col.column_default=\n" + + " 'nextval('''||(case when sequence_schema='public' then '' else sequence_schema||'.' end)||sequence_name||'''::regclass)')\n" + + "WHERE col.table_catalog = '%s'\n" + + " AND col.table_schema = '%s' %s\n" + + "ORDER BY col.table_name, col.ordinal_position"; + + protected final static String PG_ALL_COLUMN_LOWER_VERSION = "SELECT\n" + " col.table_name \"tableName\",\n" + " col.column_name \"columnName\",\n" + @@ -193,10 +241,33 @@ public DataMap getTableInfo(String tableName) { " AND a.attnum = ANY(ix.indkey)\n" + " AND t.relkind IN ('r', 'p')\n" + " AND tt.table_name=t.relname\n" + + " AND tt.table_schema=(select nspname from pg_namespace where oid=t.relnamespace)\n" + " AND tt.table_catalog='%s'\n" + " AND tt.table_schema='%s' %s\n" + "ORDER BY t.relname, i.relname, a.attnum"; + protected final static String PG_ALL_FOREIGN_KEY = "select con.\"constraintName\",con.\"tableName\",con.\"referencesTableName\",con.\"onUpdate\",con.\"onDelete\",\n" + + "(select column_name from information_schema.columns where table_schema=con.nspname and table_name=con.\"tableName\" and ordinal_position=con.fk) fk,\n" + + "(select column_name from information_schema.columns where table_schema=con.nspname and table_name=con.\"referencesTableName\" and ordinal_position=con.rfk) rfk\n" + + "from\n" + + "(select c.conname \"constraintName\",pn.nspname,\n" + + "pc.relname \"tableName\",\n" + + "(select relname from pg_class where oid=c.confrelid) \"referencesTableName\",\n" + + "unnest(conkey) fk, unnest(confkey) rfk,\n" + + "(case c.confupdtype\n" + + " when 'a' then 'NO_ACTION'\n" + + " when 'n' then 'SET_NULL'\n" + + " when 'r' then 'RESTRICT'\n" + + " when 'c' then 'CASCADE' else 'SET_DEFAULT' end) \"onUpdate\",\n" + + "(case c.confdeltype\n" + + " when 'a' then 'NO_ACTION'\n" + + " when 'n' then 'SET_NULL'\n" + + " when 'r' then 'RESTRICT'\n" + + " when 'c' then 'CASCADE' else 'SET_DEFAULT' end) \"onDelete\"\n" + + "from pg_constraint c\n" + + "join pg_class pc on c.conrelid = pc.oid\n" + + "join pg_namespace pn on c.connamespace = pn.oid\n" + + "where c.contype='f' and pn.nspname='%s' %s) con"; protected final static String TABLE_INFO_SQL = "SELECT\n" + " pg_total_relation_size('\"' || table_schema || '\".\"' || table_name || '\"') AS size,\n" + diff --git a/connectors-common/postgres-core/src/main/java/io/tapdata/connector/postgres/PostgresSqlMaker.java b/connectors-common/postgres-core/src/main/java/io/tapdata/connector/postgres/PostgresSqlMaker.java index 26e032963..e0768b44d 100644 --- a/connectors-common/postgres-core/src/main/java/io/tapdata/connector/postgres/PostgresSqlMaker.java +++ b/connectors-common/postgres-core/src/main/java/io/tapdata/connector/postgres/PostgresSqlMaker.java @@ -4,6 +4,7 @@ import io.tapdata.connector.postgres.bean.PostgresColumn; import io.tapdata.entity.schema.TapField; import io.tapdata.kit.EmptyKit; +import io.tapdata.kit.StringKit; import io.tapdata.pdk.apis.entity.Collate; import io.tapdata.pdk.apis.entity.SortOn; import io.tapdata.pdk.apis.entity.TapAdvanceFilter; @@ -18,6 +19,14 @@ public class PostgresSqlMaker extends CommonSqlMaker { private char escapeChar = '"'; + protected void buildDataTypeDefinition(StringBuilder builder, TapField tapField) { + if (createAutoInc && Boolean.TRUE.equals(tapField.getAutoInc()) && tapField.getDataType().startsWith("numeric")) { + builder.append(escapeChar).append(tapField.getName()).append(escapeChar).append(" bigint "); + } else { + super.buildDataTypeDefinition(builder, tapField); + } + } + protected void buildNullDefinition(StringBuilder builder, TapField tapField) { boolean nullable = !(EmptyKit.isNotNull(tapField.getNullable()) && !tapField.getNullable()); if (closeNotNull) { @@ -29,12 +38,14 @@ protected void buildNullDefinition(StringBuilder builder, TapField tapField) { } protected void buildDefaultDefinition(StringBuilder builder, TapField tapField) { - if (EmptyKit.isNotNull(tapField.getDefaultValue()) && !"".equals(tapField.getDefaultValue())) { + if (EmptyKit.isNotNull(tapField.getDefaultValue())) { builder.append("DEFAULT").append(' '); if (EmptyKit.isNotNull(tapField.getDefaultFunction())) { builder.append(PostgresColumn.PostgresDefaultFunction.valueOf(tapField.getDefaultFunction().toString()).getFunction()).append(' '); } else if (tapField.getDefaultValue() instanceof Number) { builder.append(tapField.getDefaultValue()).append(' '); + } else if (EmptyKit.isNotBlank(tapField.getSequenceName())) { + builder.append("nextval('\"").append(StringKit.escape(schema, "'\"")).append("\".\"").append(StringKit.escape(tapField.getSequenceName(), "'\"")).append("\"') "); } else { builder.append("'").append(tapField.getDefaultValue()).append("' "); } @@ -42,6 +53,9 @@ protected void buildDefaultDefinition(StringBuilder builder, TapField tapField) } protected void buildAutoIncDefinition(StringBuilder builder, TapField tapField) { + if (EmptyKit.isNotBlank(tapField.getSequenceName())) { + return; + } long startValue; if (EmptyKit.isNotNull(tapField.getAutoIncStartValue())) { startValue = tapField.getAutoIncStartValue(); @@ -50,7 +64,9 @@ protected void buildAutoIncDefinition(StringBuilder builder, TapField tapField) } builder.append("GENERATED BY DEFAULT AS IDENTITY (START WITH ") .append(startValue) - .append(" INCREMENT BY 1) "); + .append(" INCREMENT BY 1 CACHE ") + .append(autoIncCacheValue) + .append(") "); } @Override @@ -71,14 +87,19 @@ public String buildKeyAndValue(Map record, String splitSymbol, S StringBuilder builder = new StringBuilder(); if (EmptyKit.isNotEmpty(record)) { record.forEach((fieldName, value) -> { - builder.append(escapeChar).append(fieldName).append(escapeChar).append(operator); - builder.append(buildValueString(value)); - Collate collate = EmptyKit.isEmpty(collateList) ? null : collateList.stream() - .filter(c -> c.getFieldName().equals(fieldName)) - .findFirst() - .orElse(null); - if (null != collate) { - builder.append(' ').append(buildCollate(collate.getCollateName())); + if (null != value) { + builder.append(escapeChar).append(fieldName).append(escapeChar).append(operator); + builder.append(buildValueString(value)); + Collate collate = EmptyKit.isEmpty(collateList) ? null : collateList.stream() + .filter(c -> c.getFieldName().equals(fieldName)) + .findFirst() + .orElse(null); + if (null != collate) { + builder.append(' ').append(buildCollate(collate.getCollateName())); + } + } else { + builder.append(escapeChar).append(fieldName).append(escapeChar).append(' '); + builder.append("IS NULL"); } builder.append(' ').append(splitSymbol).append(' '); }); diff --git a/connectors-common/postgres-core/src/main/java/io/tapdata/connector/postgres/PostgresTest.java b/connectors-common/postgres-core/src/main/java/io/tapdata/connector/postgres/PostgresTest.java index 8942fb212..3200f7289 100644 --- a/connectors-common/postgres-core/src/main/java/io/tapdata/connector/postgres/PostgresTest.java +++ b/connectors-common/postgres-core/src/main/java/io/tapdata/connector/postgres/PostgresTest.java @@ -5,6 +5,7 @@ import io.tapdata.connector.postgres.config.PostgresConfig; import io.tapdata.entity.simplify.TapSimplify; import io.tapdata.kit.EmptyKit; +import io.tapdata.kit.StringKit; import io.tapdata.pdk.apis.entity.ConnectionOptions; import io.tapdata.pdk.apis.entity.TestItem; import io.tapdata.pdk.apis.exception.testItem.TapTestCurrentTimeConsistentEx; @@ -40,11 +41,12 @@ public PostgresTest initContext() { } public PostgresTest withPostgresVersion(String version) { - ((PostgresJdbcContext)jdbcContext).withPostgresVersion(version); + ((PostgresJdbcContext) jdbcContext).withPostgresVersion(version); return this; } + public PostgresTest withPostgresVersion() throws SQLException { - ((PostgresJdbcContext)jdbcContext).withPostgresVersion(jdbcContext.queryVersion()); + ((PostgresJdbcContext) jdbcContext).withPostgresVersion(jdbcContext.queryVersion()); return this; } @@ -57,8 +59,8 @@ protected List supportVersions() { public Boolean testReadPrivilege() { try { AtomicInteger tableSelectPrivileges = new AtomicInteger(); - jdbcContext.queryWithNext(String.format(PG_TABLE_SELECT_NUM, commonDbConfig.getUser(), - commonDbConfig.getDatabase(), commonDbConfig.getSchema()), resultSet -> tableSelectPrivileges.set(resultSet.getInt(1))); + jdbcContext.queryWithNext(String.format(PG_TABLE_SELECT_NUM, StringKit.escape(commonDbConfig.getUser(), "'"), + StringKit.escape(commonDbConfig.getDatabase(), "'"), StringKit.escape(commonDbConfig.getSchema(), "'")), resultSet -> tableSelectPrivileges.set(resultSet.getInt(1))); if (tableSelectPrivileges.get() >= tableCount()) { consumer.accept(testItem(TestItem.ITEM_READ, TestItem.RESULT_SUCCESSFULLY, "All tables can be selected")); } else { @@ -114,7 +116,7 @@ public Boolean testWalMiner() { protected int tableCount() throws Throwable { AtomicInteger tableCount = new AtomicInteger(); - jdbcContext.queryWithNext(PG_TABLE_NUM, resultSet -> tableCount.set(resultSet.getInt(1))); + jdbcContext.queryWithNext(String.format(PG_TABLE_NUM, StringKit.escape(commonDbConfig.getSchema(), "'")), resultSet -> tableCount.set(resultSet.getInt(1))); return tableCount.get(); } @@ -129,7 +131,7 @@ protected int tableCount() throws Throwable { protected Boolean testWritePrivilege() { try { List sqls = new ArrayList<>(); - String schemaPrefix = EmptyKit.isNotEmpty(commonDbConfig.getSchema()) ? ("\"" + commonDbConfig.getSchema() + "\".") : ""; + String schemaPrefix = EmptyKit.isNotEmpty(commonDbConfig.getSchema()) ? ("\"" + StringKit.escape(commonDbConfig.getSchema(), "\"") + "\".") : ""; if (jdbcContext.queryAllTables(Arrays.asList(TEST_WRITE_TABLE, TEST_WRITE_TABLE.toUpperCase())).size() > 0) { sqls.add(String.format(TEST_DROP_TABLE, schemaPrefix + TEST_WRITE_TABLE)); } @@ -161,6 +163,7 @@ public Boolean testTimeDifference() { } return true; } + @Override protected Boolean testDatasourceInstanceInfo() { buildDatasourceInstanceInfo(connectionOptions); diff --git a/connectors-common/postgres-core/src/main/java/io/tapdata/connector/postgres/bean/PostgresColumn.java b/connectors-common/postgres-core/src/main/java/io/tapdata/connector/postgres/bean/PostgresColumn.java index c40b969ce..9e75d63b5 100644 --- a/connectors-common/postgres-core/src/main/java/io/tapdata/connector/postgres/bean/PostgresColumn.java +++ b/connectors-common/postgres-core/src/main/java/io/tapdata/connector/postgres/bean/PostgresColumn.java @@ -29,16 +29,39 @@ public PostgresColumn(DataMap dataMap) { this.pureDataType = this.dataType; } this.nullable = dataMap.getString("nullable"); + this.autoInc = dataMap.getString("autoInc"); + this.seedValue = dataMap.getString("seedValue"); + this.incrementValue = dataMap.getString("incrementValue"); + this.autoIncCacheValue = dataMap.getString("cacheValue"); + this.sequenceName = dataMap.getString("sequenceName"); this.remarks = dataMap.getString("columnComment"); - //create table in target has no need to set default value - this.columnDefaultValue = null; -// this.columnDefaultValue = getDefaultValue(dataMap.getString("column_default")); + this.columnDefaultValue = getDefaultValue(dataMap.getString("columnDefault")); + if (EmptyKit.isNotNull(this.sequenceName)) { + this.autoInc = "YES"; + this.seedValue = dataMap.getString("seedValue2"); + this.incrementValue = dataMap.getString("incrementValue2"); + this.columnDefaultValue = dataMap.getString("columnDefault"); + } } @Override public TapField getTapField() { - return new TapField(this.columnName, this.dataType).pureDataType(this.pureDataType).nullable(this.isNullable()). - defaultValue(columnDefaultValue).comment(this.remarks); + TapField field = new TapField(this.columnName, this.dataType).pureDataType(this.pureDataType) + .nullable(this.isNullable()).autoInc(isAutoInc()) + .defaultValue(columnDefaultValue).comment(this.remarks).sequenceName(sequenceName); + if (isAutoInc()) { + if (EmptyKit.isNotNull(seedValue)) { + field.autoIncStartValue(Long.parseLong(seedValue)); + } + if (EmptyKit.isNotNull(incrementValue)) { + field.autoIncrementValue(Long.parseLong(incrementValue)); + } + if (EmptyKit.isNotNull(autoIncCacheValue)) { + field.setAutoIncCacheValue(Long.parseLong(autoIncCacheValue)); + } + } + generateDefaultValue(field); + return field; } @Override @@ -46,19 +69,38 @@ protected Boolean isNullable() { return "YES".equals(this.nullable); } - private String getDefaultValue(String defaultValue) { + @Override + protected Boolean isAutoInc() { + return "YES".equals(this.autoInc); + } + + protected String getDefaultValue(String defaultValue) { + String res; if (EmptyKit.isNull(defaultValue) || defaultValue.startsWith("NULL::")) { return null; } else if (defaultValue.contains("::")) { - return defaultValue.substring(0, defaultValue.lastIndexOf("::")); + res = defaultValue.substring(0, defaultValue.lastIndexOf("::")); + } else { + res = defaultValue; + } + if (res.startsWith("\"")) { + res = StringKit.removeHeadTail(res, "\"", null); + } else if (res.startsWith("'")) { + res = StringKit.removeHeadTail(res, "'", null).replace("''", "'"); } else { - return defaultValue; + isString = false; } + return res; + } + + protected String parseDefaultFunction(String defaultValue) { + return PostgresDefaultFunction.parseFunction(columnDefaultValue); } public enum PostgresDefaultFunction { - _CURRENT_TIMESTAMP("current_timestamp"), - _CURRENT_USER("current_user"); + _CURRENT_TIMESTAMP("CURRENT_TIMESTAMP"), + _CURRENT_USER("CURRENT_USER"), + _GENERATE_UUID("gen_random_uuid()"); private final String function; private static final Map map = new HashMap<>(); diff --git a/connectors-common/postgres-core/src/main/java/io/tapdata/connector/postgres/cdc/config/PostgresDebeziumConfig.java b/connectors-common/postgres-core/src/main/java/io/tapdata/connector/postgres/cdc/config/PostgresDebeziumConfig.java index 94c3d24f9..9999bc5bf 100644 --- a/connectors-common/postgres-core/src/main/java/io/tapdata/connector/postgres/cdc/config/PostgresDebeziumConfig.java +++ b/connectors-common/postgres-core/src/main/java/io/tapdata/connector/postgres/cdc/config/PostgresDebeziumConfig.java @@ -3,7 +3,9 @@ import io.debezium.config.Configuration; import io.tapdata.connector.postgres.config.PostgresConfig; import io.tapdata.kit.EmptyKit; +import io.tapdata.kit.StringKit; +import java.net.URLEncoder; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -85,7 +87,7 @@ public Configuration create() { // .with("offset.storage.file.filename", "d:/cdc/offset/" + slotName + ".dat") //path must be changed with requirement .with("offset.flush.interval.ms", 60000) .with("name", slotName + "-postgres-connector") - .with("database.server.name", postgresConfig.getDatabase()) + .with("database.server.name", URLEncoder.encode(postgresConfig.getDatabase())) .with("database.hostname", postgresConfig.getHost()) .with("database.port", postgresConfig.getPort()) .with("database.user", postgresConfig.getUser()) @@ -113,7 +115,7 @@ public Configuration create() { .with("geometry.schema.name", "io.debezium.postgresql.type.Geometry") .with("other.type", "io.tapdata.connector.postgres.converters.OtherConverter") .with("other.schema.name", "io.debezium.postgresql.type.Other") - .with("heartbeat.interval.ms", 3000) +// .with("heartbeat.interval.ms", 3000) .with("plugin.name", postgresConfig.getLogPluginName()) .with("max.queue.size", postgresConfig.getMaximumQueueSize()) .with("max.batch.size", postgresConfig.getMaximumQueueSize() / 8); @@ -130,12 +132,12 @@ public Configuration create() { } if (EmptyKit.isNotEmpty(observedTableList)) { //construct tableWhiteList with schema.table(,) as - String tableWhiteList = observedTableList.stream().map(v -> postgresConfig.getSchema() + "." + v).collect(Collectors.joining(", ")); + String tableWhiteList = observedTableList.stream().map(v -> StringKit.escapeRegex(postgresConfig.getSchema()) + "." + StringKit.escapeRegex(v)).collect(Collectors.joining(", ")); builder.with("table.whitelist", tableWhiteList); } if (EmptyKit.isNotEmpty(schemaTableMap)) { //construct tableWhiteList with schema.table(,) as - String tableWhiteList = schemaTableMap.entrySet().stream().map(v -> v.getValue().stream().map(l -> v.getKey() + "." + l).collect(Collectors.joining(", "))).collect(Collectors.joining(", ")); + String tableWhiteList = schemaTableMap.entrySet().stream().map(v -> v.getValue().stream().map(l -> StringKit.escapeRegex(v.getKey()) + "." + StringKit.escapeRegex(l)).collect(Collectors.joining(", "))).collect(Collectors.joining(", ")); builder.with("table.whitelist", tableWhiteList); } return builder.build(); diff --git a/connectors-common/postgres-core/src/main/java/io/tapdata/connector/postgres/config/PostgresConfig.java b/connectors-common/postgres-core/src/main/java/io/tapdata/connector/postgres/config/PostgresConfig.java index a29224b3b..2324d1037 100644 --- a/connectors-common/postgres-core/src/main/java/io/tapdata/connector/postgres/config/PostgresConfig.java +++ b/connectors-common/postgres-core/src/main/java/io/tapdata/connector/postgres/config/PostgresConfig.java @@ -21,6 +21,7 @@ public class PostgresConfig extends CommonDbConfig implements Serializable { private String logPluginName = "pgoutput"; //default log plugin for postgres, pay attention to lower version private Boolean closeNotNull = false; + private String tableOwner = ""; private String replaceBlank = ""; private Boolean partitionRoot = false; private Integer maximumQueueSize = 8000; @@ -83,6 +84,14 @@ public void setCloseNotNull(Boolean closeNotNull) { this.closeNotNull = closeNotNull; } + public String getTableOwner() { + return tableOwner; + } + + public void setTableOwner(String tableOwner) { + this.tableOwner = tableOwner; + } + public String getReplaceBlank() { return replaceBlank; } diff --git a/connectors-common/postgres-core/src/main/java/io/tapdata/connector/postgres/dml/PostgresRecordWriter.java b/connectors-common/postgres-core/src/main/java/io/tapdata/connector/postgres/dml/PostgresRecordWriter.java index 011c5e225..4d3f9c88b 100644 --- a/connectors-common/postgres-core/src/main/java/io/tapdata/connector/postgres/dml/PostgresRecordWriter.java +++ b/connectors-common/postgres-core/src/main/java/io/tapdata/connector/postgres/dml/PostgresRecordWriter.java @@ -7,6 +7,7 @@ import java.sql.Connection; import java.sql.SQLException; +import java.sql.Statement; public class PostgresRecordWriter extends NormalRecordWriter { @@ -46,4 +47,16 @@ public PostgresRecordWriter(JdbcContext jdbcContext, Connection connection, TapT } } + public void closeConstraintCheck() { + String sql = "SET session_replication_role = 'replica'"; + try (Statement statement = connection.createStatement()) { + statement.execute(sql); + } catch (Exception e) { + try { + connection.rollback(); + } catch (SQLException ignored) { + } + } + } + } diff --git a/connectors-common/postgres-core/src/main/java/io/tapdata/connector/postgres/dml/PostgresWriteRecorder.java b/connectors-common/postgres-core/src/main/java/io/tapdata/connector/postgres/dml/PostgresWriteRecorder.java index ffde71d96..348ed7866 100644 --- a/connectors-common/postgres-core/src/main/java/io/tapdata/connector/postgres/dml/PostgresWriteRecorder.java +++ b/connectors-common/postgres-core/src/main/java/io/tapdata/connector/postgres/dml/PostgresWriteRecorder.java @@ -98,40 +98,40 @@ protected void insertUpdate(Map after, Map befor protected String getInsertUpdateSql(boolean containsNull) { if (!containsNull) { - return "WITH upsert AS (UPDATE " + escapeChar + schema + escapeChar + "." + escapeChar + tapTable.getId() + escapeChar + " SET " + updatedColumn.stream().map(k -> escapeChar + k + escapeChar + "=?") - .collect(Collectors.joining(", ")) + " WHERE " + uniqueCondition.stream().map(k -> escapeChar + k + escapeChar + "=?") - .collect(Collectors.joining(" AND ")) + " RETURNING *) INSERT INTO " + escapeChar + schema + escapeChar + "." + escapeChar + tapTable.getId() + escapeChar + " (" - + allColumn.stream().map(k -> escapeChar + k + escapeChar).collect(Collectors.joining(", ")) + ") SELECT " + return "WITH upsert AS (UPDATE " + getSchemaAndTable() + " SET " + updatedColumn.stream().map(k -> quoteAndEscape(k) + "=?") + .collect(Collectors.joining(", ")) + " WHERE " + uniqueCondition.stream().map(k -> quoteAndEscape(k) + "=?") + .collect(Collectors.joining(" AND ")) + " RETURNING *) INSERT INTO " + getSchemaAndTable() + " (" + + allColumn.stream().map(this::quoteAndEscape).collect(Collectors.joining(", ")) + ") SELECT " + StringKit.copyString("?", allColumn.size(), ",") + " WHERE NOT EXISTS (SELECT * FROM upsert)"; } else { - return "WITH upsert AS (UPDATE " + escapeChar + schema + escapeChar + "." + escapeChar + tapTable.getId() + escapeChar + " SET " + updatedColumn.stream().map(k -> escapeChar + k + escapeChar + "=?") - .collect(Collectors.joining(", ")) + " WHERE " + uniqueCondition.stream().map(k -> "(" + escapeChar + k + escapeChar + "=? OR (" + escapeChar + k + escapeChar + " IS NULL AND ?::text IS NULL))") - .collect(Collectors.joining(" AND ")) + " RETURNING *) INSERT INTO " + escapeChar + schema + escapeChar + "." + escapeChar + tapTable.getId() + escapeChar + " (" - + allColumn.stream().map(k -> escapeChar + k + escapeChar).collect(Collectors.joining(", ")) + ") SELECT " + return "WITH upsert AS (UPDATE " + getSchemaAndTable() + " SET " + updatedColumn.stream().map(k -> quoteAndEscape(k) + "=?") + .collect(Collectors.joining(", ")) + " WHERE " + uniqueCondition.stream().map(k -> "(" + quoteAndEscape(k) + "=? OR (" + quoteAndEscape(k) + " IS NULL AND ?::text IS NULL))") + .collect(Collectors.joining(" AND ")) + " RETURNING *) INSERT INTO " + getSchemaAndTable() + " (" + + allColumn.stream().map(this::quoteAndEscape).collect(Collectors.joining(", ")) + ") SELECT " + StringKit.copyString("?", allColumn.size(), ",") + " WHERE NOT EXISTS (SELECT * FROM upsert)"; } } protected String getUpdateSql(Map after, Map before, boolean containsNull) { if (!containsNull) { - return "UPDATE " + escapeChar + schema + escapeChar + "." + escapeChar + tapTable.getId() + escapeChar + " SET " + - after.keySet().stream().map(k -> escapeChar + k + escapeChar + "=?").collect(Collectors.joining(", ")) + " WHERE " + - before.keySet().stream().map(k -> escapeChar + k + escapeChar + "=?").collect(Collectors.joining(" AND ")); + return "UPDATE " + getSchemaAndTable() + " SET " + + after.keySet().stream().map(k -> quoteAndEscape(k) + "=?").collect(Collectors.joining(", ")) + " WHERE " + + before.keySet().stream().map(k -> quoteAndEscape(k) + "=?").collect(Collectors.joining(" AND ")); } else { - return "UPDATE " + escapeChar + schema + escapeChar + "." + escapeChar + tapTable.getId() + escapeChar + " SET " + - after.keySet().stream().map(k -> escapeChar + k + escapeChar + "=?").collect(Collectors.joining(", ")) + " WHERE " + - before.keySet().stream().map(k -> "(" + escapeChar + k + escapeChar + "=? OR (" + escapeChar + k + escapeChar + " IS NULL AND ?::text IS NULL))") + return "UPDATE " + getSchemaAndTable() + " SET " + + after.keySet().stream().map(k -> quoteAndEscape(k) + "=?").collect(Collectors.joining(", ")) + " WHERE " + + before.keySet().stream().map(k -> "(" + quoteAndEscape(k) + "=? OR (" + quoteAndEscape(k) + " IS NULL AND ?::text IS NULL))") .collect(Collectors.joining(" AND ")); } } protected String getDeleteSql(Map before, boolean containsNull) { if (!containsNull) { - return "DELETE FROM " + escapeChar + schema + escapeChar + "." + escapeChar + tapTable.getId() + escapeChar + " WHERE " + - before.keySet().stream().map(k -> escapeChar + k + escapeChar + "=?").collect(Collectors.joining(" AND ")); + return "DELETE FROM " + getSchemaAndTable() + " WHERE " + + before.keySet().stream().map(k -> quoteAndEscape(k) + "=?").collect(Collectors.joining(" AND ")); } else { - return "DELETE FROM " + escapeChar + schema + escapeChar + "." + escapeChar + tapTable.getId() + escapeChar + " WHERE " + - before.keySet().stream().map(k -> "(" + escapeChar + k + escapeChar + "=? OR (" + escapeChar + k + escapeChar + " IS NULL AND ?::text IS NULL))") + return "DELETE FROM " + getSchemaAndTable() + " WHERE " + + before.keySet().stream().map(k -> "(" + quoteAndEscape(k) + "=? OR (" + quoteAndEscape(k) + " IS NULL AND ?::text IS NULL))") .collect(Collectors.joining(" AND ")); } } @@ -203,7 +203,7 @@ public void addAndCheckCommit(TapRecordEvent recordEvent, WriteListResult after, WriteListResult } protected String getUpsertSql() { - return "INSERT INTO " + escapeChar + schema + escapeChar + "." + escapeChar + tapTable.getId() + escapeChar + " (" - + allColumn.stream().map(k -> escapeChar + k + escapeChar).collect(Collectors.joining(", ")) + ") " + + return "INSERT INTO " + getSchemaAndTable() + " (" + + allColumn.stream().map(this::quoteAndEscape).collect(Collectors.joining(", ")) + ") " + "VALUES(" + StringKit.copyString("?", allColumn.size(), ",") + ") ON CONFLICT(" - + uniqueCondition.stream().map(k -> escapeChar + k + escapeChar).collect(Collectors.joining(", ")) - + ") DO UPDATE SET " + updatedColumn.stream().map(k -> escapeChar + k + escapeChar + "=?").collect(Collectors.joining(", ")); + + uniqueCondition.stream().map(this::quoteAndEscape).collect(Collectors.joining(", ")) + + ") DO UPDATE SET " + updatedColumn.stream().map(k -> quoteAndEscape(k) + "=?").collect(Collectors.joining(", ")); } @Override @@ -275,10 +275,10 @@ protected void insertIgnore(Map after, WriteListResult escapeChar + k + escapeChar).collect(Collectors.joining(", ")) + ") " + + return "INSERT INTO " + getSchemaAndTable() + " (" + + allColumn.stream().map(this::quoteAndEscape).collect(Collectors.joining(", ")) + ") " + "VALUES(" + StringKit.copyString("?", allColumn.size(), ",") + ") ON CONFLICT(" - + uniqueCondition.stream().map(k -> escapeChar + k + escapeChar).collect(Collectors.joining(", ")) + + uniqueCondition.stream().map(this::quoteAndEscape).collect(Collectors.joining(", ")) + ") DO NOTHING "; } @@ -360,15 +360,15 @@ protected void oldInsertIgnore(Map after, WriteListResult "\"" + k + "\"").collect(Collectors.joining(", ")) + ") SELECT " - + StringKit.copyString("?", allColumn.size(), ",") + " WHERE NOT EXISTS (SELECT 1 FROM \"" + schema + "\".\"" + tapTable.getId() - + "\" WHERE " + uniqueCondition.stream().map(k -> "\"" + k + "\"=?").collect(Collectors.joining(" AND ")) + " )"; + return "INSERT INTO " + getSchemaAndTable() + " (" + + allColumn.stream().map(this::quoteAndEscape).collect(Collectors.joining(", ")) + ") SELECT " + + StringKit.copyString("?", allColumn.size(), ",") + " WHERE NOT EXISTS (SELECT 1 FROM " + getSchemaAndTable() + + " WHERE " + uniqueCondition.stream().map(k -> quoteAndEscape(k) + "=?").collect(Collectors.joining(" AND ")) + " )"; } else { - return "INSERT INTO \"" + schema + "\".\"" + tapTable.getId() + "\" (" - + allColumn.stream().map(k -> "\"" + k + "\"").collect(Collectors.joining(", ")) + ") SELECT " - + StringKit.copyString("?", allColumn.size(), ",") + " WHERE NOT EXISTS (SELECT 1 FROM \"" + schema + "\".\"" + tapTable.getId() - + "\" WHERE " + uniqueCondition.stream().map(k -> "(\"" + k + "\"=? OR (\"" + k + "\" IS NULL AND ?::text IS NULL))").collect(Collectors.joining(" AND ")) + " )"; + return "INSERT INTO " + getSchemaAndTable() + " (" + + allColumn.stream().map(this::quoteAndEscape).collect(Collectors.joining(", ")) + ") SELECT " + + StringKit.copyString("?", allColumn.size(), ",") + " WHERE NOT EXISTS (SELECT 1 FROM " + getSchemaAndTable() + + " WHERE " + uniqueCondition.stream().map(k -> "(" + quoteAndEscape(k) + "=? OR (" + quoteAndEscape(k) + " IS NULL AND ?::text IS NULL))").collect(Collectors.joining(" AND ")) + " )"; } } } diff --git a/connectors-common/postgres-core/src/main/java/io/tapdata/connector/postgres/exception/PostgresExceptionCollector.java b/connectors-common/postgres-core/src/main/java/io/tapdata/connector/postgres/exception/PostgresExceptionCollector.java index d4e339ff4..45b65a0ae 100644 --- a/connectors-common/postgres-core/src/main/java/io/tapdata/connector/postgres/exception/PostgresExceptionCollector.java +++ b/connectors-common/postgres-core/src/main/java/io/tapdata/connector/postgres/exception/PostgresExceptionCollector.java @@ -111,6 +111,11 @@ public void collectViolateNull(String targetFieldName, Throwable cause) { } } + @Override + public boolean violateIndexName(Throwable cause) { + return cause instanceof SQLException && "42P07".equals(((SQLException) cause).getSQLState()); + } + @Override public void collectCdcConfigInvalid(Throwable cause) { if (cause instanceof SQLException) { diff --git a/connectors-common/read-partition/pom.xml b/connectors-common/read-partition/pom.xml index c23bc5856..79735faa0 100644 --- a/connectors-common/read-partition/pom.xml +++ b/connectors-common/read-partition/pom.xml @@ -20,13 +20,13 @@ io.tapdata tapdata-api - 1.4.1-SNAPSHOT + 1.4.4-SNAPSHOT provided io.tapdata tapdata-pdk-api - 1.4.1-SNAPSHOT + 1.4.4-SNAPSHOT provided diff --git a/connectors-common/sql-core/src/main/java/io/tapdata/common/CommonColumn.java b/connectors-common/sql-core/src/main/java/io/tapdata/common/CommonColumn.java index c67c54bc9..6fe2028a6 100644 --- a/connectors-common/sql-core/src/main/java/io/tapdata/common/CommonColumn.java +++ b/connectors-common/sql-core/src/main/java/io/tapdata/common/CommonColumn.java @@ -2,6 +2,9 @@ import io.tapdata.entity.schema.TapField; import io.tapdata.entity.utils.DataMap; +import io.tapdata.kit.EmptyKit; + +import java.math.BigDecimal; /** * attributes for common columns @@ -17,10 +20,15 @@ public class CommonColumn { protected String remarks; protected String columnDefaultValue; protected String autoInc; + protected String seedValue; + protected String incrementValue; + protected String autoIncCacheValue; protected String pureDataType; protected Integer dataLength; protected Integer dataPrecision; protected Integer dataScale; + protected boolean isString = true; + protected String sequenceName; public CommonColumn() { } @@ -34,7 +42,7 @@ public CommonColumn(DataMap dataMap) { this.dataScale = dataMap.getInteger("dataScale"); this.nullable = dataMap.getString("nullable"); this.remarks = dataMap.getString("columnComment"); - this.columnDefaultValue = null; + this.columnDefaultValue = getDefaultValue(dataMap.getString("columnDefault")); } protected Boolean isNullable() { @@ -49,4 +57,26 @@ public TapField getTapField() { return new TapField(this.columnName, this.dataType).nullable(this.isNullable()). defaultValue(columnDefaultValue).comment(this.remarks); } + + protected String getDefaultValue(String defaultValue) { + return null; + } + + protected void generateDefaultValue(TapField field) { + Object defaultValueObj = columnDefaultValue; + String tapDefaultFunction = null; + if (EmptyKit.isNotNull(columnDefaultValue)) { + tapDefaultFunction = parseDefaultFunction(columnDefaultValue); + if (!isString && columnDefaultValue.matches("-?\\d+(\\.\\d+)?")) { + defaultValueObj = new BigDecimal(columnDefaultValue); + } else { + defaultValueObj = columnDefaultValue; + } + } + field.defaultValue(defaultValueObj).defaultFunction(tapDefaultFunction); + } + + protected String parseDefaultFunction(String defaultValue) { + return null; + } } diff --git a/connectors-common/sql-core/src/main/java/io/tapdata/common/CommonDbConfig.java b/connectors-common/sql-core/src/main/java/io/tapdata/common/CommonDbConfig.java index cd04eec53..afc7ee2b6 100644 --- a/connectors-common/sql-core/src/main/java/io/tapdata/common/CommonDbConfig.java +++ b/connectors-common/sql-core/src/main/java/io/tapdata/common/CommonDbConfig.java @@ -45,6 +45,7 @@ public class CommonDbConfig implements Serializable { private Boolean oldVersionTimezone = false; private Boolean createAutoInc = false; private long autoIncJumpValue = 1000000L; + private long autoIncCacheValue = 100; private Boolean applyDefault = false; private Integer writeThreadSize = 15; protected String timezone = "+00:00"; @@ -305,6 +306,14 @@ public void setAutoIncJumpValue(long autoIncJumpValue) { this.autoIncJumpValue = autoIncJumpValue; } + public long getAutoIncCacheValue() { + return autoIncCacheValue; + } + + public void setAutoIncCacheValue(long autoIncCacheValue) { + this.autoIncCacheValue = autoIncCacheValue; + } + public Boolean getApplyDefault() { return applyDefault; } diff --git a/connectors-common/sql-core/src/main/java/io/tapdata/common/CommonDbConnector.java b/connectors-common/sql-core/src/main/java/io/tapdata/common/CommonDbConnector.java index f0e1f5b5f..8768f9c66 100644 --- a/connectors-common/sql-core/src/main/java/io/tapdata/common/CommonDbConnector.java +++ b/connectors-common/sql-core/src/main/java/io/tapdata/common/CommonDbConnector.java @@ -4,20 +4,21 @@ import io.tapdata.common.ddl.DDLSqlGenerator; import io.tapdata.common.exception.AbstractExceptionCollector; import io.tapdata.common.exception.ExceptionCollector; +import io.tapdata.entity.TapConstraintException; import io.tapdata.entity.event.TapEvent; +import io.tapdata.entity.event.ddl.constraint.TapCreateConstraintEvent; +import io.tapdata.entity.event.ddl.constraint.TapDropConstraintEvent; import io.tapdata.entity.event.ddl.index.TapCreateIndexEvent; import io.tapdata.entity.event.ddl.index.TapDeleteIndexEvent; import io.tapdata.entity.event.ddl.table.*; import io.tapdata.entity.logger.Log; -import io.tapdata.entity.schema.TapField; -import io.tapdata.entity.schema.TapIndex; -import io.tapdata.entity.schema.TapIndexField; -import io.tapdata.entity.schema.TapTable; +import io.tapdata.entity.schema.*; import io.tapdata.entity.simplify.TapSimplify; import io.tapdata.entity.simplify.pretty.BiClassHandlers; import io.tapdata.entity.utils.DataMap; import io.tapdata.kit.DbKit; import io.tapdata.kit.EmptyKit; +import io.tapdata.kit.StringKit; import io.tapdata.pdk.apis.context.TapConnectionContext; import io.tapdata.pdk.apis.context.TapConnectorContext; import io.tapdata.pdk.apis.entity.FilterResult; @@ -51,7 +52,10 @@ public abstract class CommonDbConnector extends ConnectorBase { //offset for Primary key sorting area reading private final static Long offsetSize = 1000000L; protected static final int BATCH_ADVANCE_READ_LIMIT = 1000; - + protected Map writtenTableMap = new ConcurrentHashMap<>(); + protected static final String HAS_UNIQUE_INDEX = "HAS_UNIQUE_INDEX"; + protected static final String HAS_AUTO_INCR = "HAS_AUTO_INCR"; + protected static final String HAS_REMOVED_COLUMN = "HAS_REMOVED_COLUMN"; //ddlHandlers which for ddl collection protected BiClassHandlers> fieldDDLHandlers; //ddlSqlMaker which for ddl execution @@ -84,7 +88,7 @@ public void discoverSchema(TapConnectionContext connectionContext, List * when your connector need to support partition table and main-curl table * you should impl this function to discover those tablies relations */ - public List discoverPartitionInfo(List tapTableList) { + protected List discoverPartitionInfo(List tapTableList) { return tapTableList; } @@ -94,6 +98,7 @@ protected void singleThreadDiscoverSchema(List subList, Consumer subTableNames = subList.stream().map(v -> v.getString("tableName")).collect(Collectors.toList()); List columnList = jdbcContext.queryAllColumns(subTableNames); List indexList = jdbcContext.queryAllIndexes(subTableNames); + List fkList = jdbcContext.queryAllForeignKeys(subTableNames); subList.forEach(subTable -> { //2、table name/comment String table = subTable.getString("tableName"); @@ -119,6 +124,7 @@ protected void singleThreadDiscoverSchema(List subList, Consumer subList, Consumer indexList, String table, List primaryKey, List tapIndexList) { - Map> indexMap = indexList.stream().filter(idx -> table.equals(idx.getString("tableName"))) + Map> indexMap = indexList.stream().filter(idx -> table.equals(idx.getString("tableName")) && EmptyKit.isNotBlank(idx.getString("indexName"))) .collect(Collectors.groupingBy(idx -> idx.getString("indexName"), LinkedHashMap::new, Collectors.toList())); indexMap.forEach((key, value) -> { if (value.stream().anyMatch(v -> ("1".equals(v.getString("isPk"))))) { @@ -140,6 +146,12 @@ protected TapField makeTapField(DataMap dataMap) { return new CommonColumn(dataMap).getTapField(); } + protected List makeForeignKey(List fkList, String table) { + List tapConstraints = new ArrayList<>(); + fkList.stream().filter(v -> Objects.nonNull(v) && table.equals(v.getString("tableName"))).collect(Collectors.groupingBy(map -> map.getString("constraintName"))).forEach((constraintName, fk) -> tapConstraints.add(makeTapConstraint(constraintName, fk))); + return tapConstraints; + } + protected Map getSpecificAttr(DataMap dataMap) { return null; } @@ -161,11 +173,14 @@ protected CreateTableOptions createTable(TapConnectorContext connectorContext, T Map fieldMap = tapTable.getNameFieldMap(); for (String field : fieldMap.keySet()) { - String fieldDefault = (String) fieldMap.get(field).getDefaultValue(); - if (EmptyKit.isNotEmpty(fieldDefault)) { - if (fieldDefault.contains("'")) { - fieldDefault = fieldDefault.replaceAll("'", "''"); - fieldMap.get(field).setDefaultValue(fieldDefault); + Object defaultValue = fieldMap.get(field).getDefaultValue(); + if (defaultValue instanceof String) { + String fieldDefault = (String) fieldMap.get(field).getDefaultValue(); + if (EmptyKit.isNotEmpty(fieldDefault) && !Boolean.TRUE.equals(fieldMap.get(field).getAutoInc())) { + if (fieldDefault.contains("'")) { + fieldDefault = fieldDefault.replaceAll("'", "''"); + fieldMap.get(field).setDefaultValue(fieldDefault); + } } } } @@ -412,20 +427,62 @@ protected void createIndex(TapConnectorContext connectorContext, TapTable tapTab .collect(Collectors.toList()); if (EmptyKit.isNotEmpty(indexList)) { indexList.stream().filter(i -> !i.isPrimary()).forEach(i -> { + String sql = getCreateIndexSql(tapTable, i); try { - jdbcContext.execute(getCreateIndexSql(tapTable, i)); + jdbcContext.execute(sql); } catch (SQLException e) { - String rename = i.getName() + "_" + UUID.randomUUID().toString().replaceAll("-", "").substring(28); - tapLogger.warn("Create index failed {}, rename {} to {} and retry ...", e.getMessage(), i.getName(), rename); - i.setName(rename); - String sql = getCreateIndexSql(tapTable, i); + if (!exceptionCollector.violateIndexName(e)) { + tapLogger.warn("Create index failed {}, please execute it manually [{}]", e.getMessage(), sql); + } else { + String rename = i.getName() + "_" + UUID.randomUUID().toString().replaceAll("-", "").substring(28); + tapLogger.warn("Create index failed {}, rename {} to {} and retry ...", e.getMessage(), i.getName(), rename); + i.setName(rename); + sql = getCreateIndexSql(tapTable, i); + try { + jdbcContext.execute(sql); + } catch (SQLException e1) { + tapLogger.warn("Create index failed again {}, please execute it manually [{}]", e1.getMessage(), sql); + } + } + } + }); + } + } + + protected void createConstraint(TapConnectorContext connectorContext, TapTable tapTable, TapCreateConstraintEvent createConstraintEvent, boolean create) { + List constraintList = createConstraintEvent.getConstraintList(); + if (EmptyKit.isNotEmpty(constraintList)) { + List constraintSqlList = new ArrayList<>(); + TapConstraintException exception = new TapConstraintException(tapTable.getId()); + constraintList.forEach(c -> { + String sql = getCreateConstraintSql(tapTable, c); + if (create) { try { jdbcContext.execute(sql); - } catch (SQLException e1) { - tapLogger.warn("Create index failed again {}, please execute it manually [{}]", e1.getMessage(), sql); + } catch (Exception e) { + if (!exceptionCollector.violateConstraintName(e)) { + exception.addException(c, sql, e); + } else { + String rename = c.getName() + "_" + UUID.randomUUID().toString().replaceAll("-", "").substring(28); + c.setName(rename); + sql = getCreateConstraintSql(tapTable, c); + try { + jdbcContext.execute(sql); + } catch (Exception e1) { + exception.addException(c, sql, e1); + } + } } + } else { + constraintSqlList.add(sql); } }); + if (!create) { + createConstraintEvent.setConstraintSqlList(constraintSqlList); + } + if (EmptyKit.isNotEmpty(exception.getExceptions())) { + throw exception; + } } } @@ -459,6 +516,39 @@ protected List discoverIndex(String tableName) { return tapIndexList; } + protected TapConstraint makeTapConstraint(String key, List value) { + TapConstraint tapConstraint = new TapConstraint(key, TapConstraint.ConstraintType.FOREIGN_KEY); + value.forEach(f -> { + tapConstraint.referencesTable(f.getString("referencesTableName")); + tapConstraint.add(new TapConstraintMapping() + .foreignKey(f.getString("fk")) + .referenceKey(f.getString("rfk"))); + if (EmptyKit.isNotBlank(f.getString("onUpdate"))) { + tapConstraint.onUpdate(f.getString("onUpdate")); + } + if (EmptyKit.isNotBlank(f.getString("onDelete"))) { + tapConstraint.onDelete(f.getString("onDelete")); + } + }); + return tapConstraint; + } + + protected List discoverConstraint(String tableName) { + List constraintList; + try { + constraintList = jdbcContext.queryAllForeignKeys(Collections.singletonList(tableName)); + } catch (SQLException e) { + throw new RuntimeException(e); + } + return makeForeignKey(constraintList, tableName); + } + + protected void beforeWriteRecord(TapTable tapTable) throws SQLException { + if (EmptyKit.isNull(writtenTableMap.get(tapTable.getId()))) { + writtenTableMap.put(tapTable.getId(), DataMap.create()); + } + } + protected void fieldDDLHandler(TapConnectorContext tapConnectorContext, TapFieldBaseEvent tapFieldBaseEvent) throws SQLException { List sqlList = fieldDDLHandlers.handle(tapFieldBaseEvent, tapConnectorContext); if (null == sqlList) { @@ -508,9 +598,9 @@ protected String getSchemaAndTable(String tableId) { StringBuilder sb = new StringBuilder(); char escapeChar = commonDbConfig.getEscapeChar(); if (EmptyKit.isNotBlank(commonDbConfig.getSchema())) { - sb.append(escapeChar).append(commonDbConfig.getSchema()).append(escapeChar).append('.'); + sb.append(escapeChar).append(StringKit.escape(commonDbConfig.getSchema(), escapeChar)).append(escapeChar).append('.'); } - sb.append(escapeChar).append(tableId).append(escapeChar); + sb.append(escapeChar).append(StringKit.escape(tableId, escapeChar)).append(escapeChar); return sb.toString(); } @@ -521,16 +611,20 @@ private String getCreateTableSql(TapTable tapTable, Boolean commentInField) { Collection primaryKeys = tapTable.primaryKeys(); if (EmptyKit.isNotEmpty(primaryKeys)) { sb.append(", primary key (").append(escapeChar) - .append(String.join(escapeChar + "," + escapeChar, primaryKeys)) + .append(primaryKeys.stream().map(pk -> StringKit.escape(pk, escapeChar)).collect(Collectors.joining(escapeChar + "," + escapeChar))) .append(escapeChar).append(')'); } sb.append(')'); if (commentInField && EmptyKit.isNotBlank(tapTable.getComment())) { - sb.append(" comment='").append(tapTable.getComment().replaceAll("'", "''")).append("'"); + commentOnTable(sb, tapTable); } return sb.toString(); } + protected void commentOnTable(StringBuilder sb, TapTable tapTable) { + sb.append(" comment='").append(tapTable.getComment().replaceAll("'", "''")).append("'"); + } + protected String getCreateIndexSql(TapTable tapTable, TapIndex tapIndex) { StringBuilder sb = new StringBuilder("create "); char escapeChar = commonDbConfig.getEscapeChar(); @@ -541,7 +635,9 @@ protected String getCreateIndexSql(TapTable tapTable, TapIndex tapIndex) { if (EmptyKit.isNotBlank(tapIndex.getName())) { sb.append(escapeChar).append(tapIndex.getName()).append(escapeChar); } else { - sb.append(escapeChar).append(DbKit.buildIndexName(tapTable.getId())).append(escapeChar); + String indexName = DbKit.buildIndexName(tapTable.getId()); + tapIndex.setName(indexName); + sb.append(escapeChar).append(indexName).append(escapeChar); } sb.append(" on ").append(getSchemaAndTable(tapTable.getId())).append('(') .append(tapIndex.getIndexFields().stream().map(f -> escapeChar + f.getName() + escapeChar + " " + (f.getFieldAsc() ? "asc" : "desc")) @@ -549,6 +645,26 @@ protected String getCreateIndexSql(TapTable tapTable, TapIndex tapIndex) { return sb.toString(); } + protected String getCreateConstraintSql(TapTable tapTable, TapConstraint tapConstraint) { + char escapeChar = commonDbConfig.getEscapeChar(); + StringBuilder sb = new StringBuilder("alter table "); + sb.append(getSchemaAndTable(tapTable.getId())).append(" add constraint "); + if (EmptyKit.isNotBlank(tapConstraint.getName())) { + sb.append(escapeChar).append(tapConstraint.getName()).append(escapeChar); + } else { + sb.append(escapeChar).append(DbKit.buildForeignKeyName(tapTable.getId())).append(escapeChar); + } + sb.append(" foreign key (").append(escapeChar).append(tapConstraint.getMappingFields().stream().map(TapConstraintMapping::getForeignKey).collect(Collectors.joining(escapeChar + "," + escapeChar))).append(escapeChar).append(") references ") + .append(getSchemaAndTable(tapConstraint.getReferencesTableName())).append('(').append(escapeChar).append(tapConstraint.getMappingFields().stream().map(TapConstraintMapping::getReferenceKey).collect(Collectors.joining(escapeChar + "," + escapeChar))).append(escapeChar).append(')'); + if (EmptyKit.isNotNull(tapConstraint.getOnUpdate())) { + sb.append(" on update ").append(tapConstraint.getOnUpdate().toString().replaceAll("_", " ")); + } + if (EmptyKit.isNotNull(tapConstraint.getOnDelete())) { + sb.append(" on delete ").append(tapConstraint.getOnDelete().toString().replaceAll("_", " ")); + } + return sb.toString(); + } + private String getTableCommentSql(TapTable tapTable) { return "comment on table " + getSchemaAndTable(tapTable.getId()) + " is '" + tapTable.getComment().replace("'", "''") + '\''; @@ -631,7 +747,7 @@ protected String getHashSplitModConditions(TapTable tapTable, int maxSplit, int } protected String getBatchReadSelectSql(TapTable tapTable) { - String columns = tapTable.getNameFieldMap().keySet().stream().map(c -> commonDbConfig.getEscapeChar() + c + commonDbConfig.getEscapeChar()).collect(Collectors.joining(",")); + String columns = tapTable.getNameFieldMap().keySet().stream().map(c -> commonDbConfig.getEscapeChar() + StringKit.escape(c, commonDbConfig.getEscapeChar()) + commonDbConfig.getEscapeChar()).collect(Collectors.joining(",")); return String.format("SELECT %s FROM " + getSchemaAndTable(tapTable.getId()), columns); } @@ -794,6 +910,10 @@ protected void queryIndexes(TapConnectorContext connectorContext, TapTable table consumer.accept(discoverIndex(table.getId())); } + protected void queryConstraint(TapConnectorContext connectorContext, TapTable table, Consumer> consumer) throws Throwable { + consumer.accept(discoverConstraint(table.getId())); + } + protected void dropIndexes(TapConnectorContext connectorContext, TapTable table, TapDeleteIndexEvent deleteIndexEvent) throws SQLException { char escapeChar = commonDbConfig.getEscapeChar(); List dropIndexesSql = new ArrayList<>(); @@ -801,6 +921,13 @@ protected void dropIndexes(TapConnectorContext connectorContext, TapTable table, jdbcContext.batchExecute(dropIndexesSql); } + protected void dropConstraint(TapConnectorContext connectorContext, TapTable table, TapDropConstraintEvent tapDropConstraintEvent) throws SQLException { + char escapeChar = commonDbConfig.getEscapeChar(); + List dropConstraintsSql = new ArrayList<>(); + tapDropConstraintEvent.getConstraintList().forEach(fk -> dropConstraintsSql.add("alter table " + getSchemaAndTable(table.getId()) + " drop constraint " + escapeChar + fk.getName() + escapeChar)); + jdbcContext.batchExecute(dropConstraintsSql); + } + protected long countRawCommand(TapConnectorContext connectorContext, String command, TapTable tapTable) throws SQLException { AtomicLong count = new AtomicLong(0); if (EmptyKit.isNotBlank(command) && command.trim().toLowerCase().startsWith("select")) { diff --git a/connectors-common/sql-core/src/main/java/io/tapdata/common/CommonSqlMaker.java b/connectors-common/sql-core/src/main/java/io/tapdata/common/CommonSqlMaker.java index 089d428ca..f86a2978f 100644 --- a/connectors-common/sql-core/src/main/java/io/tapdata/common/CommonSqlMaker.java +++ b/connectors-common/sql-core/src/main/java/io/tapdata/common/CommonSqlMaker.java @@ -7,6 +7,7 @@ import io.tapdata.entity.schema.value.DateTime; import io.tapdata.entity.utils.DataMap; import io.tapdata.kit.EmptyKit; +import io.tapdata.kit.StringKit; import io.tapdata.pdk.apis.entity.Projection; import io.tapdata.pdk.apis.entity.QueryOperator; import io.tapdata.pdk.apis.entity.TapAdvanceFilter; @@ -29,11 +30,13 @@ */ public class CommonSqlMaker { - private char escapeChar = '"'; + protected char escapeChar = '"'; public static final String COLLATE = "COLLATE"; protected Boolean closeNotNull = false; protected Boolean createAutoInc = false; + protected long autoIncCacheValue = 1; protected Boolean applyDefault = false; + protected String schema; public CommonSqlMaker() { @@ -53,11 +56,21 @@ public T createAutoInc(Boolean createAutoInc) { return (T) this; } + public T autoIncCacheValue(long autoIncCacheValue) { + this.autoIncCacheValue = autoIncCacheValue; + return (T) this; + } + public T applyDefault(Boolean applyDefault) { this.applyDefault = applyDefault; return (T) this; } + public T schema(String schema) { + this.schema = schema; + return (T) this; + } + public char getEscapeChar() { return escapeChar; } @@ -83,14 +96,14 @@ public String buildColumnDefinition(TapTable tapTable, boolean needComment) { if (tapField.getDataType() == null) { return ""; } - builder.append(escapeChar).append(tapField.getName()).append(escapeChar).append(' ').append(tapField.getDataType()).append(' '); - if (Boolean.TRUE.equals(applyDefault) && EmptyKit.isNotNull(tapField.getDefaultValue()) && !"".equals(tapField.getDefaultValue())) { + buildDataTypeDefinition(builder, tapField); + if (Boolean.TRUE.equals(applyDefault) && EmptyKit.isNotNull(tapField.getDefaultValue())) { buildDefaultDefinition(builder, tapField); } - buildNullDefinition(builder, tapField); if (Boolean.TRUE.equals(createAutoInc) && Boolean.TRUE.equals(tapField.getAutoInc())) { buildAutoIncDefinition(builder, tapField); } + buildNullDefinition(builder, tapField); if (needComment) { buildCommentDefinition(builder, tapField); } @@ -98,6 +111,10 @@ public String buildColumnDefinition(TapTable tapTable, boolean needComment) { }).collect(Collectors.joining(", ")); } + protected void buildDataTypeDefinition(StringBuilder builder, TapField tapField) { + builder.append(escapeChar).append(StringKit.escape(tapField.getName(), escapeChar)).append(escapeChar).append(' ').append(tapField.getDataType()).append(' '); + } + protected void buildNullDefinition(StringBuilder builder, TapField tapField) { if ((EmptyKit.isNotNull(tapField.getNullable()) && !tapField.getNullable()) || tapField.getPrimaryKey()) { builder.append("NOT NULL").append(' '); @@ -105,9 +122,11 @@ protected void buildNullDefinition(StringBuilder builder, TapField tapField) { } protected void buildDefaultDefinition(StringBuilder builder, TapField tapField) { - if (EmptyKit.isNotNull(tapField.getDefaultValue()) && !"".equals(tapField.getDefaultValue())) { + if (EmptyKit.isNotNull(tapField.getDefaultValue())) { builder.append("DEFAULT").append(' '); - if (tapField.getDefaultValue() instanceof Number) { + if (EmptyKit.isNotNull(tapField.getDefaultFunction())) { + builder.append(buildDefaultFunction(tapField)).append(' '); + } else if (tapField.getDefaultValue() instanceof Number || Boolean.TRUE.equals(tapField.getAutoInc())) { builder.append(tapField.getDefaultValue()).append(' '); } else { builder.append("'").append(tapField.getDefaultValue()).append("' "); @@ -115,6 +134,10 @@ protected void buildDefaultDefinition(StringBuilder builder, TapField tapField) } } + protected String buildDefaultFunction(TapField tapField) { + return "'" + tapField.getDefaultValue() + "' "; + } + protected void buildAutoIncDefinition(StringBuilder builder, TapField tapField) { } @@ -301,8 +324,13 @@ public String buildKeyAndValue(Map record, String splitSymbol, S StringBuilder builder = new StringBuilder(); if (EmptyKit.isNotEmpty(record)) { record.forEach((fieldName, value) -> { - builder.append(escapeChar).append(fieldName).append(escapeChar).append(operator); - builder.append(buildValueString(value)); + if (null != value) { + builder.append(escapeChar).append(fieldName).append(escapeChar).append(operator); + builder.append(buildValueString(value)); + } else { + builder.append(escapeChar).append(fieldName).append(escapeChar).append(' '); + builder.append("IS NULL"); + } builder.append(' ').append(splitSymbol).append(' '); }); builder.delete(builder.length() - splitSymbol.length() - 1, builder.length()); diff --git a/connectors-common/sql-core/src/main/java/io/tapdata/common/JdbcContext.java b/connectors-common/sql-core/src/main/java/io/tapdata/common/JdbcContext.java index 4ef33cd6a..519f02577 100644 --- a/connectors-common/sql-core/src/main/java/io/tapdata/common/JdbcContext.java +++ b/connectors-common/sql-core/src/main/java/io/tapdata/common/JdbcContext.java @@ -241,6 +241,20 @@ protected String queryAllIndexesSql(String schema, List tableNames) { throw new UnsupportedOperationException(); } + public List queryAllForeignKeys(List tableNames) throws SQLException { + List foreignKeyList = list(); + try { + query(queryAllForeignKeysSql(getConfig().getSchema(), tableNames), + resultSet -> foreignKeyList.addAll(DbKit.getDataFromResultSet(resultSet))); + } catch (UnsupportedOperationException ignore) { + } + return foreignKeyList; + } + + protected String queryAllForeignKeysSql(String schema, List tableNames) { + throw new UnsupportedOperationException(); + } + public Long queryTimestamp() throws SQLException { throw new UnsupportedOperationException(); } diff --git a/connectors-common/sql-core/src/main/java/io/tapdata/common/dml/NormalRecordWriter.java b/connectors-common/sql-core/src/main/java/io/tapdata/common/dml/NormalRecordWriter.java index 8678a8762..91327d371 100644 --- a/connectors-common/sql-core/src/main/java/io/tapdata/common/dml/NormalRecordWriter.java +++ b/connectors-common/sql-core/src/main/java/io/tapdata/common/dml/NormalRecordWriter.java @@ -176,6 +176,13 @@ public NormalRecordWriter setTapLogger(Log tapLogger) { return this; } + public NormalRecordWriter setRemovedColumn(List removedColumn) { + insertRecorder.setRemovedColumn(removedColumn); + updateRecorder.setRemovedColumn(removedColumn); + deleteRecorder.setRemovedColumn(removedColumn); + return this; + } + public void setAutoIncFields(List autoIncFields) { this.autoIncFields = autoIncFields; } @@ -189,4 +196,30 @@ protected String upsertDoubleActive() { return "update " + escapeChar + commonDbConfig.getSchema() + escapeChar + "." + escapeChar + "_tap_double_active" + escapeChar + " set " + escapeChar + "c2" + escapeChar + " = '" + UUID.randomUUID() + "' where " + escapeChar + "c1" + escapeChar + " = '1'"; } + + public void closeConstraintCheck() throws SQLException { + String sql = getCloseConstraintCheckSql(); + if (EmptyKit.isNotBlank(sql)) { + try (Statement statement = connection.createStatement()) { + statement.execute(sql); + } + } + } + + protected String getCloseConstraintCheckSql() { + return null; + } + + public void closeIdentity() throws SQLException { + String sql = getIdentitySql(); + if (EmptyKit.isNotBlank(sql)) { + try (Statement statement = connection.createStatement()) { + statement.execute(sql); + } + } + } + + protected String getIdentitySql() { + return null; + } } diff --git a/connectors-common/sql-core/src/main/java/io/tapdata/common/dml/NormalWriteRecorder.java b/connectors-common/sql-core/src/main/java/io/tapdata/common/dml/NormalWriteRecorder.java index 3bdf5057b..3db582052 100644 --- a/connectors-common/sql-core/src/main/java/io/tapdata/common/dml/NormalWriteRecorder.java +++ b/connectors-common/sql-core/src/main/java/io/tapdata/common/dml/NormalWriteRecorder.java @@ -35,6 +35,7 @@ public abstract class NormalWriteRecorder { protected final List allColumn; protected List updatedColumn; + protected List removedColumn; protected final List uniqueCondition; protected final Map columnTypeMap; protected boolean hasPk = false; @@ -90,6 +91,16 @@ public void setLargeSql(boolean largeSql) { } } + public void setRemovedColumn(List removedColumn) { + this.removedColumn = removedColumn; + allColumn.removeAll(removedColumn); + updatedColumn.removeAll(removedColumn); + if (EmptyKit.isEmpty(updatedColumn)) { + updatedColumn.addAll(allColumn); + } + uniqueCondition.removeAll(removedColumn); + } + /** * batch write events * @@ -279,11 +290,19 @@ protected void largeInsert(Map after) throws SQLException { throw new UnsupportedOperationException("largeInsert is not supported"); } + protected String quoteAndEscape(String value) { + return escapeChar + StringKit.escape(value, escapeChar) + escapeChar; + } + + protected String getSchemaAndTable() { + return quoteAndEscape(schema) + "." + quoteAndEscape(tapTable.getId()); + } + //直接插入 protected void justInsert(Map after) throws SQLException { if (EmptyKit.isNull(preparedStatement)) { - String insertSql = "INSERT INTO " + escapeChar + schema + escapeChar + "." + escapeChar + tapTable.getId() + escapeChar + " (" - + allColumn.stream().map(k -> escapeChar + k + escapeChar).collect(Collectors.joining(", ")) + ") " + + String insertSql = "INSERT INTO " + getSchemaAndTable() + " (" + + allColumn.stream().map(this::quoteAndEscape).collect(Collectors.joining(", ")) + ") " + "VALUES(" + StringKit.copyString("?", allColumn.size(), ",") + ") "; preparedStatement = connection.prepareStatement(insertSql); } @@ -357,20 +376,20 @@ protected void justUpdate(Map after, Map before, } protected String getLargeInsertSql() { - return "INSERT INTO " + escapeChar + schema + escapeChar + "." + escapeChar + tapTable.getId() + escapeChar + " (" - + allColumn.stream().map(k -> escapeChar + k + escapeChar).collect(Collectors.joining(", ")) + ") VALUES " + return "INSERT INTO " + getSchemaAndTable() + " (" + + allColumn.stream().map(this::quoteAndEscape).collect(Collectors.joining(", ")) + ") VALUES " + String.join(", ", largeSqlValues); } protected String getUpdateSql(Map after, Map before, boolean containsNull) { if (!containsNull) { - return "UPDATE " + escapeChar + schema + escapeChar + "." + escapeChar + tapTable.getId() + escapeChar + " SET " + - after.keySet().stream().map(k -> escapeChar + k + escapeChar + "=?").collect(Collectors.joining(", ")) + " WHERE " + - before.keySet().stream().map(k -> escapeChar + k + escapeChar + "=?").collect(Collectors.joining(" AND ")); + return "UPDATE " + getSchemaAndTable() + " SET " + + after.keySet().stream().map(k -> quoteAndEscape(k) + "=?").collect(Collectors.joining(", ")) + " WHERE " + + before.keySet().stream().map(k -> quoteAndEscape(k) + "=?").collect(Collectors.joining(" AND ")); } else { - return "UPDATE " + escapeChar + schema + escapeChar + "." + escapeChar + tapTable.getId() + escapeChar + " SET " + - after.keySet().stream().map(k -> escapeChar + k + escapeChar + "=?").collect(Collectors.joining(", ")) + " WHERE " + - before.keySet().stream().map(k -> "(" + escapeChar + k + escapeChar + "=? OR (" + escapeChar + k + escapeChar + " IS NULL AND ? IS NULL))") + return "UPDATE " + getSchemaAndTable() + " SET " + + after.keySet().stream().map(k -> quoteAndEscape(k) + "=?").collect(Collectors.joining(", ")) + " WHERE " + + before.keySet().stream().map(k -> "(" + quoteAndEscape(k) + "=? OR (" + quoteAndEscape(k) + " IS NULL AND ? IS NULL))") .collect(Collectors.joining(" AND ")); } } @@ -383,10 +402,12 @@ public void addDeleteBatch(Map before, WriteListResult lastBefore.put(v, before.get(v))); //Mongo为源端时，非_id为更新条件时，lastBefore为空，此时需要原始before直接删除 if (EmptyKit.isEmpty(lastBefore)) { - justDelete(before, listResult); - } else { - justDelete(lastBefore, listResult); + lastBefore.putAll(before); + } + if (EmptyKit.isNotEmpty(removedColumn)) { + removedColumn.forEach(lastBefore::remove); } + justDelete(lastBefore, listResult); preparedStatement.addBatch(); } @@ -417,11 +438,11 @@ protected void justDelete(Map before, WriteListResult before, boolean containsNull) { if (!containsNull) { - return "DELETE FROM " + escapeChar + schema + escapeChar + "." + escapeChar + tapTable.getId() + escapeChar + " WHERE " + - before.keySet().stream().map(k -> escapeChar + k + escapeChar + "=?").collect(Collectors.joining(" AND ")); + return "DELETE FROM " + getSchemaAndTable() + " WHERE " + + before.keySet().stream().map(k -> quoteAndEscape(k) + "=?").collect(Collectors.joining(" AND ")); } else { - return "DELETE FROM " + escapeChar + schema + escapeChar + "." + escapeChar + tapTable.getId() + escapeChar + " WHERE " + - before.keySet().stream().map(k -> "(" + escapeChar + k + escapeChar + "=? OR (" + escapeChar + k + escapeChar + " IS NULL AND ? IS NULL))") + return "DELETE FROM " + getSchemaAndTable() + " WHERE " + + before.keySet().stream().map(k -> "(" + quoteAndEscape(k) + "=? OR (" + quoteAndEscape(k) + " IS NULL AND ? IS NULL))") .collect(Collectors.joining(" AND ")); } } diff --git a/connectors-common/sql-core/src/main/java/io/tapdata/common/exception/ExceptionCollector.java b/connectors-common/sql-core/src/main/java/io/tapdata/common/exception/ExceptionCollector.java index d10ad85f9..a2add32e0 100644 --- a/connectors-common/sql-core/src/main/java/io/tapdata/common/exception/ExceptionCollector.java +++ b/connectors-common/sql-core/src/main/java/io/tapdata/common/exception/ExceptionCollector.java @@ -25,4 +25,12 @@ public interface ExceptionCollector { void collectCdcConfigInvalid(Throwable cause) throws RuntimeException; void revealException(Throwable cause) throws RuntimeException; + + default boolean violateConstraintName(Throwable cause) { + return false; + } + + default boolean violateIndexName(Throwable cause) { + return false; + } } diff --git a/connectors-javascript/pom.xml b/connectors-javascript/pom.xml index 0a1474f9d..1ec832847 100644 --- a/connectors-javascript/pom.xml +++ b/connectors-javascript/pom.xml @@ -31,8 +31,8 @@ ${project.artifactId}-v${project.version} 8 1.10-SNAPSHOT - 1.4.1-SNAPSHOT - 1.4.1-SNAPSHOT + 1.4.4-SNAPSHOT + 1.4.4-SNAPSHOT 1.0-SNAPSHOT 5.8.1 1.8.1 diff --git a/connectors-unpackage/hive1-connector/src/main/java/io/tapdata/connector/hive1/Hive1Connector.java b/connectors-unpackage/hive1-connector/src/main/java/io/tapdata/connector/hive1/Hive1Connector.java index ab79b817b..191a439ba 100644 --- a/connectors-unpackage/hive1-connector/src/main/java/io/tapdata/connector/hive1/Hive1Connector.java +++ b/connectors-unpackage/hive1-connector/src/main/java/io/tapdata/connector/hive1/Hive1Connector.java @@ -150,8 +150,8 @@ public void registerCapabilities(ConnectorFunctions connectorFunctions, TapCodec return "null"; }); codecRegistry.registerFromTapValue(TapBinaryValue.class, "STRING", tapValue -> { - if (tapValue != null && tapValue.getValue() != null) - return new String(Base64.encodeBase64(tapValue.getValue())); + if (tapValue != null && tapValue.getValue() != null && tapValue.getValue().getValue() != null) + return new String(Base64.encodeBase64(tapValue.getValue().getValue())); return null; }); codecRegistry.registerFromTapValue(TapTimeValue.class, "STRING", tapTimeValue -> formatTapDateTime(tapTimeValue.getValue(), "HH:mm:ss.SS")); diff --git a/connectors-unpackage/pom.xml b/connectors-unpackage/pom.xml index 346927f02..3edf82623 100644 --- a/connectors-unpackage/pom.xml +++ b/connectors-unpackage/pom.xml @@ -16,7 +16,7 @@ ${project.artifactId}-v${project.version} 8 1.10-SNAPSHOT - 1.4.1-SNAPSHOT + 1.4.4-SNAPSHOT 1.0-SNAPSHOT 5.8.1 1.8.1 diff --git a/connectors/aliyun-adb-mysql-connector/pom.xml b/connectors/aliyun-adb-mysql-connector/pom.xml index 6014492d6..657e2bd17 100644 --- a/connectors/aliyun-adb-mysql-connector/pom.xml +++ b/connectors/aliyun-adb-mysql-connector/pom.xml @@ -22,7 +22,7 @@ - 1.4.1-SNAPSHOT + 1.4.4-SNAPSHOT 8 diff --git a/connectors/aliyun-adb-postgres-connector/pom.xml b/connectors/aliyun-adb-postgres-connector/pom.xml index 1d76629fc..c7e01ca0b 100644 --- a/connectors/aliyun-adb-postgres-connector/pom.xml +++ b/connectors/aliyun-adb-postgres-connector/pom.xml @@ -22,7 +22,7 @@ - 1.4.1-SNAPSHOT + 1.4.4-SNAPSHOT diff --git a/connectors/aliyun-mongodb-connector/pom.xml b/connectors/aliyun-mongodb-connector/pom.xml index 81b1089a5..84aafe54b 100644 --- a/connectors/aliyun-mongodb-connector/pom.xml +++ b/connectors/aliyun-mongodb-connector/pom.xml @@ -23,7 +23,7 @@ 8 - 1.4.1-SNAPSHOT + 1.4.4-SNAPSHOT diff --git a/connectors/aliyun-rds-mariadb-connector/pom.xml b/connectors/aliyun-rds-mariadb-connector/pom.xml index 06992acb4..7a3864b77 100644 --- a/connectors/aliyun-rds-mariadb-connector/pom.xml +++ b/connectors/aliyun-rds-mariadb-connector/pom.xml @@ -23,7 +23,7 @@ 8 - 1.4.1-SNAPSHOT + 1.4.4-SNAPSHOT diff --git a/connectors/aliyun-rds-mysql-connector/pom.xml b/connectors/aliyun-rds-mysql-connector/pom.xml index f38a0e938..95bbb2846 100644 --- a/connectors/aliyun-rds-mysql-connector/pom.xml +++ b/connectors/aliyun-rds-mysql-connector/pom.xml @@ -23,7 +23,7 @@ 8 - 1.4.1-SNAPSHOT + 1.4.4-SNAPSHOT diff --git a/connectors/aliyun-rds-postgres-connector/pom.xml b/connectors/aliyun-rds-postgres-connector/pom.xml index bf10e79ee..6f10f9fb1 100644 --- a/connectors/aliyun-rds-postgres-connector/pom.xml +++ b/connectors/aliyun-rds-postgres-connector/pom.xml @@ -23,7 +23,7 @@ - 1.4.1-SNAPSHOT + 1.4.4-SNAPSHOT diff --git a/connectors/aws-clickhouse-connector/pom.xml b/connectors/aws-clickhouse-connector/pom.xml index 24745bee5..3e41c2cc7 100644 --- a/connectors/aws-clickhouse-connector/pom.xml +++ b/connectors/aws-clickhouse-connector/pom.xml @@ -19,7 +19,7 @@ 1.0-SNAPSHOT 3.12.0 31.0.1-jre - 1.4.1-SNAPSHOT + 1.4.4-SNAPSHOT diff --git a/connectors/aws-rds-mysql-connector/pom.xml b/connectors/aws-rds-mysql-connector/pom.xml index 7d3ca7f33..f6d0b556d 100644 --- a/connectors/aws-rds-mysql-connector/pom.xml +++ b/connectors/aws-rds-mysql-connector/pom.xml @@ -23,7 +23,7 @@ 8 - 1.4.1-SNAPSHOT + 1.4.4-SNAPSHOT diff --git a/connectors/azure-cosmosdb-connector/pom.xml b/connectors/azure-cosmosdb-connector/pom.xml index 35ce0e200..d63691b5b 100644 --- a/connectors/azure-cosmosdb-connector/pom.xml +++ b/connectors/azure-cosmosdb-connector/pom.xml @@ -19,7 +19,7 @@ 8 - 1.4.1-SNAPSHOT + 1.4.4-SNAPSHOT diff --git a/connectors/clickhouse-connector/pom.xml b/connectors/clickhouse-connector/pom.xml index d4c411a9d..c96c91754 100644 --- a/connectors/clickhouse-connector/pom.xml +++ b/connectors/clickhouse-connector/pom.xml @@ -19,7 +19,7 @@ 1.0-SNAPSHOT 3.12.0 31.0.1-jre - 1.4.1-SNAPSHOT + 1.4.4-SNAPSHOT diff --git a/connectors/clickhouse-connector/src/main/java/io/tapdata/connector/clickhouse/ClickhouseConnector.java b/connectors/clickhouse-connector/src/main/java/io/tapdata/connector/clickhouse/ClickhouseConnector.java index 1f9b88110..cc1e435f6 100644 --- a/connectors/clickhouse-connector/src/main/java/io/tapdata/connector/clickhouse/ClickhouseConnector.java +++ b/connectors/clickhouse-connector/src/main/java/io/tapdata/connector/clickhouse/ClickhouseConnector.java @@ -186,8 +186,8 @@ public void registerCapabilities(ConnectorFunctions connectorFunctions, TapCodec }); codecRegistry.registerFromTapValue(TapYearValue.class, "FixedString(4)", TapValue::getOriginValue); codecRegistry.registerFromTapValue(TapBinaryValue.class, "String", tapValue -> { - if (tapValue != null && tapValue.getValue() != null) - return new String(Base64.encodeBase64(tapValue.getValue())); + if (tapValue != null && tapValue.getValue() != null && tapValue.getValue().getValue() != null) + return new String(Base64.encodeBase64(tapValue.getValue().getValue())); return null; }); diff --git a/connectors/csv-connector/pom.xml b/connectors/csv-connector/pom.xml index 2bec60589..de15dfb1e 100644 --- a/connectors/csv-connector/pom.xml +++ b/connectors/csv-connector/pom.xml @@ -15,7 +15,7 @@ jar - 1.4.1-SNAPSHOT + 1.4.4-SNAPSHOT 8 diff --git a/connectors/custom-connector/pom.xml b/connectors/custom-connector/pom.xml index a6a0b5c92..581e3229a 100644 --- a/connectors/custom-connector/pom.xml +++ b/connectors/custom-connector/pom.xml @@ -12,7 +12,7 @@ custom-connector - 1.4.1-SNAPSHOT + 1.4.4-SNAPSHOT diff --git a/connectors/databend-connector/src/main/java/io/tapdata/databend/DatabendConnector.java b/connectors/databend-connector/src/main/java/io/tapdata/databend/DatabendConnector.java index 0376296d8..d8aa581ca 100644 --- a/connectors/databend-connector/src/main/java/io/tapdata/databend/DatabendConnector.java +++ b/connectors/databend-connector/src/main/java/io/tapdata/databend/DatabendConnector.java @@ -136,8 +136,8 @@ public void registerCapabilities(ConnectorFunctions connectorFunctions, TapCodec codecRegistry.registerFromTapValue(TapTimeValue.class, tapTimeValue -> formatTapDateTime(tapTimeValue.getValue(), "HH:mm:ss.SS")); codecRegistry.registerFromTapValue(TapBinaryValue.class, "String", tapValue -> { - if (tapValue != null && tapValue.getValue() != null) - return new String(Base64.encodeBase64(tapValue.getValue())); + if (tapValue != null && tapValue.getValue() != null && tapValue.getValue().getValue() != null) + return new String(Base64.encodeBase64(tapValue.getValue().getValue())); return null; }); diff --git a/connectors/doris-connector/pom.xml b/connectors/doris-connector/pom.xml index b83477ef0..5251cc73e 100644 --- a/connectors/doris-connector/pom.xml +++ b/connectors/doris-connector/pom.xml @@ -21,7 +21,7 @@ 31.0.1-jre 1.0-SNAPSHOT - 1.4.1-SNAPSHOT + 1.4.4-SNAPSHOT diff --git a/connectors/doris-connector/src/main/java/io/tapdata/connector/doris/DorisConnector.java b/connectors/doris-connector/src/main/java/io/tapdata/connector/doris/DorisConnector.java index c4cd6ffc4..e588aeda5 100644 --- a/connectors/doris-connector/src/main/java/io/tapdata/connector/doris/DorisConnector.java +++ b/connectors/doris-connector/src/main/java/io/tapdata/connector/doris/DorisConnector.java @@ -72,6 +72,7 @@ public void onStart(TapConnectionContext tapConnectionContext) { if (tapConnectionContext instanceof TapConnectorContext) { ddlSqlGenerator = new DorisDDLSqlGenerator(); } + tapLogger = tapConnectionContext.getLog(); commonDbConfig = dorisConfig; jdbcContext = dorisJdbcContext; commonSqlMaker = new DorisSqlMaker(); @@ -136,8 +137,8 @@ public void registerCapabilities(ConnectorFunctions connectorFunctions, TapCodec return 0; }); codecRegistry.registerFromTapValue(TapBinaryValue.class, "text", tapValue -> { - if (tapValue != null && tapValue.getValue() != null) - return toJson(tapValue.getValue()); + if (tapValue != null && tapValue.getValue() != null && tapValue.getValue().getValue() != null) + return toJson(tapValue.getValue().getValue()); return "null"; }); diff --git a/connectors/doris-connector/src/main/java/io/tapdata/connector/doris/DorisSqlMaker.java b/connectors/doris-connector/src/main/java/io/tapdata/connector/doris/DorisSqlMaker.java index 4e4777b8f..a85b87614 100644 --- a/connectors/doris-connector/src/main/java/io/tapdata/connector/doris/DorisSqlMaker.java +++ b/connectors/doris-connector/src/main/java/io/tapdata/connector/doris/DorisSqlMaker.java @@ -30,8 +30,8 @@ public String buildColumnDefinitionByOrder(TapTable tapTable, Collection if (aggregate && Boolean.FALSE.equals(tapField.getPrimaryKey())) { builder.append("REPLACE_IF_NOT_NULL "); } - buildDefaultDefinition(builder, tapField); buildNullDefinition(builder, tapField); + buildDefaultDefinition(builder, tapField); buildCommentDefinition(builder, tapField); return builder.toString(); }).collect(Collectors.joining(", ")); diff --git a/connectors/dummy-connector/pom.xml b/connectors/dummy-connector/pom.xml index 3151287c2..20a1e87e6 100644 --- a/connectors/dummy-connector/pom.xml +++ b/connectors/dummy-connector/pom.xml @@ -15,7 +15,7 @@ 8 - 1.4.1-SNAPSHOT + 1.4.4-SNAPSHOT diff --git a/connectors/dws-connector/pom.xml b/connectors/dws-connector/pom.xml index feae55e01..fef198185 100644 --- a/connectors/dws-connector/pom.xml +++ b/connectors/dws-connector/pom.xml @@ -15,7 +15,7 @@ 8 - 1.4.1-SNAPSHOT + 1.4.4-SNAPSHOT diff --git a/connectors/dws-connector/src/main/java/io/tapdata/connector/dws/DwsWriteRecorder.java b/connectors/dws-connector/src/main/java/io/tapdata/connector/dws/DwsWriteRecorder.java index e6082abd8..5b91ab085 100644 --- a/connectors/dws-connector/src/main/java/io/tapdata/connector/dws/DwsWriteRecorder.java +++ b/connectors/dws-connector/src/main/java/io/tapdata/connector/dws/DwsWriteRecorder.java @@ -47,10 +47,11 @@ protected Object filterValue(Object value, String dataType) throws SQLException if (value == null) { return null; } + value = super.filterValue(value, dataType); if (value instanceof String && EmptyKit.isEmpty((String) value)) { return replaceBlank; } - return super.filterValue(value, dataType); + return value; } } diff --git a/connectors/elasticsearch-connector/pom.xml b/connectors/elasticsearch-connector/pom.xml index cc4880055..66fc84fb2 100644 --- a/connectors/elasticsearch-connector/pom.xml +++ b/connectors/elasticsearch-connector/pom.xml @@ -12,7 +12,7 @@ elasticsearch-connector - 1.4.1-SNAPSHOT + 1.4.4-SNAPSHOT diff --git a/connectors/excel-connector/pom.xml b/connectors/excel-connector/pom.xml index 70d449556..aa151a720 100644 --- a/connectors/excel-connector/pom.xml +++ b/connectors/excel-connector/pom.xml @@ -15,7 +15,7 @@ jar - 1.4.1-SNAPSHOT + 1.4.4-SNAPSHOT 8 diff --git a/connectors/greenplum-connector/pom.xml b/connectors/greenplum-connector/pom.xml index 3374b5e6e..b89adede9 100644 --- a/connectors/greenplum-connector/pom.xml +++ b/connectors/greenplum-connector/pom.xml @@ -15,7 +15,7 @@ UTF-8 8 - 1.4.1-SNAPSHOT + 1.4.4-SNAPSHOT diff --git a/connectors/greenplum-connector/src/main/java/io/tapdata/connector/greenplum/GreenplumConnector.java b/connectors/greenplum-connector/src/main/java/io/tapdata/connector/greenplum/GreenplumConnector.java index ae2802736..430a1036a 100644 --- a/connectors/greenplum-connector/src/main/java/io/tapdata/connector/greenplum/GreenplumConnector.java +++ b/connectors/greenplum-connector/src/main/java/io/tapdata/connector/greenplum/GreenplumConnector.java @@ -146,6 +146,7 @@ public void onStart(TapConnectionContext connectorContext) { jdbcContext = postgresJdbcContext; commonSqlMaker = new CommonSqlMaker(); postgresVersion = postgresJdbcContext.queryVersion(); + postgresJdbcContext.withPostgresVersion(postgresVersion); ddlSqlGenerator = new PostgresDDLSqlGenerator(); tapLogger = connectorContext.getLog(); fieldDDLHandlers = new BiClassHandlers<>(); @@ -173,12 +174,12 @@ protected void writeRecord(TapConnectorContext connectorContext, List consumer) { + public ConnectionOptions connectionTest(TapConnectionContext connectionContext, Consumer consumer) throws SQLException { postgresConfig = (PostgresConfig) new PostgresConfig().load(connectionContext.getConnectionConfig()); ConnectionOptions connectionOptions = ConnectionOptions.create(); connectionOptions.connectionString(postgresConfig.getConnectionString()); try ( - GreenplumTest greenplumTest = new GreenplumTest(postgresConfig, consumer, connectionOptions) + GreenplumTest greenplumTest = (GreenplumTest) new GreenplumTest(postgresConfig, consumer, connectionOptions).withPostgresVersion() ) { greenplumTest.testOneByOne(); return connectionOptions; diff --git a/connectors/highgo-connector/pom.xml b/connectors/highgo-connector/pom.xml index 4647761b2..3e3d53f65 100644 --- a/connectors/highgo-connector/pom.xml +++ b/connectors/highgo-connector/pom.xml @@ -15,7 +15,7 @@ 8 - 1.4.1-SNAPSHOT + 1.4.4-SNAPSHOT 1.5.4.Final diff --git a/connectors/highgo-connector/src/main/java/io/tapdata/connector/postgres/HighgoConnector.java b/connectors/highgo-connector/src/main/java/io/tapdata/connector/postgres/HighgoConnector.java index 763e8dcb6..9cecbc032 100644 --- a/connectors/highgo-connector/src/main/java/io/tapdata/connector/postgres/HighgoConnector.java +++ b/connectors/highgo-connector/src/main/java/io/tapdata/connector/postgres/HighgoConnector.java @@ -340,8 +340,7 @@ private void initConnection(TapConnectionContext connectionContext) { } private void openIdentity(TapTable tapTable) throws SQLException { - if (EmptyKit.isEmpty(tapTable.primaryKeys()) - && (EmptyKit.isEmpty(tapTable.getIndexList()) || tapTable.getIndexList().stream().noneMatch(TapIndex::isUnique))) { + if (EmptyKit.isEmpty(tapTable.primaryKeys())) { jdbcContext.execute("ALTER TABLE \"" + jdbcContext.getConfig().getSchema() + "\".\"" + tapTable.getId() + "\" REPLICA IDENTITY FULL"); } } diff --git a/connectors/http-receiver-connector/pom.xml b/connectors/http-receiver-connector/pom.xml index ed2b1926b..8d0854127 100644 --- a/connectors/http-receiver-connector/pom.xml +++ b/connectors/http-receiver-connector/pom.xml @@ -16,7 +16,7 @@ 8 - 1.4.1-SNAPSHOT + 1.4.4-SNAPSHOT diff --git a/connectors/huawei-cloud-gaussdb-connector/pom.xml b/connectors/huawei-cloud-gaussdb-connector/pom.xml index 2a22e63b2..43302afcb 100644 --- a/connectors/huawei-cloud-gaussdb-connector/pom.xml +++ b/connectors/huawei-cloud-gaussdb-connector/pom.xml @@ -15,7 +15,7 @@ jar - 1.4.2-SNAPSHOT + 1.4.4-SNAPSHOT 8 42.3.4 diff --git a/connectors/hudi-connector/pom.xml b/connectors/hudi-connector/pom.xml index 4027856ea..6cd645abd 100644 --- a/connectors/hudi-connector/pom.xml +++ b/connectors/hudi-connector/pom.xml @@ -12,7 +12,7 @@ hudi-connector - 1.4.1-SNAPSHOT + 1.4.4-SNAPSHOT 1.0-SNAPSHOT 3.1.1-hw-ei-312005 8 diff --git a/connectors/hudi-connector/src/main/java/io/tapdata/connector/hudi/HudiConnector.java b/connectors/hudi-connector/src/main/java/io/tapdata/connector/hudi/HudiConnector.java index 294c8d159..bdb4fc104 100644 --- a/connectors/hudi-connector/src/main/java/io/tapdata/connector/hudi/HudiConnector.java +++ b/connectors/hudi-connector/src/main/java/io/tapdata/connector/hudi/HudiConnector.java @@ -108,8 +108,8 @@ public void registerCapabilities(ConnectorFunctions connectorFunctions, TapCodec .registerFromTapValue(TapMapValue.class, "string", this::registerString) .registerFromTapValue(TapArrayValue.class, "string", this::registerString) .registerFromTapValue(TapBinaryValue.class, "string", tapValue -> { - if (tapValue != null && tapValue.getValue() != null) - return new String(Base64.encodeBase64(tapValue.getValue())); + if (tapValue != null && tapValue.getValue() != null && tapValue.getValue().getValue() != null) + return new String(Base64.encodeBase64(tapValue.getValue().getValue())); return null; }).registerFromTapValue(TapDateTimeValue.class, "timestamp", tapValue -> { if (tapValue != null && tapValue.getValue() != null) { diff --git a/connectors/json-connector/pom.xml b/connectors/json-connector/pom.xml index 6cb361881..9759bb2a8 100644 --- a/connectors/json-connector/pom.xml +++ b/connectors/json-connector/pom.xml @@ -15,7 +15,7 @@ jar - 1.4.1-SNAPSHOT + 1.4.4-SNAPSHOT 8 diff --git a/connectors/kafka-connector/pom.xml b/connectors/kafka-connector/pom.xml index 795552ae0..184389e98 100644 --- a/connectors/kafka-connector/pom.xml +++ b/connectors/kafka-connector/pom.xml @@ -12,7 +12,7 @@ kafka-connector - 1.4.1-SNAPSHOT + 1.4.4-SNAPSHOT diff --git a/connectors/kafka-enhanced-connector/pom.xml b/connectors/kafka-enhanced-connector/pom.xml index 0605f3da8..383928995 100644 --- a/connectors/kafka-enhanced-connector/pom.xml +++ b/connectors/kafka-enhanced-connector/pom.xml @@ -15,7 +15,7 @@ 8 - 1.4.1-SNAPSHOT + 1.4.4-SNAPSHOT 2.8.2 1.7.25 diff --git a/connectors/mariadb-connector/pom.xml b/connectors/mariadb-connector/pom.xml index b39f765e7..e1a3221f2 100644 --- a/connectors/mariadb-connector/pom.xml +++ b/connectors/mariadb-connector/pom.xml @@ -16,7 +16,7 @@ 4.0.3 1.5.4.Final 4.4 - 1.4.1-SNAPSHOT + 1.4.4-SNAPSHOT diff --git a/connectors/mock-source-connector/pom.xml b/connectors/mock-source-connector/pom.xml index 1cf23a118..3d8a57eec 100644 --- a/connectors/mock-source-connector/pom.xml +++ b/connectors/mock-source-connector/pom.xml @@ -15,7 +15,7 @@ 8 - 1.4.1-SNAPSHOT + 1.4.4-SNAPSHOT diff --git a/connectors/mock-target-connector/pom.xml b/connectors/mock-target-connector/pom.xml index 92c791bad..96b53b796 100644 --- a/connectors/mock-target-connector/pom.xml +++ b/connectors/mock-target-connector/pom.xml @@ -15,7 +15,7 @@ 8 - 1.4.1-SNAPSHOT + 1.4.4-SNAPSHOT diff --git a/connectors/mongodb-atlas-connector/pom.xml b/connectors/mongodb-atlas-connector/pom.xml index 27cb5522d..e78598204 100644 --- a/connectors/mongodb-atlas-connector/pom.xml +++ b/connectors/mongodb-atlas-connector/pom.xml @@ -13,7 +13,7 @@ 8 - 1.4.1-SNAPSHOT + 1.4.4-SNAPSHOT diff --git a/connectors/mongodb-connector/pom.xml b/connectors/mongodb-connector/pom.xml index 30d12be91..8da120846 100644 --- a/connectors/mongodb-connector/pom.xml +++ b/connectors/mongodb-connector/pom.xml @@ -13,7 +13,7 @@ 8 - 1.4.3-SNAPSHOT + 1.4.4-SNAPSHOT diff --git a/connectors/mongodb-connector/src/main/java/io/tapdata/mongodb/MongodbConnector.java b/connectors/mongodb-connector/src/main/java/io/tapdata/mongodb/MongodbConnector.java index 525442b81..fab7e177e 100644 --- a/connectors/mongodb-connector/src/main/java/io/tapdata/mongodb/MongodbConnector.java +++ b/connectors/mongodb-connector/src/main/java/io/tapdata/mongodb/MongodbConnector.java @@ -1,5 +1,6 @@ package io.tapdata.mongodb; +import com.alibaba.fastjson.JSONObject; import com.mongodb.*; import com.mongodb.bulk.BulkWriteError; import com.mongodb.client.*; @@ -495,7 +496,8 @@ public void registerCapabilities(ConnectorFunctions connectorFunctions, TapCodec }); codecRegistry.registerToTapValue(Binary.class, (value, tapType) -> { Binary binary = (Binary) value; - return new TapBinaryValue(binary.getData()); + ByteData byteData = new ByteData(binary.getType(), binary.getData()); + return new TapBinaryValue(byteData); }); codecRegistry.registerToTapValue(Code.class, (value, tapType) -> { @@ -517,7 +519,9 @@ public void registerCapabilities(ConnectorFunctions connectorFunctions, TapCodec codecRegistry.registerFromTapValue(TapDateTimeValue.class, "DATE_TIME", tapDateTimeValue -> tapDateTimeValue.getValue().toDate()); codecRegistry.registerFromTapValue(TapDateValue.class, "DATE_TIME", tapDateValue -> tapDateValue.getValue().toDate()); codecRegistry.registerFromTapValue(TapYearValue.class, "STRING(4)", TapValue::getOriginValue); - + codecRegistry.registerFromTapValue(TapBinaryValue.class,"BINARY",tapBinaryValue -> { + return new Binary(tapBinaryValue.getValue().getType(), tapBinaryValue.getValue().getValue()); + }); //Handle ObjectId when the source is also mongodb, we convert ObjectId to String before enter incremental engine. //We need check the TapStringValue, when will write to mongodb, if the originValue is ObjectId, then use originValue instead of the converted String value. codecRegistry.registerFromTapValue(TapStringValue.class, tapValue -> { @@ -729,7 +733,12 @@ private boolean createSharedCollection(DataMap nodeConfig, TapTable table, Colle boolean isShardCollection = shardCollection instanceof Boolean && ((Boolean) shardCollection); if (isShardCollection && null != table) { isShardCollection = false; - TapIndexEx partitionIndex = table.getPartitionIndex(); + TapIndexEx partitionIndex; + if(null != table.getTableAttr().get("partitionIndex")){ + partitionIndex = JSONObject.parseObject((String)table.getTableAttr().get("partitionIndex"), TapIndexEx.class); + }else { + partitionIndex = table.getPartitionIndex(); + } if (null != partitionIndex) { Boolean unique = Optional.ofNullable(partitionIndex.getUnique()).orElse(false); List indexFields = partitionIndex.getIndexFields(); diff --git a/connectors/mongodb-connector/src/main/java/io/tapdata/mongodb/util/MongoShardUtil.java b/connectors/mongodb-connector/src/main/java/io/tapdata/mongodb/util/MongoShardUtil.java index 5cd6a9587..2bd51ac6b 100644 --- a/connectors/mongodb-connector/src/main/java/io/tapdata/mongodb/util/MongoShardUtil.java +++ b/connectors/mongodb-connector/src/main/java/io/tapdata/mongodb/util/MongoShardUtil.java @@ -1,5 +1,6 @@ package io.tapdata.mongodb.util; +import com.alibaba.fastjson.JSONObject; import io.tapdata.entity.schema.TapIndexEx; import io.tapdata.entity.schema.TapIndexField; import io.tapdata.entity.schema.TapTable; @@ -61,6 +62,7 @@ public static void saveCollectionStats(TapTable table, Map attrs indexEx.setPrimary(false); indexEx.setName(UUID.randomUUID().toString()); indexEx.setCluster(false); + map.put("partitionIndex", JSONObject.toJSONString(indexEx)); table.setPartitionIndex(indexEx); } } else { diff --git a/connectors/mongodb-lower-connector/pom.xml b/connectors/mongodb-lower-connector/pom.xml index 3a5d16853..c884c7887 100644 --- a/connectors/mongodb-lower-connector/pom.xml +++ b/connectors/mongodb-lower-connector/pom.xml @@ -13,7 +13,7 @@ 8 - 1.4.1-SNAPSHOT + 1.4.4-SNAPSHOT diff --git a/connectors/mysql-connector/pom.xml b/connectors/mysql-connector/pom.xml index 82e2e1a04..66bb9bfd0 100644 --- a/connectors/mysql-connector/pom.xml +++ b/connectors/mysql-connector/pom.xml @@ -23,7 +23,7 @@ 8 - 1.4.1-SNAPSHOT + 1.4.4-SNAPSHOT diff --git a/connectors/mysql-connector/src/main/java/io/tapdata/connector/mysql/MysqlConnector.java b/connectors/mysql-connector/src/main/java/io/tapdata/connector/mysql/MysqlConnector.java index f43a87ca2..2a7eaac65 100644 --- a/connectors/mysql-connector/src/main/java/io/tapdata/connector/mysql/MysqlConnector.java +++ b/connectors/mysql-connector/src/main/java/io/tapdata/connector/mysql/MysqlConnector.java @@ -7,6 +7,7 @@ import io.tapdata.common.ResultSetConsumer; import io.tapdata.common.SqlExecuteCommandFunction; import io.tapdata.common.ddl.type.DDLParserType; +import io.tapdata.common.dml.NormalRecordWriter; import io.tapdata.connector.mysql.bean.MysqlColumn; import io.tapdata.connector.mysql.config.MysqlConfig; import io.tapdata.connector.mysql.constant.DeployModeEnum; @@ -16,13 +17,15 @@ import io.tapdata.connector.mysql.util.MysqlUtil; import io.tapdata.connector.mysql.writer.MysqlSqlBatchWriter; import io.tapdata.connector.mysql.writer.MysqlWriter; +import io.tapdata.entity.TapConstraintException; import io.tapdata.entity.codec.TapCodecsRegistry; import io.tapdata.entity.codec.ToTapValueCodec; -import io.tapdata.entity.error.CoreException; import io.tapdata.entity.event.TapEvent; +import io.tapdata.entity.event.ddl.constraint.TapCreateConstraintEvent; import io.tapdata.entity.event.ddl.table.*; import io.tapdata.entity.event.dml.TapInsertRecordEvent; import io.tapdata.entity.event.dml.TapRecordEvent; +import io.tapdata.entity.schema.TapConstraint; import io.tapdata.entity.schema.TapField; import io.tapdata.entity.schema.TapIndex; import io.tapdata.entity.schema.TapTable; @@ -68,6 +71,7 @@ import java.util.concurrent.atomic.AtomicReference; import java.util.function.BiConsumer; import java.util.function.Consumer; +import java.util.stream.Collectors; /** * @author samuel @@ -87,6 +91,9 @@ public class MysqlConnector extends CommonDbConnector { protected ZoneId zoneId; protected ZoneId dbZoneId; protected int zoneOffsetHour; + protected long autoIncrementValue = 1; + protected long autoIncCacheValue = 1; + protected long autoStartValue = 1; protected final AtomicBoolean started = new AtomicBoolean(false); public static final String MASTER_NODE_KEY = "MASTER_NODE"; @@ -108,6 +115,12 @@ public void onStart(TapConnectionContext tapConnectionContext) throws Throwable commonDbConfig = mysqlConfig; jdbcContext = mysqlJdbcContext; commonSqlMaker = new CommonSqlMaker('`'); + if (Boolean.TRUE.equals(mysqlConfig.getCreateAutoInc())) { + commonSqlMaker.createAutoInc(true); + } + if (Boolean.TRUE.equals(mysqlConfig.getApplyDefault())) { + commonSqlMaker.applyDefault(true); + } tapLogger = tapConnectionContext.getLog(); exceptionCollector = new MysqlExceptionCollector(); ((MysqlExceptionCollector) exceptionCollector).setMysqlConfig(mysqlConfig); @@ -148,6 +161,33 @@ public void onStart(TapConnectionContext tapConnectionContext) throws Throwable started.set(true); } + @Override + public void discoverSchema(TapConnectionContext connectionContext, List tables, int tableSize, Consumer> consumer) throws SQLException { + mysqlJdbcContext.normalQuery("SHOW VARIABLES LIKE 'auto_inc%'", rs -> { + while (rs.next()) { + String variableName = rs.getString("Variable_name"); + if ("auto_increment_increment".equals(variableName)) { + autoIncrementValue = rs.getLong("Value"); + } else if ("auto_increment_offset".equals(variableName)) { + autoStartValue = rs.getLong("Value"); + } + } + }); + mysqlJdbcContext.normalQuery("SHOW VARIABLES LIKE 'innodb_autoinc_lock_mode'", rs -> { + if (rs.next()) { + String value = rs.getString("Value"); + if ("0".equals(value)) { + autoIncCacheValue = 1; + } else if ("1".equals(value)) { + autoIncCacheValue = 100; + } else { + autoIncCacheValue = 1000; + } + } + }); + super.discoverSchema(connectionContext, tables, tableSize, consumer); + } + @Override public void onLightStart(TapConnectionContext tapConnectionContext) throws Throwable { mysqlConfig = new MysqlConfig().load(tapConnectionContext.getConnectionConfig()); @@ -236,8 +276,12 @@ public void registerCapabilities(ConnectorFunctions connectorFunctions, TapCodec connectorFunctions.supportQueryByAdvanceFilter(this::queryByAdvanceFilterWithOffset); connectorFunctions.supportCountByPartitionFilterFunction(this::countByAdvanceFilter); connectorFunctions.supportWriteRecord(this::writeRecord); + connectorFunctions.supportAfterInitialSync(this::afterInitialSync); connectorFunctions.supportCreateIndex(this::createIndex); connectorFunctions.supportQueryIndexes(this::queryIndexes); + connectorFunctions.supportCreateConstraint(this::createConstraint); + connectorFunctions.supportQueryConstraints(this::queryConstraint); + connectorFunctions.supportDropConstraint(this::dropConstraint); connectorFunctions.supportNewFieldFunction(this::fieldDDLHandler); connectorFunctions.supportAlterFieldNameFunction(this::fieldDDLHandler); connectorFunctions.supportAlterFieldAttributesFunction(this::fieldDDLHandler); @@ -395,7 +439,11 @@ public void onStop(TapConnectionContext connectionContext) { } protected TapField makeTapField(DataMap dataMap) { - return new MysqlColumn(dataMap).withVersion(version).getTapField(); + return new MysqlColumn(dataMap).withVersion(version) + .withSeedValue(autoStartValue) + .withIncrementValue(autoIncrementValue) + .withAutoIncCacheValue(autoIncCacheValue) + .getTapField(); } protected CreateTableOptions createTableV2(TapConnectorContext tapConnectorContext, TapCreateTableEvent tapCreateTableEvent) throws SQLException { @@ -412,7 +460,9 @@ protected CreateTableOptions createTableV2(TapConnectorContext tapConnectorConte tapLogger.info("Table \"{}.{}\" exists, skip auto create table", database, tableId); } else { String mysqlVersion = mysqlJdbcContext.queryVersion(); - SqlMaker sqlMaker = new MysqlMaker(); + MysqlMaker sqlMaker = new MysqlMaker(); + sqlMaker.setCreateAutoInc(mysqlConfig.getCreateAutoInc()); + sqlMaker.setApplyDefault(mysqlConfig.getApplyDefault()); if (null == tapCreateTableEvent.getTable()) { tapLogger.warn("Create table event's tap table is null, will skip it: " + tapCreateTableEvent); return createTableOptions; @@ -472,9 +522,23 @@ protected CreateTableOptions createTableV2(TapConnectorContext tapConnectorConte return createTableOptions; } + protected void beforeWriteRecord(TapTable tapTable) throws SQLException { + super.beforeWriteRecord(tapTable); + List autoIncFields = new ArrayList<>(); + if (mysqlConfig.getCreateAutoInc()) { + if (!writtenTableMap.get(tapTable.getId()).containsKey(HAS_AUTO_INCR)) { + List fields = tapTable.getNameFieldMap().values().stream().filter(TapField::getAutoInc).collect(Collectors.toList()); + autoIncFields.addAll(fields.stream().map(TapField::getName).collect(Collectors.toList())); + writtenTableMap.get(tapTable.getId()).put(HAS_AUTO_INCR, autoIncFields); + } else { + autoIncFields.addAll(writtenTableMap.get(tapTable.getId()).getValue(HAS_AUTO_INCR, new ArrayList<>())); + } + } + } + private void writeRecord(TapConnectorContext tapConnectorContext, List tapRecordEvents, TapTable tapTable, Consumer> consumer) throws Throwable { -// WriteListResult writeListResult = this.mysqlWriter.write(tapConnectorContext, tapTable, tapRecordEvents); -// consumer.accept(writeListResult); + beforeWriteRecord(tapTable); + List autoIncFields = writtenTableMap.get(tapTable.getId()).getValue(HAS_AUTO_INCR, new ArrayList<>()); String insertDmlPolicy = tapConnectorContext.getConnectorCapabilities().getCapabilityAlternative(ConnectionOptions.DML_INSERT_POLICY); if (insertDmlPolicy == null) { insertDmlPolicy = ConnectionOptions.DML_INSERT_POLICY_UPDATE_ON_EXISTS; @@ -483,7 +547,8 @@ private void writeRecord(TapConnectorContext tapConnectorContext, List alterSqls = new ArrayList<>(); + mysqlRecordWriter.getAutoIncMap().forEach((k, v) -> { + alterSqls.add("alter table " + getSchemaAndTable(tapTable.getId()) + " auto_increment " + (Long.parseLong(String.valueOf(v)) + mysqlConfig.getAutoIncJumpValue())); + }); + jdbcContext.batchExecute(alterSqls); + } + } else { + mysqlRecordWriter.write(tapRecordEvents, consumer, this::isAlive); + } + } + protected void afterInitialSync(TapConnectorContext connectorContext, TapTable tapTable) throws Throwable { + beforeWriteRecord(tapTable); + List autoIncFields = writtenTableMap.get(tapTable.getId()).getValue(HAS_AUTO_INCR, new ArrayList<>()); + autoIncFields.forEach(field -> { + try ( + Connection connection = jdbcContext.getConnection(); + Statement statement = connection.createStatement(); + ResultSet resultSet = statement.executeQuery("select max(" + field + ") from " + getSchemaAndTable(tapTable.getId())) + ) { + if (resultSet.next()) { + statement.execute("alter table " + getSchemaAndTable(tapTable.getId()) + " auto_increment " + (resultSet.getLong(1) + mysqlConfig.getAutoIncJumpValue())); + connection.commit(); + } + } catch (SQLException e) { + tapLogger.warn("Failed to get auto increment value for table {} field {}", tapTable.getId(), field, e); + } + }); } protected Map filterTimeForMysql( @@ -644,10 +743,9 @@ public void buildIllegalDateFieldName(TapRecordEvent event, List illegal @Override protected String getBatchReadSelectSql(TapTable tapTable) { if (tapTable.getNameFieldMap().size() > 50) { - return String.format("SELECT * FROM `%s`.`%s`", mysqlConfig.getDatabase(), tapTable.getId()); + return String.format("SELECT * FROM %s", getSchemaAndTable(tapTable.getId())); } else { - String columns = String.join("`, `", tapTable.getNameFieldMap().keySet()); - return String.format("SELECT `%s` FROM `%s`.`%s`", columns, mysqlConfig.getDatabase(), tapTable.getId()); + return super.getBatchReadSelectSql(tapTable); } } @@ -826,4 +924,68 @@ protected void queryTableHash(TapConnectorContext connectorContext, TapAdvanceFi }); } + protected void clearTable(TapConnectorContext tapConnectorContext, TapClearTableEvent tapClearTableEvent) throws SQLException { + if (jdbcContext.queryAllTables(Collections.singletonList(tapClearTableEvent.getTableId())).size() >= 1) { + List sqls = new ArrayList<>(); + sqls.add("SET FOREIGN_KEY_CHECKS=0"); + sqls.add("truncate table " + getSchemaAndTable(tapClearTableEvent.getTableId())); + jdbcContext.batchExecute(sqls); + } else { + tapLogger.warn("Table {} not exists, skip truncate", tapClearTableEvent.getTableId()); + } + } + + protected void dropTable(TapConnectorContext tapConnectorContext, TapDropTableEvent tapDropTableEvent) throws SQLException { + if (jdbcContext.queryAllTables(Collections.singletonList(tapDropTableEvent.getTableId())).size() >= 1) { + List sqls = new ArrayList<>(); + sqls.add("SET FOREIGN_KEY_CHECKS=0"); + sqls.add("drop table " + getSchemaAndTable(tapDropTableEvent.getTableId())); + jdbcContext.batchExecute(sqls); + } else { + tapLogger.warn("Table {} not exists, skip drop", tapDropTableEvent.getTableId()); + } + } + + protected void createConstraint(TapConnectorContext connectorContext, TapTable tapTable, TapCreateConstraintEvent createConstraintEvent, boolean create) { + List constraintList = createConstraintEvent.getConstraintList(); + if (EmptyKit.isNotEmpty(constraintList)) { + List constraintSqlList = new ArrayList<>(); + TapConstraintException exception = new TapConstraintException(tapTable.getId()); + constraintList.forEach(c -> { + String sql = getCreateConstraintSql(tapTable, c); + if (create) { + try { + jdbcContext.execute(sql); + } catch (Exception e) { + if (e instanceof SQLException && ((SQLException) e).getErrorCode() == 3780) { + TapTable referenceTable = connectorContext.getTableMap().get(c.getReferencesTableName()); + c.getMappingFields().stream().filter(m -> Boolean.TRUE.equals(referenceTable.getNameFieldMap().get(m.getReferenceKey()).getAutoInc()) && referenceTable.getNameFieldMap().get(m.getReferenceKey()).getDataType().startsWith("decimal")).forEach(m -> { + try { + jdbcContext.execute("alter table " + getSchemaAndTable(tapTable.getId()) + " modify `" + m.getForeignKey() + "` bigint"); + } catch (SQLException e1) { + exception.addException(c, "alter table modify column failed", e1); + } + }); + try { + jdbcContext.execute(sql); + } catch (Exception e1) { + exception.addException(c, sql, e1); + } + } else { + exception.addException(c, sql, e); + } + } + } else { + constraintSqlList.add(sql); + } + }); + if (!create) { + createConstraintEvent.setConstraintSqlList(constraintSqlList); + } + if (EmptyKit.isNotEmpty(exception.getExceptions())) { + throw exception; + } + } + } + } diff --git a/connectors/mysql-connector/src/main/resources/mysql-spec.json b/connectors/mysql-connector/src/main/resources/mysql-spec.json index 9e967d31d..005a2a79b 100644 --- a/connectors/mysql-connector/src/main/resources/mysql-spec.json +++ b/connectors/mysql-connector/src/main/resources/mysql-spec.json @@ -4,7 +4,7 @@ "icon": "icons/mysql.png", "id": "mysql", "doc": "${doc}", - "tags": ["Database", "ssl", "doubleActive"] + "tags": ["Database", "ssl", "doubleActive", "disableForeignKey"] }, "configOptions": { "capabilities": [ @@ -331,6 +331,67 @@ "node": { "type": "object", "properties": { + "createAutoInc": { + "type": "boolean", + "title": "${createAutoInc}", + "default": false, + "x-index": 5, + "x-decorator": "FormItem", + "x-component": "Switch", + "x-decorator-props": { + "tooltip": "${createAutoIncTooltip}" + }, + "x-reactions": [ + { + "dependencies": ["$inputs"], + "fulfill": { + "state": { + "display": "{{$deps[0].length > 0 ? \"visible\":\"hidden\"}}" + } + } + } + ] + }, + "autoIncJumpValue": { + "required": true, + "type": "string", + "title": "${autoIncJumpValue}", + "default": 1000000, + "x-index": 6, + "x-decorator": "FormItem", + "x-component": "InputNumber", + "x-reactions": [ + { + "dependencies": ["$inputs", ".createAutoInc"], + "fulfill": { + "state": { + "display": "{{$deps[0].length > 0 && $deps[1] ? \"visible\":\"hidden\"}}" + } + } + } + ] + }, + "applyDefault": { + "type": "boolean", + "title": "${applyDefault}", + "default": false, + "x-index": 7, + "x-decorator": "FormItem", + "x-component": "Switch", + "x-decorator-props": { + "tooltip": "${applyDefaultTooltip}" + }, + "x-reactions": [ + { + "dependencies": ["$inputs"], + "fulfill": { + "state": { + "display": "{{$deps[0].length > 0 ? \"visible\":\"hidden\"}}" + } + } + } + ] + }, "hashSplit": { "type": "boolean", "title": "${hashSplit}", @@ -448,6 +509,11 @@ "Address" : "Please enter the server address", "serverPort": "Server port", "prompt": "Add", + "createAutoInc": "Sync auto-increment column", + "createAutoIncTooltip": "After turning on, the MySQL target will synchronize the auto-increment column, but the step of the auto-increment key will only be affected by the global configuration auto_increment_increment, please configure it as needed", + "autoIncJumpValue": "Auto-increment key jump value", + "applyDefault": "Apply default value", + "applyDefaultTooltip": "When the switch is turned on, the default value will be applied to the target. If there are unadapted functions or expressions, it may cause an error", "hashSplit": "Hash split", "hashSplitTooltip": "When the switch is turned on, it can be sharded according to the hash value, suitable for large table full-stage sharded synchronization", "maxSplit": "Maximum number of splits", @@ -479,6 +545,11 @@ "Address" : "服务器地址", "serverPort": "服务器端口", "prompt": "添加", + "createAutoInc": "同步自增列", + "createAutoIncTooltip": "开启后，MySQL目标会同步自增列，但自增键的步长只会受全局配置auto_increment_increment影响，请自行按需配置", + "autoIncJumpValue": "自增键跳跃值", + "applyDefault": "应用默认值", + "applyDefaultTooltip": "开关打开时会将默认值应用到目标，如果有未适配的函数或表达式，可能会导致报错", "hashSplit": "哈希分片", "hashSplitTooltip": "开关打开时，可以根据哈希值进行分片，适用于大表全量阶段分片同步", "maxSplit": "最大分片数", @@ -510,6 +581,11 @@ "Address" : "服務器地址", "serverPort": "端口", "prompt": "添加", + "createAutoInc": "同步自增列", + "createAutoIncTooltip": "開啓後，MySQL目標會同步自增列，但自增鍵的步長只會受全局配置auto_increment_increment影響，請自行按需配置", + "autoIncJumpValue": "自增鍵跳躍值", + "applyDefault": "應用默認值", + "applyDefaultTooltip": "開關打開時會將默認值應用到目標，如果有未適配的函數或表達式，可能會導致報錯", "hashSplit": "哈希分片", "hashSplitTooltip": "開關打開時，可以根據哈希值進行分片，適用於大表全量階段分片同步", "maxSplit": "最大分片數", @@ -554,7 +630,7 @@ "pkEnablement": false }, "json": { - "to": "TapMap", + "to": "TapJson", "byte": "4g", "pkEnablement": false }, diff --git a/connectors/mysql-pxc-connector/pom.xml b/connectors/mysql-pxc-connector/pom.xml index 92aa8b269..4f68740af 100644 --- a/connectors/mysql-pxc-connector/pom.xml +++ b/connectors/mysql-pxc-connector/pom.xml @@ -23,7 +23,7 @@ 8 - 1.4.1-SNAPSHOT + 1.4.4-SNAPSHOT diff --git a/connectors/oceanbase-mysql-connector/pom.xml b/connectors/oceanbase-mysql-connector/pom.xml index fdeef2ab2..ca9b12314 100644 --- a/connectors/oceanbase-mysql-connector/pom.xml +++ b/connectors/oceanbase-mysql-connector/pom.xml @@ -21,7 +21,7 @@ 4.0.3 3.12.0 31.0.1-jre - 1.4.1-SNAPSHOT + 1.4.4-SNAPSHOT oceanbase-mysql-connector diff --git a/connectors/oceanbase-mysql-connector/src/main/java/io/tapdata/oceanbase/connector/OceanbaseConnector.java b/connectors/oceanbase-mysql-connector/src/main/java/io/tapdata/oceanbase/connector/OceanbaseConnector.java index 8a685b16c..b31f404e0 100644 --- a/connectors/oceanbase-mysql-connector/src/main/java/io/tapdata/oceanbase/connector/OceanbaseConnector.java +++ b/connectors/oceanbase-mysql-connector/src/main/java/io/tapdata/oceanbase/connector/OceanbaseConnector.java @@ -121,8 +121,8 @@ public void registerCapabilities(ConnectorFunctions connectorFunctions, TapCodec return 0; }); codecRegistry.registerFromTapValue(TapBinaryValue.class, "text", tapValue -> { - if (tapValue != null && tapValue.getValue() != null) - return toJson(tapValue.getValue()); + if (tapValue != null && tapValue.getValue() != null && tapValue.getValue().getValue() != null) + return toJson(tapValue.getValue().getValue()); return "null"; }); //TapTimeValue, TapDateTimeValue and TapDateValue's value is DateTime, need convert into Date object. diff --git a/connectors/opengauss-connector/pom.xml b/connectors/opengauss-connector/pom.xml index 95a7c5a4d..5ccfbcd89 100644 --- a/connectors/opengauss-connector/pom.xml +++ b/connectors/opengauss-connector/pom.xml @@ -15,7 +15,7 @@ UTF-8 8 - 1.4.1-SNAPSHOT + 1.4.4-SNAPSHOT diff --git a/connectors/polar-db-mysql-connector/pom.xml b/connectors/polar-db-mysql-connector/pom.xml index 942ea7923..a851dccb4 100644 --- a/connectors/polar-db-mysql-connector/pom.xml +++ b/connectors/polar-db-mysql-connector/pom.xml @@ -23,7 +23,7 @@ 8 - 1.4.1-SNAPSHOT + 1.4.4-SNAPSHOT diff --git a/connectors/polar-db-postgres-connector/pom.xml b/connectors/polar-db-postgres-connector/pom.xml index 109798cea..0952fdeec 100644 --- a/connectors/polar-db-postgres-connector/pom.xml +++ b/connectors/polar-db-postgres-connector/pom.xml @@ -23,7 +23,7 @@ 8 - 1.4.1-SNAPSHOT + 1.4.4-SNAPSHOT diff --git a/connectors/pom.xml b/connectors/pom.xml index b14bf8b17..c65eec2e1 100644 --- a/connectors/pom.xml +++ b/connectors/pom.xml @@ -73,7 +73,7 @@ ${project.artifactId}-v${project.version} 8 1.10-SNAPSHOT - 1.4.1-SNAPSHOT + 1.4.4-SNAPSHOT 1.0-SNAPSHOT 5.8.1 1.8.1 diff --git a/connectors/postgres-connector/pom.xml b/connectors/postgres-connector/pom.xml index 82113b366..173d398a1 100644 --- a/connectors/postgres-connector/pom.xml +++ b/connectors/postgres-connector/pom.xml @@ -16,7 +16,7 @@ 8 42.3.4 - 1.4.3-SNAPSHOT + 1.4.4-SNAPSHOT diff --git a/connectors/postgres-connector/src/main/java/io/tapdata/connector/postgres/PostgresConnector.java b/connectors/postgres-connector/src/main/java/io/tapdata/connector/postgres/PostgresConnector.java index cec6d212f..b05261212 100644 --- a/connectors/postgres-connector/src/main/java/io/tapdata/connector/postgres/PostgresConnector.java +++ b/connectors/postgres-connector/src/main/java/io/tapdata/connector/postgres/PostgresConnector.java @@ -13,16 +13,15 @@ import io.tapdata.connector.postgres.exception.PostgresExceptionCollector; import io.tapdata.connector.postgres.partition.PostgresPartitionContext; import io.tapdata.connector.postgres.partition.TableType; +import io.tapdata.entity.TapConstraintException; import io.tapdata.entity.codec.TapCodecsRegistry; import io.tapdata.entity.error.CoreException; +import io.tapdata.entity.event.ddl.constraint.TapCreateConstraintEvent; import io.tapdata.entity.event.ddl.index.TapCreateIndexEvent; -import io.tapdata.entity.event.ddl.table.TapAlterFieldAttributesEvent; -import io.tapdata.entity.event.ddl.table.TapAlterFieldNameEvent; -import io.tapdata.entity.event.ddl.table.TapDropFieldEvent; -import io.tapdata.entity.event.ddl.table.TapNewFieldEvent; +import io.tapdata.entity.event.ddl.table.*; import io.tapdata.entity.event.dml.TapRecordEvent; +import io.tapdata.entity.schema.TapConstraint; import io.tapdata.entity.schema.TapField; -import io.tapdata.entity.schema.TapIndex; import io.tapdata.entity.schema.TapTable; import io.tapdata.entity.schema.type.TapType; import io.tapdata.entity.schema.value.*; @@ -48,6 +47,7 @@ import io.tapdata.pdk.apis.functions.connector.common.vo.TapHashResult; import io.tapdata.pdk.apis.functions.connector.common.vo.TapPartitionResult; import io.tapdata.pdk.apis.functions.connector.source.ConnectionConfigWithTables; +import io.tapdata.pdk.apis.functions.connector.target.CreateTableOptions; import org.postgresql.geometric.*; import org.postgresql.jdbc.PgArray; import org.postgresql.jdbc.PgSQLXML; @@ -62,7 +62,6 @@ import java.time.ZoneId; import java.time.ZoneOffset; import java.util.*; -import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.CopyOnWriteArraySet; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicReference; @@ -83,9 +82,6 @@ public class PostgresConnector extends CommonDbConnector { private PostgresCdcRunner cdcRunner; //only when task start-pause this variable can be shared private Object slotName; //must be stored in stateMap protected String postgresVersion; - protected Map writtenTableMap = new ConcurrentHashMap<>(); - protected static final String HAS_UNIQUE_INDEX = "HAS_UNIQUE_INDEX"; - protected static final String HAS_AUTO_INCR = "HAS_AUTO_INCR"; protected PostgresPartitionContext postgresPartitionContext; @Override @@ -105,7 +101,7 @@ public ConnectionOptions connectionTest(TapConnectionContext connectionContext, try ( PostgresTest postgresTest = new PostgresTest(postgresConfig, consumer, connectionOptions) .initContext() - .withPostgresVersion(); + .withPostgresVersion() ) { postgresTest.testOneByOne(); connectionOptions.setInstanceUniqueId(StringKit.md5(String.join("|" @@ -128,12 +124,16 @@ public void registerCapabilities(ConnectorFunctions connectorFunctions, TapCodec connectorFunctions.supportReleaseExternalFunction(this::onDestroy); // target connectorFunctions.supportWriteRecord(this::writeRecord); + connectorFunctions.supportAfterInitialSync(this::afterInitialSync); connectorFunctions.supportCreateTableV2(this::createTableV2); connectorFunctions.supportClearTable(this::clearTable); connectorFunctions.supportDropTable(this::dropTable); connectorFunctions.supportCreateIndex(this::createIndex); connectorFunctions.supportQueryIndexes(this::queryIndexes); connectorFunctions.supportDeleteIndex(this::dropIndexes); + connectorFunctions.supportQueryConstraints(this::queryConstraint); + connectorFunctions.supportCreateConstraint(this::createConstraint); + connectorFunctions.supportDropConstraint(this::dropConstraint); // source connectorFunctions.supportBatchCount(this::batchCount); connectorFunctions.supportBatchRead(this::batchReadWithoutOffset); @@ -356,8 +356,8 @@ private void initConnection(TapConnectionContext connectionContext) { postgresConfig.load(tapConnectorContext.getNodeConfig()); }); postgresVersion = postgresJdbcContext.queryVersion(); - commonSqlMaker = new PostgresSqlMaker().closeNotNull(postgresConfig.getCloseNotNull()); - if (Boolean.TRUE.equals(postgresConfig.getCreateAutoInc()) && postgresVersion.compareTo("100000") >= 0) { + commonSqlMaker = new PostgresSqlMaker().schema(postgresConfig.getSchema()).closeNotNull(postgresConfig.getCloseNotNull()).autoIncCacheValue(postgresConfig.getAutoIncCacheValue()); + if (Boolean.TRUE.equals(postgresConfig.getCreateAutoInc()) && Integer.parseInt(postgresVersion) > 100000) { commonSqlMaker.createAutoInc(true); } if (Boolean.TRUE.equals(postgresConfig.getApplyDefault())) { @@ -380,8 +380,7 @@ private void initConnection(TapConnectionContext connectionContext) { } protected void openIdentity(TapTable tapTable) throws SQLException { - if (EmptyKit.isEmpty(tapTable.primaryKeys()) - && (EmptyKit.isEmpty(tapTable.getIndexList()) || tapTable.getIndexList().stream().noneMatch(TapIndex::isUnique))) { + if (EmptyKit.isEmpty(tapTable.primaryKeys())) { jdbcContext.execute("ALTER TABLE \"" + jdbcContext.getConfig().getSchema() + "\".\"" + tapTable.getId() + "\" REPLICA IDENTITY FULL"); } } @@ -390,26 +389,49 @@ protected boolean makeSureHasUnique(TapTable tapTable) throws SQLException { return jdbcContext.queryAllIndexes(Collections.singletonList(tapTable.getId())).stream().anyMatch(v -> "1".equals(v.getString("isUnique"))); } - //write records as all events, prepared - protected void writeRecord(TapConnectorContext connectorContext, List tapRecordEvents, TapTable tapTable, Consumer> writeListResultConsumer) throws SQLException { - boolean hasUniqueIndex; - List autoIncFields = new ArrayList<>(); + protected CreateTableOptions createTableV2(TapConnectorContext connectorContext, TapCreateTableEvent createTableEvent) throws SQLException { + if (Boolean.TRUE.equals(postgresConfig.getCreateAutoInc()) && Integer.parseInt(postgresVersion) > 100000) { + createTableEvent.getTable().getNameFieldMap().entrySet().stream().filter(entry -> EmptyKit.isNotBlank(entry.getValue().getSequenceName())).forEach(entry -> { + StringBuilder sequenceSql = new StringBuilder("CREATE SEQUENCE IF NOT EXISTS " + getSchemaAndTable(entry.getValue().getSequenceName())); + if (EmptyKit.isNotNull(entry.getValue().getAutoIncStartValue())) { + sequenceSql.append(" START ").append(entry.getValue().getAutoIncStartValue()); + } + if (EmptyKit.isNotNull(entry.getValue().getAutoIncrementValue())) { + sequenceSql.append(" INCREMENT ").append(entry.getValue().getAutoIncrementValue()); + } + try { + postgresJdbcContext.execute(sequenceSql.toString()); + } catch (SQLException e) { + tapLogger.warn("Failed to create sequence for table {} field {}", createTableEvent.getTable().getId(), entry.getKey(), e); + } + }); + } + CreateTableOptions options = super.createTableV2(connectorContext, createTableEvent); + if (EmptyKit.isNotBlank(postgresConfig.getTableOwner())) { + jdbcContext.execute(String.format("alter table %s owner to %s", getSchemaAndTable(createTableEvent.getTableId()), postgresConfig.getTableOwner())); + } + return options; + } + + protected void beforeWriteRecord(TapTable tapTable) throws SQLException { if (EmptyKit.isNull(writtenTableMap.get(tapTable.getId()))) { openIdentity(tapTable); - hasUniqueIndex = makeSureHasUnique(tapTable); + boolean hasUniqueIndex = makeSureHasUnique(tapTable); writtenTableMap.put(tapTable.getId(), DataMap.create().kv(HAS_UNIQUE_INDEX, hasUniqueIndex)); - } else { - hasUniqueIndex = writtenTableMap.get(tapTable.getId()).getValue(HAS_UNIQUE_INDEX, false); } - if (postgresConfig.getCreateAutoInc() && postgresVersion.compareTo("100000") >= 0) { + if (postgresConfig.getCreateAutoInc() && Integer.parseInt(postgresVersion) > 100000) { if (!writtenTableMap.get(tapTable.getId()).containsKey(HAS_AUTO_INCR)) { - List fields = tapTable.getNameFieldMap().values().stream().filter(TapField::getAutoInc).collect(Collectors.toList()); - autoIncFields.addAll(fields.stream().map(TapField::getName).collect(Collectors.toList())); + List autoIncFields = tapTable.getNameFieldMap().values().stream().filter(TapField::getAutoInc).map(TapField::getName).collect(Collectors.toList()); writtenTableMap.get(tapTable.getId()).put(HAS_AUTO_INCR, autoIncFields); - } else { - autoIncFields.addAll(writtenTableMap.get(tapTable.getId()).getValue(HAS_AUTO_INCR, new ArrayList<>())); } } + } + + //write records as all events, prepared + protected void writeRecord(TapConnectorContext connectorContext, List tapRecordEvents, TapTable tapTable, Consumer> writeListResultConsumer) throws SQLException { + beforeWriteRecord(tapTable); + boolean hasUniqueIndex = writtenTableMap.get(tapTable.getId()).getValue(HAS_UNIQUE_INDEX, false); + List autoIncFields = writtenTableMap.get(tapTable.getId()).getValue(HAS_AUTO_INCR, new ArrayList<>()); String insertDmlPolicy = connectorContext.getConnectorCapabilities().getCapabilityAlternative(ConnectionOptions.DML_INSERT_POLICY); if (insertDmlPolicy == null) { insertDmlPolicy = ConnectionOptions.DML_INSERT_POLICY_UPDATE_ON_EXISTS; @@ -438,14 +460,22 @@ protected void writeRecord(TapConnectorContext connectorContext, List= 0 && EmptyKit.isNotEmpty(autoIncFields) + if (EmptyKit.isNotEmpty(tapTable.getConstraintList())) { + postgresRecordWriter.closeConstraintCheck(); + } + if (postgresConfig.getCreateAutoInc() && Integer.parseInt(postgresVersion) > 100000 && EmptyKit.isNotEmpty(autoIncFields) && "CDC".equals(tapRecordEvents.get(0).getInfo().get(TapRecordEvent.INFO_KEY_SYNC_STAGE))) { postgresRecordWriter.setAutoIncFields(autoIncFields); postgresRecordWriter.write(tapRecordEvents, writeListResultConsumer, this::isAlive); if (EmptyKit.isNotEmpty(postgresRecordWriter.getAutoIncMap())) { List alterSqls = new ArrayList<>(); postgresRecordWriter.getAutoIncMap().forEach((k, v) -> { - alterSqls.add("ALTER TABLE \"" + jdbcContext.getConfig().getSchema() + "\".\"" + tapTable.getId() + "\" ALTER COLUMN \"" + k + "\" SET GENERATED BY DEFAULT RESTART WITH " + (Long.parseLong(String.valueOf(v)) + postgresConfig.getAutoIncJumpValue())); + String sequenceName = tapTable.getNameFieldMap().get(k).getSequenceName(); + if (EmptyKit.isNotBlank(sequenceName)) { + alterSqls.add("select setval('" + getSchemaAndTable(sequenceName) + "'," + (Long.parseLong(String.valueOf(v)) + postgresConfig.getAutoIncJumpValue()) + ", false) "); + } else { + alterSqls.add("ALTER TABLE " + getSchemaAndTable(tapTable.getId()) + " ALTER COLUMN \"" + k + "\" SET GENERATED BY DEFAULT RESTART WITH " + (Long.parseLong(String.valueOf(v)) + postgresConfig.getAutoIncJumpValue())); + } }); jdbcContext.batchExecute(alterSqls); } @@ -454,6 +484,30 @@ protected void writeRecord(TapConnectorContext connectorContext, List autoIncFields = writtenTableMap.get(tapTable.getId()).getValue(HAS_AUTO_INCR, new ArrayList<>()); + autoIncFields.forEach(field -> { + try ( + Connection connection = jdbcContext.getConnection(); + Statement statement = connection.createStatement(); + ResultSet resultSet = statement.executeQuery("select max(" + field + ") from " + getSchemaAndTable(tapTable.getId())) + ) { + if (resultSet.next()) { + String sequenceName = tapTable.getNameFieldMap().get(field).getSequenceName(); + if (EmptyKit.isNotBlank(sequenceName)) { + statement.execute("select setval('" + getSchemaAndTable(sequenceName) + "'," + (resultSet.getLong(1) + postgresConfig.getAutoIncJumpValue()) + ", false) "); + } else { + statement.execute("ALTER TABLE " + getSchemaAndTable(tapTable.getId()) + " ALTER COLUMN \"" + field + "\" SET GENERATED BY DEFAULT RESTART WITH " + (resultSet.getLong(1) + postgresConfig.getAutoIncJumpValue())); + } + connection.commit(); + } + } catch (SQLException e) { + tapLogger.warn("Failed to get auto increment value for table {} field {}", tapTable.getId(), field, e); + } + }); + } + private void streamRead(TapConnectorContext nodeContext, List tableList, Object offsetState, int recordSize, StreamReadConsumer consumer) throws Throwable { if ("walminer".equals(postgresConfig.getLogPluginName())) { new WalLogMinerV2(postgresJdbcContext, tapLogger) @@ -543,7 +597,7 @@ private Object timestampToStreamOffset(TapConnectorContext connectorContext, Lon //test streamRead log plugin boolean canCdc = Boolean.TRUE.equals(postgresTest.testStreamRead()); if (canCdc) { - if ("pgoutput".equals(postgresConfig.getLogPluginName()) && postgresVersion.compareTo("100000") > 0) { + if ("pgoutput".equals(postgresConfig.getLogPluginName()) && Integer.parseInt(postgresVersion) > 100000) { createPublicationIfNotExist(); } testReplicateIdentity(connectorContext.getTableMap()); @@ -635,7 +689,7 @@ protected String getWalDirectory() throws SQLException { break; } if ("data_directory".equals(resultSet.getString(1)) && EmptyKit.isNotEmpty(resultSet.getString(2))) { - walDirectory.set(resultSet.getString(2) + (postgresVersion.compareTo("100000") >= 0 ? "/pg_wal/" : "/pg_xlog/")); + walDirectory.set(resultSet.getString(2) + (Integer.parseInt(postgresVersion) > 100000 ? "/pg_wal/" : "/pg_xlog/")); } } }); @@ -802,6 +856,22 @@ List> splitToPieces(List data, int eachPieceSize) { return result; } + protected void clearTable(TapConnectorContext tapConnectorContext, TapClearTableEvent tapClearTableEvent) throws SQLException { + if (jdbcContext.queryAllTables(Collections.singletonList(tapClearTableEvent.getTableId())).size() >= 1) { + jdbcContext.execute("truncate table " + getSchemaAndTable(tapClearTableEvent.getTableId()) + " cascade"); + } else { + tapLogger.warn("Table {} not exists, skip truncate", tapClearTableEvent.getTableId()); + } + } + + protected void dropTable(TapConnectorContext tapConnectorContext, TapDropTableEvent tapDropTableEvent) throws SQLException { + if (jdbcContext.queryAllTables(Collections.singletonList(tapDropTableEvent.getTableId())).size() >= 1) { + jdbcContext.execute("drop table " + getSchemaAndTable(tapDropTableEvent.getTableId()) + " cascade"); + } else { + tapLogger.warn("Table {} not exists, skip drop", tapDropTableEvent.getTableId()); + } + } + protected void createIndex(TapConnectorContext connectorContext, TapTable tapTable, TapCreateIndexEvent createIndexEvent) throws SQLException { super.createIndex(connectorContext, tapTable, createIndexEvent); createIndexEvent.getIndexList().stream().filter(v -> Boolean.TRUE.equals(v.getCluster())).forEach(v -> { @@ -812,4 +882,46 @@ protected void createIndex(TapConnectorContext connectorContext, TapTable tapTab } }); } + + protected void createConstraint(TapConnectorContext connectorContext, TapTable tapTable, TapCreateConstraintEvent createConstraintEvent, boolean create) { + List constraintList = createConstraintEvent.getConstraintList(); + if (EmptyKit.isNotEmpty(constraintList)) { + List constraintSqlList = new ArrayList<>(); + TapConstraintException exception = new TapConstraintException(tapTable.getId()); + constraintList.forEach(c -> { + String sql = getCreateConstraintSql(tapTable, c); + if (create) { + try { + jdbcContext.execute(sql); + } catch (Exception e) { + if (e instanceof SQLException && ((SQLException) e).getSQLState().equals("42804")) { + TapTable referenceTable = connectorContext.getTableMap().get(c.getReferencesTableName()); + c.getMappingFields().stream().filter(m -> Boolean.TRUE.equals(referenceTable.getNameFieldMap().get(m.getReferenceKey()).getAutoInc()) && referenceTable.getNameFieldMap().get(m.getReferenceKey()).getDataType().startsWith("numeric")).forEach(m -> { + try { + jdbcContext.execute("alter table " + getSchemaAndTable(tapTable.getId()) + " alter column \"" + m.getForeignKey() + "\" type bigint"); + } catch (SQLException e1) { + exception.addException(c, "alter table alter column failed", e1); + } + }); + try { + jdbcContext.execute(sql); + } catch (Exception e1) { + exception.addException(c, sql, e1); + } + } else { + exception.addException(c, sql, e); + } + } + } else { + constraintSqlList.add(sql); + } + }); + if (!create) { + createConstraintEvent.setConstraintSqlList(constraintSqlList); + } + if (EmptyKit.isNotEmpty(exception.getExceptions())) { + throw exception; + } + } + } } diff --git a/connectors/postgres-connector/src/main/resources/spec_postgres.json b/connectors/postgres-connector/src/main/resources/spec_postgres.json index 12324f203..fa099c6d5 100644 --- a/connectors/postgres-connector/src/main/resources/spec_postgres.json +++ b/connectors/postgres-connector/src/main/resources/spec_postgres.json @@ -4,7 +4,7 @@ "icon": "icons/postgres.png", "doc" : "${doc}", "id": "postgres", - "tags": ["Database", "ssl", "doubleActive"] + "tags": ["Database", "ssl", "doubleActive", "disableForeignKey"] }, "configOptions": { "capabilities":[ @@ -16,6 +16,10 @@ "id": "dml_update_policy", "alternatives": ["ignore_on_nonexists", "insert_on_nonexists", "log_on_nonexists"] }, + { + "id": "dml_check_policy", + "alternatives": ["ignore_all_check", "default_check"] + }, { "id": "api_server_supported" }, @@ -303,6 +307,26 @@ } ] }, + "tableOwner": { + "type": "string", + "title": "${tableOwner}", + "x-decorator": "FormItem", + "x-component": "Input", + "x-index": 3, + "x-decorator-props": { + "tooltip": "${tableOwnerTooltip}" + }, + "x-reactions": [ + { + "dependencies": ["$inputs"], + "fulfill": { + "state": { + "display": "{{$deps[0].length > 0 ? \"visible\":\"hidden\"}}" + } + } + } + ] + }, "createAutoInc": { "type": "boolean", "title": "${createAutoInc}", @@ -343,11 +367,30 @@ } ] }, + "autoIncCacheValue": { + "required": true, + "type": "string", + "title": "${autoIncCacheValue}", + "default": 100, + "x-index": 7, + "x-decorator": "FormItem", + "x-component": "InputNumber", + "x-reactions": [ + { + "dependencies": ["$inputs", ".createAutoInc"], + "fulfill": { + "state": { + "display": "{{$deps[0].length > 0 && $deps[1] ? \"visible\":\"hidden\"}}" + } + } + } + ] + }, "applyDefault": { "type": "boolean", "title": "${applyDefault}", "default": false, - "x-index": 7, + "x-index": 8, "x-decorator": "FormItem", "x-component": "Switch", "x-decorator-props": { @@ -516,9 +559,12 @@ "timezoneTip": "Specify the time zone, otherwise no time zone processing will be done", "closeNotNull": "Ignore NotNull", "closeNotNullTooltip": "When the switch is turned on, non empty restrictions are discarded", + "tableOwner": "Specify table owner", + "tableOwnerTooltip": "You can specify the table owner, if not specified, it defaults to the current user. If the permission is insufficient, use a user with sufficient permissions to execute: grant to ", "createAutoInc": "Synchronize auto-increment columns", "createAutoIncTooltip": "Only PG 10 and above are supported. When the switch is turned on, the auto-increment attribute is synchronized when entering the incremental", "autoIncJumpValue": "Auto-increment key jump value", + "autoIncCacheValue": "Auto-increment key cache value", "applyDefault": "Apply default value", "applyDefaultTooltip": "When the switch is turned on, the default value is applied to the target. If there are unadapted functions or expressions, it may cause an error", "hashSplit": "Hash split", @@ -551,9 +597,12 @@ "timezoneTip": "指定时区，否则不做时区处理", "closeNotNull": "忽略NotNull", "closeNotNullTooltip": "开关打开时会将非空限制丢弃", + "tableOwner": "指定表所有者", + "tableOwnerTooltip": "可以指定表所有者，不指定则默认为当前用户，如果权限不足请使用足够权限的用户执行：grant <被指定owner> to <数据同步用户> ", "createAutoInc": "同步自增列", "createAutoIncTooltip": "仅支持PG10版本以上，开关打开时会在进入增量时同步自增属性", "autoIncJumpValue": "自增键跳跃值", + "autoIncCacheValue": "自增键缓存值", "applyDefault": "应用默认值", "applyDefaultTooltip": "开关打开时会将默认值应用到目标，如果有未适配的函数或表达式，可能会导致报错", "hashSplit": "哈希分片", @@ -586,9 +635,12 @@ "timezoneTip": "指定時區，否則不做時區處理", "closeNotNull": "忽略NotNull", "closeNotNullTooltip": "開關打開時會將非空限制丟棄", + "tableOwner": "指定表所有者", + "tableOwnerTooltip": "可以指定表所有者，不指定則默認為當前用戶，如果權限不足請使用足夠權限的用戶執行：grant <被指定owner> to <數據同步用戶> ", "createAutoInc": "同步自增列", "createAutoIncTooltip": "僅支持PG10版本以上，開關打開時會在進入增量時同步自增屬性", "autoIncJumpValue": "自增鍵跳躍值", + "autoIncCacheValue": "自增鍵緩存值", "applyDefault": "應用默認值", "applyDefaultTooltip": "開關打開時會將默認值應用到目標，如果有未適配的函數或表達式，可能會導致報錯", "hashSplit": "哈希分片", @@ -845,12 +897,10 @@ "queryOnly": true }, "xml": { - "to": "TapString", - "queryOnly": true + "to": "TapXml" }, "json": { - "to": "TapString", - "queryOnly": true + "to": "TapJson" }, "tsvector": { "to": "TapString", diff --git a/connectors/redis-connector/pom.xml b/connectors/redis-connector/pom.xml index c707d1f23..256694f3c 100644 --- a/connectors/redis-connector/pom.xml +++ b/connectors/redis-connector/pom.xml @@ -61,7 +61,7 @@ 8 UTF-8 - 1.4.1-SNAPSHOT + 1.4.4-SNAPSHOT diff --git a/connectors/selectdb-connector/pom.xml b/connectors/selectdb-connector/pom.xml index 0706d5ed9..e877ee7b3 100644 --- a/connectors/selectdb-connector/pom.xml +++ b/connectors/selectdb-connector/pom.xml @@ -15,7 +15,7 @@ 8 - 1.4.1-SNAPSHOT + 1.4.4-SNAPSHOT 1.0-SNAPSHOT diff --git a/connectors/selectdb-connector/src/main/java/io/tapdata/connector/selectdb/SelectDbConnector.java b/connectors/selectdb-connector/src/main/java/io/tapdata/connector/selectdb/SelectDbConnector.java index f2b8da8e1..3aca3f47c 100644 --- a/connectors/selectdb-connector/src/main/java/io/tapdata/connector/selectdb/SelectDbConnector.java +++ b/connectors/selectdb-connector/src/main/java/io/tapdata/connector/selectdb/SelectDbConnector.java @@ -144,7 +144,7 @@ public void registerCapabilities(ConnectorFunctions connectorFunctions, TapCodec return 0; }); codecRegistry.registerFromTapValue(TapBinaryValue.class, "text", tapValue -> { - if (tapValue != null && tapValue.getValue() != null) + if (tapValue != null && tapValue.getValue() != null && tapValue.getValue().getValue() != null) return toJson(tapValue.getValue()); return "null"; }); @@ -515,4 +515,4 @@ protected void dropTable(TapConnectorContext tapConnectorContext, TapDropTableEv protected void getTableNames(TapConnectionContext tapConnectionContext, int batchSize, Consumer> listConsumer) { selectDbJdbcContext.queryAllTables(TapSimplify.list(), batchSize, listConsumer); } -} \ No newline at end of file +} diff --git a/connectors/tdd-connector/src/main/java/io/tapdata/connector/tdd/TDDBenchmarkTargetConnector.java b/connectors/tdd-connector/src/main/java/io/tapdata/connector/tdd/TDDBenchmarkTargetConnector.java index 2bc68f8d0..257cd950d 100644 --- a/connectors/tdd-connector/src/main/java/io/tapdata/connector/tdd/TDDBenchmarkTargetConnector.java +++ b/connectors/tdd-connector/src/main/java/io/tapdata/connector/tdd/TDDBenchmarkTargetConnector.java @@ -156,7 +156,7 @@ public void registerCapabilities(ConnectorFunctions connectorFunctions, TapCodec return 0; }); codecRegistry.registerFromTapValue(TapBinaryValue.class, "text", tapValue -> { - if (tapValue != null && tapValue.getValue() != null) + if (tapValue != null && tapValue.getValue() != null && tapValue.getValue().getValue() != null) return "binary";//toJson(tapValue.getValue()); return "null"; }); diff --git a/connectors/tdd-connector/src/main/java/io/tapdata/connector/tdd/TDDTargetConnector.java b/connectors/tdd-connector/src/main/java/io/tapdata/connector/tdd/TDDTargetConnector.java index c06221032..a098b8d86 100644 --- a/connectors/tdd-connector/src/main/java/io/tapdata/connector/tdd/TDDTargetConnector.java +++ b/connectors/tdd-connector/src/main/java/io/tapdata/connector/tdd/TDDTargetConnector.java @@ -152,8 +152,8 @@ public void registerCapabilities(ConnectorFunctions connectorFunctions, TapCodec return 0; }); codecRegistry.registerFromTapValue(TapBinaryValue.class, "text", tapValue -> { - if (tapValue != null && tapValue.getValue() != null) - return toJson(tapValue.getValue()); + if (tapValue != null && tapValue.getValue() != null && tapValue.getValue().getValue() != null) + return toJson(tapValue.getValue().getValue()); return "null"; }); codecRegistry.registerFromTapValue(TapTimeValue.class, "datetime", tapValue -> { diff --git a/connectors/tdengine-connector/src/main/java/io/tapdata/connector/tdengine/bean/TDengineColumn.java b/connectors/tdengine-connector/src/main/java/io/tapdata/connector/tdengine/bean/TDengineColumn.java index 1ae70382b..8db0a9f1d 100644 --- a/connectors/tdengine-connector/src/main/java/io/tapdata/connector/tdengine/bean/TDengineColumn.java +++ b/connectors/tdengine-connector/src/main/java/io/tapdata/connector/tdengine/bean/TDengineColumn.java @@ -58,7 +58,7 @@ protected Boolean isNullable() { return "YES".equals(this.nullable); } - private String getDefaultValue(String defaultValue) { + protected String getDefaultValue(String defaultValue) { if (EmptyKit.isNull(defaultValue) || defaultValue.startsWith("NULL::")) { return null; } else if (defaultValue.contains("::")) { diff --git a/connectors/tencent-db-mariadb-connector/pom.xml b/connectors/tencent-db-mariadb-connector/pom.xml index a7863bb69..4618ede6a 100644 --- a/connectors/tencent-db-mariadb-connector/pom.xml +++ b/connectors/tencent-db-mariadb-connector/pom.xml @@ -23,7 +23,7 @@ 8 - 1.4.1-SNAPSHOT + 1.4.4-SNAPSHOT diff --git a/connectors/tencent-db-mongodb-connector/pom.xml b/connectors/tencent-db-mongodb-connector/pom.xml index 440bff32d..b97efb25d 100644 --- a/connectors/tencent-db-mongodb-connector/pom.xml +++ b/connectors/tencent-db-mongodb-connector/pom.xml @@ -23,7 +23,7 @@ 8 - 1.4.1-SNAPSHOT + 1.4.4-SNAPSHOT diff --git a/connectors/tencent-db-postgres-connector/pom.xml b/connectors/tencent-db-postgres-connector/pom.xml index 1e9c8bd30..a4504c1ee 100644 --- a/connectors/tencent-db-postgres-connector/pom.xml +++ b/connectors/tencent-db-postgres-connector/pom.xml @@ -23,7 +23,7 @@ 8 - 1.4.1-SNAPSHOT + 1.4.4-SNAPSHOT diff --git a/connectors/tidb-connector/pom.xml b/connectors/tidb-connector/pom.xml index f8c7d92a1..8a67849bb 100644 --- a/connectors/tidb-connector/pom.xml +++ b/connectors/tidb-connector/pom.xml @@ -19,7 +19,7 @@ 1.0-SNAPSHOT 1.0-SNAPSHOT 3.12.0 - 1.4.1-SNAPSHOT + 1.4.4-SNAPSHOT 1.7.15 2.17.1 diff --git a/connectors/vastbase-connector/pom.xml b/connectors/vastbase-connector/pom.xml index 0385f920a..654bc8ae5 100644 --- a/connectors/vastbase-connector/pom.xml +++ b/connectors/vastbase-connector/pom.xml @@ -15,7 +15,7 @@ 8 - 1.4.1-SNAPSHOT + 1.4.4-SNAPSHOT 1.5.4.Final diff --git a/connectors/vastbase-connector/src/main/java/io/tapdata/connector/postgres/VastbaseConnector.java b/connectors/vastbase-connector/src/main/java/io/tapdata/connector/postgres/VastbaseConnector.java index 7469847df..d93039a41 100644 --- a/connectors/vastbase-connector/src/main/java/io/tapdata/connector/postgres/VastbaseConnector.java +++ b/connectors/vastbase-connector/src/main/java/io/tapdata/connector/postgres/VastbaseConnector.java @@ -341,8 +341,7 @@ private void initConnection(TapConnectionContext connectionContext) { } private void openIdentity(TapTable tapTable) throws SQLException { - if (EmptyKit.isEmpty(tapTable.primaryKeys()) - && (EmptyKit.isEmpty(tapTable.getIndexList()) || tapTable.getIndexList().stream().noneMatch(TapIndex::isUnique))) { + if (EmptyKit.isEmpty(tapTable.primaryKeys())) { jdbcContext.execute("ALTER TABLE \"" + jdbcContext.getConfig().getSchema() + "\".\"" + tapTable.getId() + "\" REPLICA IDENTITY FULL"); } } diff --git a/connectors/xml-connector/pom.xml b/connectors/xml-connector/pom.xml index 2714c5985..a8716e9e9 100644 --- a/connectors/xml-connector/pom.xml +++ b/connectors/xml-connector/pom.xml @@ -15,7 +15,7 @@ jar - 1.4.1-SNAPSHOT + 1.4.4-SNAPSHOT 8 diff --git a/connectors/yashandb-connector/pom.xml b/connectors/yashandb-connector/pom.xml index 78fab0e32..1d5e5f8c1 100644 --- a/connectors/yashandb-connector/pom.xml +++ b/connectors/yashandb-connector/pom.xml @@ -23,7 +23,7 @@ 8 - 1.4.1-SNAPSHOT + 1.4.4-SNAPSHOT diff --git a/connectors/yashandb-connector/src/main/java/io/tapdata/connector/yashandb/YashandbConnector.java b/connectors/yashandb-connector/src/main/java/io/tapdata/connector/yashandb/YashandbConnector.java index b1e501028..88bfcbb1c 100644 --- a/connectors/yashandb-connector/src/main/java/io/tapdata/connector/yashandb/YashandbConnector.java +++ b/connectors/yashandb-connector/src/main/java/io/tapdata/connector/yashandb/YashandbConnector.java @@ -83,7 +83,7 @@ public void registerCapabilities(ConnectorFunctions connectorFunctions, TapCodec }); codecRegistry.registerFromTapValue(TapBinaryValue.class, "CLOB", tapMapValue -> { - if (tapMapValue != null && tapMapValue.getValue() != null) return toJson(tapMapValue.getValue()); + if (tapMapValue != null && tapMapValue.getValue() != null && tapMapValue.getValue().getValue() != null) return toJson(tapMapValue.getValue().getValue()); return "null"; }); diff --git a/file-storages/local-file/pom.xml b/file-storages/local-file/pom.xml index 641a92a0c..f56fc9b63 100644 --- a/file-storages/local-file/pom.xml +++ b/file-storages/local-file/pom.xml @@ -22,7 +22,7 @@ io.tapdata tapdata-pdk-api - 1.4.1-SNAPSHOT + 1.4.4-SNAPSHOT provided diff --git a/file-storages/pom.xml b/file-storages/pom.xml index 522747e97..5930afdd2 100644 --- a/file-storages/pom.xml +++ b/file-storages/pom.xml @@ -21,7 +21,7 @@ ${project.artifactId}-v${project.version} 8 1.10-SNAPSHOT - 1.4.1-SNAPSHOT + 1.4.4-SNAPSHOT 1.0-SNAPSHOT 5.8.1 1.8.1 diff --git a/tapdata-cli/pom.xml b/tapdata-cli/pom.xml index 648869c3a..ef8207e42 100644 --- a/tapdata-cli/pom.xml +++ b/tapdata-cli/pom.xml @@ -27,12 +27,12 @@ io.tapdata tapdata-pdk-runner - 1.10-SNAPSHOT + 1.12-SNAPSHOT io.tapdata script-engine-module - 1.0-SNAPSHOT + 1.1-SNAPSHOT io.tapdata