Skip to content

Commit cb8ee3b

Browse files
mikekamornikovmikekamornikov
committed
CXP-2779: new SchemaHistory interface, removed deprecated
1 parent 1b3d22a commit cb8ee3b

24 files changed

Lines changed: 527 additions & 277 deletions

File tree

‎debezium-connector-mysql/src/main/java/io/debezium/connector/mysql/MySqlDatabaseSchema.java‎

Lines changed: 9 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,7 @@
2929
import io.debezium.relational.TableSchema;
3030
import io.debezium.relational.TableSchemaBuilder;
3131
import io.debezium.relational.Tables;
32+
import io.debezium.relational.Tables.TableFilter;
3233
import io.debezium.relational.ddl.DdlChanges;
3334
import io.debezium.relational.ddl.DdlParser;
3435
import io.debezium.relational.ddl.DdlParserListener.Event;
@@ -64,7 +65,6 @@ public class MySqlDatabaseSchema extends HistorizedRelationalDatabaseSchema {
6465
private final static Logger LOGGER = LoggerFactory.getLogger(MySqlDatabaseSchema.class);
6566

6667
private final Set<String> ignoredQueryStatements = Collect.unmodifiableSet("BEGIN", "END", "FLUSH PRIVILEGES");
67-
private final DdlParser ddlParser;
6868
private final RelationalTableFilters filters;
6969
private final DdlChanges ddlChanges;
7070
private final Map<Long, TableId> tableIdsByTableNumber = new ConcurrentHashMap<>();
@@ -88,14 +88,14 @@ public MySqlDatabaseSchema(MySqlConnectorConfig connectorConfig, MySqlValueConve
8888
connectorConfig.getSourceInfoStructMaker().schema(),
8989
connectorConfig.getFieldNamer(),
9090
false),
91-
tableIdCaseInsensitive, connectorConfig.getKeyMapper());
92-
93-
this.ddlParser = new MySqlAntlrDdlParser(
94-
true,
95-
false,
96-
connectorConfig.isSchemaCommentsHistoryEnabled(),
97-
valueConverter,
98-
getTableFilter());
91+
tableIdCaseInsensitive, connectorConfig.getKeyMapper(),
92+
new MySqlAntlrDdlParser(
93+
true,
94+
false,
95+
connectorConfig.isSchemaCommentsHistoryEnabled(),
96+
valueConverter,
97+
connectorConfig.getTableFilters().dataCollectionFilter()));
98+
9999
this.ddlChanges = this.ddlParser.getDdlChanges();
100100
this.connectorConfig = connectorConfig;
101101
filters = connectorConfig.getTableFilters();
@@ -354,11 +354,6 @@ else if (event instanceof TableIndexEvent) {
354354
return null;
355355
}
356356

357-
@Override
358-
protected DdlParser getDdlParser() {
359-
return ddlParser;
360-
}
361-
362357
/**
363358
* Return true if the database schema history entity exists
364359
*/

‎debezium-connector-mysql/src/test/java/io/debezium/relational/history/AbstractSchemaHistoryTest.java‎

Lines changed: 90 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -9,14 +9,25 @@
99
import static org.junit.Assert.fail;
1010

1111
import java.util.Map;
12+
import java.time.Instant;
13+
14+
import org.apache.kafka.connect.data.Schema;
15+
import org.apache.kafka.connect.data.Struct;
1216

1317
import org.junit.After;
1418
import org.junit.Before;
1519
import org.junit.Test;
1620

21+
import io.debezium.connector.SnapshotRecord;
22+
import io.debezium.connector.mysql.MySqlPartition;
23+
import io.debezium.connector.mysql.SourceInfo;
1724
import io.debezium.connector.mysql.antlr.MySqlAntlrDdlParser;
25+
import io.debezium.pipeline.spi.OffsetContext;
26+
import io.debezium.pipeline.spi.Offsets;
27+
import io.debezium.pipeline.txmetadata.TransactionContext;
1828
import io.debezium.relational.Tables;
1929
import io.debezium.relational.ddl.DdlParser;
30+
import io.debezium.spi.schema.DataCollectionId;
2031
import io.debezium.util.Collect;
2132
import io.debezium.util.Testing;
2233

@@ -26,30 +37,22 @@
2637
*/
2738
public abstract class AbstractSchemaHistoryTest {
2839

40+
private static String SERVERNAME = "server1";
41+
private static String BINLOG_FILE = "a.log";
42+
2943
protected SchemaHistory history;
30-
protected Map<String, Object> source1;
31-
protected Map<String, Object> source2;
32-
protected Tables tables;
33-
protected Tables t0;
34-
protected Tables t1;
35-
protected Tables t2;
36-
protected Tables t3;
37-
protected Tables t4;
38-
protected Tables all;
44+
protected Tables t0, t1, t2, t3, t4, all;
3945
protected DdlParser parser;
4046

4147
@Before
4248
public void beforeEach() {
4349
parser = new MySqlAntlrDdlParser();
44-
tables = new Tables();
4550
t0 = new Tables();
4651
t1 = new Tables();
4752
t2 = new Tables();
4853
t3 = new Tables();
4954
t4 = new Tables();
5055
all = new Tables();
51-
source1 = server("abc");
52-
source2 = server("xyz");
5356
history = createHistory();
5457
}
5558

@@ -62,17 +65,24 @@ public void afterEach() {
6265

6366
protected abstract SchemaHistory createHistory();
6467

65-
protected Map<String, Object> server(String serverName) {
66-
return Collect.linkMapOf("server", serverName);
68+
protected MySqlPartition source(String server) {
69+
return new MySqlPartition(server, "");
70+
}
71+
72+
protected Map<String, Object> position(long position, int row) {
73+
return Collect.linkMapOf(
74+
SourceInfo.BINLOG_FILENAME_OFFSET_KEY, BINLOG_FILE,
75+
SourceInfo.BINLOG_POSITION_OFFSET_KEY, position,
76+
SourceInfo.BINLOG_ROW_IN_EVENT_OFFSET_KEY, row);
6777
}
6878

69-
protected Map<String, Object> position(String filename, long position, int entry) {
70-
return Collect.linkMapOf("file", filename, "position", position, "entry", entry);
79+
protected Offsets<?, ?> offsets(String server, long pos, int row) {
80+
return Offsets.of(source(server), new TestOffsetContext(position(pos, row)));
7181
}
7282

73-
protected void record(long pos, int entry, String ddl, Tables... update) {
83+
protected void record(long pos, int row, String ddl, Tables... update) {
7484
try {
75-
history.record(source1, position("a.log", pos, entry), "db", ddl);
85+
history.record(source(SERVERNAME).getSourcePartition(), position(pos, row), "db", ddl);
7686
}
7787
catch (Throwable t) {
7888
fail(t.getMessage());
@@ -85,15 +95,15 @@ protected void record(long pos, int entry, String ddl, Tables... update) {
8595
}
8696
}
8797

88-
protected Tables recover(long pos, int entry) {
98+
protected Tables recover(long pos, int row) {
8999
Tables result = new Tables();
90-
history.recover(source1, position("a.log", pos, entry), result, parser);
100+
history.recover(offsets(SERVERNAME, pos, row), result);
91101
return result;
92102
}
93103

94104
@Test
95105
public void shouldRecordChangesAndRecoverToVariousPoints() {
96-
record(01, 0, "CREATE TABLE foo ( first VARCHAR(22) NOT NULL );", all, t3, t2, t1, t0);
106+
record(1, 0,"CREATE TABLE foo ( first VARCHAR(22) NOT NULL );", all, t3, t2, t1, t0);
97107
record(23, 1, "CREATE TABLE\nperson ( name VARCHAR(22) NOT NULL );", all, t3, t2, t1);
98108
record(30, 2, "CREATE TABLE address\n( street VARCHAR(22) NOT NULL );", all, t3, t2);
99109
record(32, 3, "ALTER TABLE address ADD city VARCHAR(22) NOT NULL;", all, t3);
@@ -106,8 +116,8 @@ public void shouldRecordChangesAndRecoverToVariousPoints() {
106116
Testing.print("t3 = " + t3);
107117
}
108118

109-
assertThat(recover(01, 0)).isEqualTo(t0);
110-
assertThat(recover(01, 3)).isEqualTo(t0);
119+
assertThat(recover(1, 0)).isEqualTo(t0);
120+
assertThat(recover(1, 3)).isEqualTo(t0);
111121
assertThat(recover(10, 1)).isEqualTo(t0);
112122
assertThat(recover(22, 999999)).isEqualTo(t0);
113123
assertThat(recover(23, 0)).isEqualTo(t0);
@@ -130,4 +140,61 @@ public void shouldRecordChangesAndRecoverToVariousPoints() {
130140
assertThat(recover(1033, 4)).isEqualTo(t3);
131141
}
132142

143+
private static class TestOffsetContext implements OffsetContext {
144+
private final Map<String, ?> offset;
145+
146+
TestOffsetContext(Map<String, ?> offset) {
147+
this.offset = offset;
148+
}
149+
150+
@Override
151+
public Map<String, ?> getOffset() {
152+
return offset;
153+
}
154+
155+
@Override
156+
public Schema getSourceInfoSchema() {
157+
return null;
158+
}
159+
160+
@Override
161+
public Struct getSourceInfo() {
162+
return null;
163+
}
164+
165+
@Override
166+
public boolean isSnapshotRunning() {
167+
return false;
168+
}
169+
170+
@Override
171+
public void markSnapshotRecord(SnapshotRecord record) {
172+
173+
}
174+
175+
@Override
176+
public void preSnapshotStart() {
177+
178+
}
179+
180+
@Override
181+
public void preSnapshotCompletion() {
182+
183+
}
184+
185+
@Override
186+
public void postSnapshotCompletion() {
187+
188+
}
189+
190+
@Override
191+
public void event(DataCollectionId collectionId, Instant timestamp) {
192+
193+
}
194+
195+
@Override
196+
public TransactionContext getTransactionContext() {
197+
return null;
198+
}
199+
}
133200
}

‎debezium-connector-mysql/src/test/java/io/debezium/relational/history/FileSchemaHistoryTest.java‎

Lines changed: 9 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -30,10 +30,17 @@ public void beforeEach() {
3030
@Override
3131
protected SchemaHistory createHistory() {
3232
SchemaHistory history = new FileSchemaHistory();
33-
history.configure(Configuration.create()
33+
34+
Configuration config = Configuration.create()
3435
.with(FileSchemaHistory.FILE_PATH, TEST_FILE_PATH.toAbsolutePath().toString())
35-
.build(), null, SchemaHistoryMetrics.NOOP, true);
36+
.build();
37+
HistoryRecordComparator comparator = null;
38+
SchemaHistoryListener listener = SchemaHistoryMetrics.NOOP;
39+
HistoryRecordProcessorProvider processorProvider = (o, s) -> new HistoryRecordProcessor(o, s, parser, config, comparator, listener, true);
40+
41+
history.configure(config, comparator, listener, processorProvider);
3642
history.start();
43+
3744
return history;
3845
}
3946
}

‎debezium-connector-mysql/src/test/java/io/debezium/relational/history/KafkaSchemaHistoryTest.java‎

Lines changed: 36 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -141,7 +141,15 @@ private void testHistoryTopicContent(String topicName, boolean skipUnparseableDD
141141
.with(KafkaSchemaHistory.INTERNAL_CONNECTOR_CLASS, "org.apache.kafka.connect.source.SourceConnector")
142142
.with(KafkaSchemaHistory.INTERNAL_CONNECTOR_ID, "dbz-test")
143143
.build();
144-
history.configure(config, null, SchemaHistoryMetrics.NOOP, true);
144+
145+
DdlParser recoveryParser = new MySqlAntlrDdlParser();
146+
DdlParser ddlParser = new MySqlAntlrDdlParser();
147+
148+
HistoryRecordComparator comparator = null;
149+
SchemaHistoryListener listener = SchemaHistoryMetrics.NOOP;
150+
HistoryRecordProcessorProvider processorProvider = (o, s) -> new HistoryRecordProcessor(o, s, recoveryParser, config, comparator, listener, true);
151+
152+
history.configure(config, comparator, listener, processorProvider);
145153
history.start();
146154

147155
// Should be able to call start more than once ...
@@ -152,16 +160,14 @@ private void testHistoryTopicContent(String topicName, boolean skipUnparseableDD
152160
// Calling it another time to ensure we can work with the DB history topic already existing
153161
history.initializeStorage();
154162

155-
DdlParser recoveryParser = new MySqlAntlrDdlParser();
156-
DdlParser ddlParser = new MySqlAntlrDdlParser();
157163
ddlParser.setCurrentSchema("db1"); // recover does this, so we need to as well
158164
Tables tables1 = new Tables();
159165
Tables tables2 = new Tables();
160166
Tables tables3 = new Tables();
161167

162168
// Recover from the very beginning ...
163169
setLogPosition(0);
164-
history.recover(offsets, tables1, recoveryParser);
170+
history.recover(offsets, tables1);
165171

166172
// There should have been nothing to recover ...
167173
assertThat(tables1.size()).isEqualTo(0);
@@ -200,31 +206,31 @@ private void testHistoryTopicContent(String topicName, boolean skipUnparseableDD
200206
// Stop the history (which should stop the producer) ...
201207
history.stop();
202208
history = new KafkaSchemaHistory();
203-
history.configure(config, null, SchemaHistoryListener.NOOP, true);
209+
history.configure(config, comparator, listener, processorProvider);
204210
// no need to start
205211

206212
// Recover from the very beginning to just past the first change ...
207213
Tables recoveredTables = new Tables();
208214
setLogPosition(15);
209-
history.recover(offsets, recoveredTables, recoveryParser);
215+
history.recover(offsets, recoveredTables);
210216
assertThat(recoveredTables).isEqualTo(tables1);
211217

212218
// Recover from the very beginning to just past the second change ...
213219
recoveredTables = new Tables();
214220
setLogPosition(50);
215-
history.recover(offsets, recoveredTables, recoveryParser);
221+
history.recover(offsets, recoveredTables);
216222
assertThat(recoveredTables).isEqualTo(tables2);
217223

218224
// Recover from the very beginning to just past the third change ...
219225
recoveredTables = new Tables();
220226
setLogPosition(10010);
221-
history.recover(offsets, recoveredTables, recoveryParser);
227+
history.recover(offsets, recoveredTables);
222228
assertThat(recoveredTables).isEqualTo(tables3);
223229

224230
// Recover from the very beginning to way past the third change ...
225231
recoveredTables = new Tables();
226232
setLogPosition(100000010);
227-
history.recover(offsets, recoveredTables, recoveryParser);
233+
history.recover(offsets, recoveredTables);
228234
assertThat(recoveredTables).isEqualTo(tables3);
229235
}
230236

@@ -355,7 +361,13 @@ public void testExists() {
355361
.with(KafkaSchemaHistory.INTERNAL_CONNECTOR_ID, "dbz-test")
356362
.build();
357363

358-
history.configure(config, null, SchemaHistoryMetrics.NOOP, true);
364+
DdlParser recoveryParser = new MySqlAntlrDdlParser();
365+
366+
HistoryRecordComparator comparator = null;
367+
SchemaHistoryListener listener = SchemaHistoryMetrics.NOOP;
368+
HistoryRecordProcessorProvider processorProvider = (o, s) -> new HistoryRecordProcessor(o, s, recoveryParser, config, comparator, listener, true);
369+
370+
history.configure(config, comparator, listener, processorProvider);
359371
history.start();
360372

361373
// dummytopic should not exist yet
@@ -380,7 +392,13 @@ public void differentiateStorageExistsFromHistoryExists() {
380392
50000)
381393
.build();
382394

383-
history.configure(config, null, SchemaHistoryMetrics.NOOP, true);
395+
DdlParser recoveryParser = new MySqlAntlrDdlParser();
396+
397+
HistoryRecordComparator comparator = null;
398+
SchemaHistoryListener listener = SchemaHistoryMetrics.NOOP;
399+
HistoryRecordProcessorProvider processorProvider = (o, s) -> new HistoryRecordProcessor(o, s, recoveryParser, config, comparator, listener, true);
400+
401+
history.configure(config, comparator, listener, processorProvider);
384402

385403
assertFalse(history.storageExists());
386404
history.initializeStorage();
@@ -425,7 +443,13 @@ public void shouldConnectionTimeoutIfValueIsTooLow() {
425443
.with(KafkaSchemaHistory.KAFKA_QUERY_TIMEOUT_MS, 1)
426444
.build();
427445

428-
history.configure(config, null, SchemaHistoryMetrics.NOOP, true);
446+
DdlParser recoveryParser = new MySqlAntlrDdlParser();
447+
448+
HistoryRecordComparator comparator = null;
449+
SchemaHistoryListener listener = SchemaHistoryMetrics.NOOP;
450+
HistoryRecordProcessorProvider processorProvider = (o, s) -> new HistoryRecordProcessor(o, s, recoveryParser, config, comparator, listener, true);
451+
452+
history.configure(config, comparator, listener, processorProvider);
429453
history.start();
430454

431455
try {

‎debezium-connector-mysql/src/test/java/io/debezium/relational/history/MemorySchemaHistoryTest.java‎

Lines changed: 13 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,13 +5,25 @@
55
*/
66
package io.debezium.relational.history;
77

8+
import io.debezium.config.Configuration;
9+
810
/**
911
* @author Randall Hauch
1012
*/
1113
public class MemorySchemaHistoryTest extends AbstractSchemaHistoryTest {
1214

1315
@Override
1416
protected SchemaHistory createHistory() {
15-
return new MemorySchemaHistory();
17+
SchemaHistory history = new MemorySchemaHistory();
18+
19+
Configuration config = Configuration.empty();
20+
HistoryRecordComparator comparator = null;
21+
SchemaHistoryListener listener = SchemaHistoryMetrics.NOOP;
22+
HistoryRecordProcessorProvider processorProvider = (o, s) -> new HistoryRecordProcessor(o, s, parser, config, comparator, listener, true);
23+
24+
history.configure(config, comparator, listener, processorProvider);
25+
history.start();
26+
27+
return history;
1628
}
1729
}

0 commit comments

Comments
 (0)