Skip to content

Commit 1c4dad8

Browse files
dtunikovclaude
andcommitted
test(mysql): add native-JSON snapshot/CDC consistency test
Companion to Test_MySQL_JSON_SnapshotCDCConsistency that enables PEERDB_CLICKHOUSE_ENABLE_JSON so the JSON column lands in a ClickHouse JSON column. ClickHouse parses and normalizes the text on insert, which absorbs the double text-format divergence the snapshot and CDC paths cannot make byte-identical themselves (e.g. 1000000.5 vs 1.0000005e+06), so the previously-diverging float values read back identically. Compares toString(js) since GetRows does not scan the native JSON type directly. Requires ClickHouse >= 25.3, same as Test_JSON. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
1 parent fdfc0bd commit 1c4dad8

1 file changed

Lines changed: 76 additions & 0 deletions

File tree

flow/e2e/clickhouse_mysql_test.go

Lines changed: 76 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -307,6 +307,82 @@ func (s ClickHouseSuite) Test_MySQL_JSON_SnapshotCDCConsistency() {
307307
RequireEnvCanceled(s.t, env)
308308
}
309309

310+
// Test_MySQL_JSON_SnapshotCDCConsistency_NativeJSON is the native-JSON companion to
311+
// Test_MySQL_JSON_SnapshotCDCConsistency. With PEERDB_CLICKHOUSE_ENABLE_JSON the JSON column
312+
// lands in a ClickHouse JSON column rather than a String, so ClickHouse parses the text and
313+
// normalizes it on insert. That absorbs the one representation difference the snapshot and
314+
// CDC paths cannot make byte-identical themselves -- doubles whose text form differs by
315+
// magnitude (MySQL's "1000000.5" vs the CDC decoder's "1.0000005e+06") parse to the same
316+
// float and therefore read back identically here. Requires ClickHouse >= 25.3 for the
317+
// native JSON type (same assumption as Test_JSON); on older versions the column falls back
318+
// to String and this test would surface the divergence.
319+
func (s ClickHouseSuite) Test_MySQL_JSON_SnapshotCDCConsistency_NativeJSON() {
320+
if _, ok := s.source.(*MySqlSource); !ok {
321+
s.t.Skip("only applies to mysql")
322+
}
323+
324+
srcTableName := "test_json_repr_native"
325+
srcFullName := s.attachSchemaSuffix(srcTableName)
326+
dstTableName := "test_json_repr_native_dst"
327+
328+
require.NoError(s.t, s.source.Exec(s.t.Context(), fmt.Sprintf(`
329+
CREATE TABLE IF NOT EXISTS %s (
330+
id serial PRIMARY KEY,
331+
js json NOT NULL
332+
)`, srcFullName)))
333+
334+
// These deliberately include the doubles that diverge in text form between the snapshot
335+
// and CDC paths; native JSON normalization is what makes them consistent at the destination.
336+
variants := []string{
337+
`{"pi": 3.14, "big": 1000000.5, "neg": -2.5}`, // magnitude where snapshot/CDC text forms differ
338+
`{"tiny": 0.000015, "huge": 150000000000.5}`, // more exponent-crossover magnitudes
339+
`{"z": 1, "aa": 2}`, // key order, normalized consistently
340+
}
341+
342+
insertVariants := func() {
343+
for _, v := range variants {
344+
require.NoError(s.t, s.source.Exec(s.t.Context(),
345+
fmt.Sprintf(`INSERT INTO %s (js) VALUES ('%s')`, srcFullName, v)))
346+
}
347+
}
348+
349+
insertVariants()
350+
351+
connectionGen := FlowConnectionGenerationConfig{
352+
FlowJobName: s.attachSuffix(srcTableName),
353+
TableNameMapping: map[string]string{srcFullName: dstTableName},
354+
Destination: s.Peer().Name,
355+
}
356+
flowConnConfig := connectionGen.GenerateFlowConnectionConfigs(s)
357+
flowConnConfig.DoInitialSnapshot = true
358+
flowConnConfig.Env = map[string]string{"PEERDB_CLICKHOUSE_ENABLE_JSON": "true"}
359+
360+
tc := NewTemporalClient(s.t)
361+
env := ExecutePeerflow(s.t, tc, flowConnConfig)
362+
SetupCDCFlowStatusQuery(s.t, env, flowConnConfig)
363+
364+
EnvWaitForCount(env, s, "waiting for initial snapshot", dstTableName, "id", len(variants))
365+
366+
insertVariants()
367+
368+
EnvWaitForCount(env, s, "waiting for cdc", dstTableName, "id", 2*len(variants))
369+
370+
// Read ClickHouse's own normalized text of the JSON column so the comparison is over a
371+
// plain String scan target (GetRows does not scan the native JSON type directly).
372+
rows, err := s.GetRows(dstTableName, "id, toString(js)")
373+
require.NoError(s.t, err)
374+
require.Len(s.t, rows.Records, 2*len(variants))
375+
376+
for i := range variants {
377+
snapshotGot := fmt.Sprint(rows.Records[i][1].Value())
378+
cdcGot := fmt.Sprint(rows.Records[i+len(variants)][1].Value())
379+
require.Equal(s.t, snapshotGot, cdcGot, "snapshot/cdc JSON representation differs for %q", variants[i])
380+
}
381+
382+
env.Cancel(s.t.Context())
383+
RequireEnvCanceled(s.t, env)
384+
}
385+
310386
func (s ClickHouseSuite) Test_MySQL_TransactionPayloadCompression() {
311387
mySource, ok := s.source.(*MySqlSource)
312388
if !ok {

0 commit comments

Comments
 (0)