From 83acb2cd1df1b1ebcd7989da3f38a8e3783e6b8a Mon Sep 17 00:00:00 2001 From: Nikhil Rajput Date: Mon, 22 Jun 2026 17:45:07 +0530 Subject: [PATCH 01/13] refactor(canonical): extract canonical types + pure builders to internal/canonical leaf MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Move types (CanonicalType, CanonicalColumn, CanonicalTable, CanonicalSchema, CanonicalRow) and MapToNative from internal/app/canonical.go to new leaf package internal/canonical/canonical.go - Add PrimaryKey bool field to CanonicalColumn - Add new types: ChangeOp (OpInsert/OpUpdate/OpDelete), CanonicalChange, Position - Export quoteIdent→QuoteIdent and placeholder→Placeholder - Move pure db-free builders (BuildSelectSQL, BuildCreateTableSQL, BuildInsertSQL, WriteJSONL, NormalizeScanned) to internal/canonical - Update canonical_emit.go and canonical_consume.go to import and qualify canonical.* symbols; db-touching orchestration stays in internal/app - Move pure-function tests to internal/canonical/canonical_test.go - internal/canonical imports stdlib only (leaf package invariant) --- internal/app/canonical.go | 140 --------- internal/app/canonical_consume.go | 69 +---- internal/app/canonical_emit.go | 59 +--- internal/app/canonical_test.go | 408 +-------------------------- internal/canonical/canonical.go | 265 +++++++++++++++++ internal/canonical/canonical_test.go | 407 ++++++++++++++++++++++++++ 6 files changed, 692 insertions(+), 656 deletions(-) delete mode 100644 internal/app/canonical.go create mode 100644 internal/canonical/canonical.go create mode 100644 internal/canonical/canonical_test.go diff --git a/internal/app/canonical.go b/internal/app/canonical.go deleted file mode 100644 index 5903768..0000000 --- a/internal/app/canonical.go +++ /dev/null @@ -1,140 +0,0 @@ -package app - -import ( - "fmt" - "strconv" - "strings" -) - -// CanonicalType is the engine-independent column type vocabulary. A source -// engine's native types are normalized into these; a target engine maps them -// back to its own dialect via MapToNative. This is the pivot that lets data -// move between Postgres, MySQL, and MariaDB. -type CanonicalType string - -const ( - CTInt CanonicalType = "int" - CTBigInt CanonicalType = "bigint" - CTText CanonicalType = "text" - CTVarchar CanonicalType = "varchar" - CTBoolean CanonicalType = "boolean" - CTNumeric CanonicalType = "numeric" - CTUUID CanonicalType = "uuid" - CTTimestampTZ CanonicalType = "timestamptz" - CTJSON CanonicalType = "json" -) - -// CanonicalColumn describes one column in canonical terms. -// -// Precision/Scale are only meaningful for CTVarchar (Precision = length) and -// CTNumeric (Precision = total digits, Scale = fractional digits). When both -// are zero the type is emitted bare (e.g. VARCHAR, DECIMAL) — acceptable for -// v1; engines apply their own defaults. -type CanonicalColumn struct { - Name string `json:"name"` - Type CanonicalType `json:"type"` - Nullable bool `json:"nullable"` - Precision int `json:"precision,omitempty"` - Scale int `json:"scale,omitempty"` -} - -// CanonicalTable is a table name plus its ordered columns. -type CanonicalTable struct { - Name string `json:"name"` - Columns []CanonicalColumn `json:"columns"` -} - -// CanonicalSchema is the full set of tables to transfer. -type CanonicalSchema struct { - Tables []CanonicalTable `json:"tables"` -} - -// CanonicalRow is one row of data, keyed by column name. The compact JSON keys -// keep the JSONL stream small across many rows. -type CanonicalRow struct { - Table string `json:"t"` - Values map[string]any `json:"v"` -} - -// MapToNative renders a canonical column as a native column type for the given -// target engine. Returns an error for an unknown engine or an unmappable type. -// -// VARCHAR/NUMERIC with Precision>0 carry their precision (and Scale for -// NUMERIC) into the rendered type; otherwise the type is emitted bare. -func MapToNative(engine string, col CanonicalColumn) (string, error) { - var m map[CanonicalType]string - switch engine { - case "postgres": - m = map[CanonicalType]string{ - CTInt: "integer", - CTBigInt: "bigint", - CTText: "text", - CTVarchar: "varchar", - CTBoolean: "boolean", - CTNumeric: "numeric", - CTUUID: "uuid", - CTTimestampTZ: "timestamptz", - CTJSON: "jsonb", - } - case "mysql", "mariadb": - m = map[CanonicalType]string{ - CTInt: "INT", - CTBigInt: "BIGINT", - CTText: "TEXT", - CTVarchar: "VARCHAR", - CTBoolean: "TINYINT(1)", - CTNumeric: "DECIMAL", - CTUUID: "CHAR(36)", - CTTimestampTZ: "TIMESTAMP", - CTJSON: "JSON", - } - default: - return "", fmt.Errorf("cross-engine: unknown engine %q", engine) - } - - native, ok := m[col.Type] - if !ok { - return "", fmt.Errorf("cross-engine: engine %q has no mapping for canonical type %q", engine, col.Type) - } - - // Decorate variable-length / fixed-precision types when precision is known. - switch col.Type { - case CTVarchar: - if col.Precision > 0 { - native = fmt.Sprintf("%s(%d)", native, col.Precision) - } - case CTNumeric: - if col.Precision > 0 { - native = fmt.Sprintf("%s(%d,%d)", native, col.Precision, col.Scale) - } - } - return native, nil -} - -// quoteIdent quotes a SQL identifier for the given engine, escaping any quote -// characters inside it. Identifiers (table/column names) CANNOT be passed as -// bound parameters, so quoting+escaping per engine is the mitigation against a -// name that contains a space, keyword, quote, or `;` — whether malicious or -// merely awkward. Every identifier that reaches generated SQL must pass through -// here. -func quoteIdent(engine, ident string) (string, error) { - switch engine { - case "postgres": - return `"` + strings.ReplaceAll(ident, `"`, `""`) + `"`, nil - case "mysql", "mariadb": - return "`" + strings.ReplaceAll(ident, "`", "``") + "`", nil - default: - return "", fmt.Errorf("cross-engine: unknown engine %q", engine) - } -} - -// placeholder returns the bind-parameter placeholder for position n (1-based) -// in the given engine's dialect: $n for Postgres, ? for MySQL/MariaDB. -func placeholder(engine string, n int) string { - switch engine { - case "postgres": - return "$" + strconv.Itoa(n) - default: - return "?" - } -} diff --git a/internal/app/canonical_consume.go b/internal/app/canonical_consume.go index ae9ca80..e417363 100644 --- a/internal/app/canonical_consume.go +++ b/internal/app/canonical_consume.go @@ -7,14 +7,15 @@ import ( "errors" "fmt" "io" - "strings" + + "github.com/nixrajput/siphon/internal/canonical" ) // ConsumeCanonical reads a stream produced by EmitCanonical and replays it into // db using engine's dialect: the first JSON value is the schema header (used to // CREATE TABLE IF NOT EXISTS for each table), every subsequent JSON value is a // CanonicalRow that is INSERTed. Values are always bound as parameters; only -// identifiers are interpolated (quoted via quoteIdent). +// identifiers are interpolated (quoted via canonical.QuoteIdent). // // A json.Decoder reads successive JSON values directly, so there is no // token-size cap: a single large row can no longer abort the stream mid-replay @@ -23,7 +24,7 @@ func ConsumeCanonical(ctx context.Context, db *sql.DB, engine string, r io.Reade dec := json.NewDecoder(r) var header struct { - Schema *CanonicalSchema `json:"schema"` + Schema *canonical.CanonicalSchema `json:"schema"` } if err := dec.Decode(&header); err != nil { if errors.Is(err, io.EOF) { @@ -40,7 +41,7 @@ func ConsumeCanonical(ctx context.Context, db *sql.DB, engine string, r io.Reade } for { - var row CanonicalRow + var row canonical.CanonicalRow if err := dec.Decode(&row); err != nil { if errors.Is(err, io.EOF) { break @@ -55,9 +56,9 @@ func ConsumeCanonical(ctx context.Context, db *sql.DB, engine string, r io.Reade } // synthCreateTables issues a CREATE TABLE IF NOT EXISTS per table. -func synthCreateTables(ctx context.Context, db *sql.DB, engine string, schema *CanonicalSchema) error { +func synthCreateTables(ctx context.Context, db *sql.DB, engine string, schema *canonical.CanonicalSchema) error { for _, t := range schema.Tables { - stmt, err := buildCreateTableSQL(engine, t) + stmt, err := canonical.BuildCreateTableSQL(engine, t) if err != nil { return err } @@ -69,7 +70,7 @@ func synthCreateTables(ctx context.Context, db *sql.DB, engine string, schema *C } // insertRow inserts a single CanonicalRow, binding values as parameters. -func insertRow(ctx context.Context, db *sql.DB, engine string, row CanonicalRow) error { +func insertRow(ctx context.Context, db *sql.DB, engine string, row canonical.CanonicalRow) error { // Build column list and value list in the SAME pass so col[i] and val[i] // stay aligned even though map iteration order is unspecified. cols := make([]string, 0, len(row.Values)) @@ -79,7 +80,7 @@ func insertRow(ctx context.Context, db *sql.DB, engine string, row CanonicalRow) vals = append(vals, v) } - stmt, err := buildInsertSQL(engine, row.Table, cols) + stmt, err := canonical.BuildInsertSQL(engine, row.Table, cols) if err != nil { return err } @@ -88,55 +89,3 @@ func insertRow(ctx context.Context, db *sql.DB, engine string, row CanonicalRow) } return nil } - -// buildCreateTableSQL renders a CREATE TABLE IF NOT EXISTS statement with every -// identifier quoted for the engine and each column mapped via MapToNative. Pure -// function — unit-testable without a DB. -func buildCreateTableSQL(engine string, t CanonicalTable) (string, error) { - qt, err := quoteIdent(engine, t.Name) - if err != nil { - return "", err - } - defs := make([]string, len(t.Columns)) - for i, c := range t.Columns { - qc, err := quoteIdent(engine, c.Name) - if err != nil { - return "", err - } - native, err := MapToNative(engine, c) - if err != nil { - return "", err - } - def := qc + " " + native - if !c.Nullable { - def += " NOT NULL" - } - defs[i] = def - } - return fmt.Sprintf("CREATE TABLE IF NOT EXISTS %s (%s)", qt, strings.Join(defs, ", ")), nil -} - -// buildInsertSQL renders an INSERT with quoted identifiers and per-engine -// placeholders. Pure function — unit-testable without a DB. Returns an error -// for an empty column set (an INSERT with no columns is malformed). -func buildInsertSQL(engine, table string, cols []string) (string, error) { - if len(cols) == 0 { - return "", fmt.Errorf("cross-engine: insert into %s: no columns", table) - } - qt, err := quoteIdent(engine, table) - if err != nil { - return "", err - } - qcols := make([]string, len(cols)) - phs := make([]string, len(cols)) - for i, c := range cols { - qc, err := quoteIdent(engine, c) - if err != nil { - return "", err - } - qcols[i] = qc - phs[i] = placeholder(engine, i+1) - } - return fmt.Sprintf("INSERT INTO %s (%s) VALUES (%s)", - qt, strings.Join(qcols, ","), strings.Join(phs, ",")), nil -} diff --git a/internal/app/canonical_emit.go b/internal/app/canonical_emit.go index b1a01f8..118e74c 100644 --- a/internal/app/canonical_emit.go +++ b/internal/app/canonical_emit.go @@ -3,10 +3,10 @@ package app import ( "context" "database/sql" - "encoding/json" "fmt" "io" - "strings" + + "github.com/nixrajput/siphon/internal/canonical" ) // EmitCanonical writes a table-by-table snapshot of schema as JSONL to w. @@ -15,8 +15,8 @@ import ( // CanonicalRow ({"t": table, "v": {col: val}}). The engine param names the // SOURCE engine and is used only to quote identifiers in the SELECT — values // are never interpolated. -func EmitCanonical(ctx context.Context, db *sql.DB, engine string, schema *CanonicalSchema, w io.Writer) error { - if err := writeJSONL(w, map[string]*CanonicalSchema{"schema": schema}); err != nil { +func EmitCanonical(ctx context.Context, db *sql.DB, engine string, schema *canonical.CanonicalSchema, w io.Writer) error { + if err := canonical.WriteJSONL(w, map[string]*canonical.CanonicalSchema{"schema": schema}); err != nil { return err } for _, t := range schema.Tables { @@ -28,8 +28,8 @@ func EmitCanonical(ctx context.Context, db *sql.DB, engine string, schema *Canon } // emitTable streams one table's rows as CanonicalRow JSONL lines. -func emitTable(ctx context.Context, db *sql.DB, engine string, t CanonicalTable, w io.Writer) error { - query, err := buildSelectSQL(engine, t) +func emitTable(ctx context.Context, db *sql.DB, engine string, t canonical.CanonicalTable, w io.Writer) error { + query, err := canonical.BuildSelectSQL(engine, t) if err != nil { return err } @@ -51,9 +51,9 @@ func emitTable(ctx context.Context, db *sql.DB, engine string, t CanonicalTable, } m := make(map[string]any, len(t.Columns)) for i, c := range t.Columns { - m[c.Name] = normalizeScanned(vals[i]) + m[c.Name] = canonical.NormalizeScanned(vals[i]) } - if err := writeJSONL(w, CanonicalRow{Table: t.Name, Values: m}); err != nil { + if err := canonical.WriteJSONL(w, canonical.CanonicalRow{Table: t.Name, Values: m}); err != nil { return err } } @@ -62,46 +62,3 @@ func emitTable(ctx context.Context, db *sql.DB, engine string, t CanonicalTable, } return nil } - -// buildSelectSQL builds `SELECT FROM ` with every -// identifier quoted for the engine. Pure function — unit-testable without a DB. -func buildSelectSQL(engine string, t CanonicalTable) (string, error) { - qt, err := quoteIdent(engine, t.Name) - if err != nil { - return "", err - } - cols := make([]string, len(t.Columns)) - for i, c := range t.Columns { - qc, err := quoteIdent(engine, c.Name) - if err != nil { - return "", err - } - cols[i] = qc - } - return fmt.Sprintf("SELECT %s FROM %s", strings.Join(cols, ", "), qt), nil -} - -// normalizeScanned converts a value scanned from database/sql into a form that -// round-trips correctly through JSON. database/sql commonly yields []byte for -// text/numeric/json columns; marshaling []byte to JSON produces base64, which -// corrupts the value when ConsumeCanonical re-inserts it. Converting to string -// makes it marshal as a JSON string instead, preserving the original bytes. -func normalizeScanned(v any) any { - if b, ok := v.([]byte); ok { - return string(b) - } - return v -} - -// writeJSONL marshals v and writes it followed by a newline. -func writeJSONL(w io.Writer, v any) error { - b, err := json.Marshal(v) - if err != nil { - return err - } - if _, err := w.Write(b); err != nil { - return err - } - _, err = w.Write([]byte("\n")) - return err -} diff --git a/internal/app/canonical_test.go b/internal/app/canonical_test.go index 438a3e7..298283e 100644 --- a/internal/app/canonical_test.go +++ b/internal/app/canonical_test.go @@ -1,407 +1,5 @@ package app -import ( - "encoding/json" - "strings" - "testing" -) - -// --- MapToNative ----------------------------------------------------------- - -func TestMapToNative_CoreTypes(t *testing.T) { - coreTypes := []CanonicalType{ - CTInt, CTBigInt, CTText, CTBoolean, CTUUID, - CTVarchar, CTNumeric, CTTimestampTZ, CTJSON, - } - for _, engine := range []string{"postgres", "mysql", "mariadb"} { - for _, ct := range coreTypes { - native, err := MapToNative(engine, CanonicalColumn{Name: "c", Type: ct}) - if err != nil { - t.Errorf("MapToNative(%q, %q): unexpected error: %v", engine, ct, err) - } - if native == "" { - t.Errorf("MapToNative(%q, %q): empty native type", engine, ct) - } - } - } -} - -func TestMapToNative_UnknownEngine(t *testing.T) { - if _, err := MapToNative("oracle", CanonicalColumn{Type: CTInt}); err == nil { - t.Fatal("MapToNative with unknown engine: want error, got nil") - } -} - -func TestMapToNative_UnknownType(t *testing.T) { - if _, err := MapToNative("postgres", CanonicalColumn{Type: CanonicalType("geometry")}); err == nil { - t.Fatal("MapToNative with unknown type: want error, got nil") - } -} - -func TestMapToNative_PrecisionDecoration(t *testing.T) { - // VARCHAR with length. - got, err := MapToNative("postgres", CanonicalColumn{Type: CTVarchar, Precision: 255}) - if err != nil { - t.Fatal(err) - } - if got != "varchar(255)" { - t.Errorf("varchar(255): got %q", got) - } - // NUMERIC with precision+scale (MySQL DECIMAL). - got, err = MapToNative("mysql", CanonicalColumn{Type: CTNumeric, Precision: 10, Scale: 2}) - if err != nil { - t.Fatal(err) - } - if got != "DECIMAL(10,2)" { - t.Errorf("DECIMAL(10,2): got %q", got) - } - // Bare when precision is zero (documented v1 behavior). - got, err = MapToNative("postgres", CanonicalColumn{Type: CTVarchar}) - if err != nil { - t.Fatal(err) - } - if got != "varchar" { - t.Errorf("bare varchar: got %q", got) - } -} - -// --- quoteIdent (INJECTION GUARD) ------------------------------------------ - -func TestQuoteIdent_Postgres(t *testing.T) { - got, err := quoteIdent("postgres", "users") - if err != nil { - t.Fatal(err) - } - if got != `"users"` { - t.Errorf("postgres quoteIdent: got %q want %q", got, `"users"`) - } -} - -func TestQuoteIdent_PostgresEscapesQuote(t *testing.T) { - got, err := quoteIdent("postgres", `we"ird`) - if err != nil { - t.Fatal(err) - } - if got != `"we""ird"` { - t.Errorf("postgres quote-doubling: got %q want %q", got, `"we""ird"`) - } -} - -func TestQuoteIdent_MySQL(t *testing.T) { - got, err := quoteIdent("mysql", "users") - if err != nil { - t.Fatal(err) - } - if got != "`users`" { - t.Errorf("mysql quoteIdent: got %q want %q", got, "`users`") - } -} - -func TestQuoteIdent_MySQLEscapesBacktick(t *testing.T) { - got, err := quoteIdent("mysql", "we`ird") - if err != nil { - t.Fatal(err) - } - if got != "`we``ird`" { - t.Errorf("mysql backtick-doubling: got %q want %q", got, "`we``ird`") - } -} - -func TestQuoteIdent_UnknownEngine(t *testing.T) { - if _, err := quoteIdent("oracle", "users"); err == nil { - t.Fatal("quoteIdent with unknown engine: want error, got nil") - } -} - -// TestQuoteIdent_InjectionIsNeutralized is the core security assertion: a -// malicious identifier is wrapped+escaped, never passed through raw, so the -// embedded statement terminator and quote cannot break out of the identifier. -func TestQuoteIdent_InjectionIsNeutralized(t *testing.T) { - evil := `x"; DROP TABLE y; --` - - pg, err := quoteIdent("postgres", evil) - if err != nil { - t.Fatal(err) - } - // The inner " must be doubled, and the whole thing wrapped in quotes. - want := `"x""; DROP TABLE y; --"` - if pg != want { - t.Errorf("postgres injection: got %q want %q", pg, want) - } - // The raw (un-doubled) attacker substring must NOT appear verbatim. - if strings.Contains(pg, `x"; DROP`) { - t.Errorf("postgres injection: raw quote leaked through: %q", pg) - } - - my, err := quoteIdent("mysql", "x`; DROP TABLE y; --") - if err != nil { - t.Fatal(err) - } - wantMy := "`x``; DROP TABLE y; --`" - if my != wantMy { - t.Errorf("mysql injection: got %q want %q", my, wantMy) - } -} - -// --- placeholder ----------------------------------------------------------- - -func TestPlaceholder(t *testing.T) { - cases := []struct { - engine string - n int - want string - }{ - {"postgres", 1, "$1"}, - {"postgres", 3, "$3"}, - {"mysql", 1, "?"}, - {"mariadb", 2, "?"}, - } - for _, c := range cases { - if got := placeholder(c.engine, c.n); got != c.want { - t.Errorf("placeholder(%q, %d): got %q want %q", c.engine, c.n, got, c.want) - } - } -} - -// --- SQL builders (pure, DB-free) ------------------------------------------ - -func twoColTable() CanonicalTable { - return CanonicalTable{ - Name: "t", - Columns: []CanonicalColumn{ - {Name: "id", Type: CTInt, Nullable: false}, - {Name: "name", Type: CTText, Nullable: true}, - }, - } -} - -func TestBuildCreateTableSQL_Postgres(t *testing.T) { - got, err := buildCreateTableSQL("postgres", twoColTable()) - if err != nil { - t.Fatal(err) - } - want := `CREATE TABLE IF NOT EXISTS "t" ("id" integer NOT NULL, "name" text)` - if got != want { - t.Errorf("postgres CREATE:\n got %q\nwant %q", got, want) - } -} - -func TestBuildCreateTableSQL_MySQL(t *testing.T) { - got, err := buildCreateTableSQL("mysql", twoColTable()) - if err != nil { - t.Fatal(err) - } - want := "CREATE TABLE IF NOT EXISTS `t` (`id` INT NOT NULL, `name` TEXT)" - if got != want { - t.Errorf("mysql CREATE:\n got %q\nwant %q", got, want) - } -} - -func TestBuildCreateTableSQL_UnknownEngine(t *testing.T) { - if _, err := buildCreateTableSQL("oracle", twoColTable()); err == nil { - t.Fatal("buildCreateTableSQL unknown engine: want error, got nil") - } -} - -// TestBuildCreateTableSQL_InjectionGuard proves a hostile column name flows -// through the builder quoted+escaped, not breaking out of the DDL. -func TestBuildCreateTableSQL_InjectionGuard(t *testing.T) { - tbl := CanonicalTable{ - Name: "t", - Columns: []CanonicalColumn{{Name: `evil"col`, Type: CTInt, Nullable: true}}, - } - got, err := buildCreateTableSQL("postgres", tbl) - if err != nil { - t.Fatal(err) - } - want := `CREATE TABLE IF NOT EXISTS "t" ("evil""col" integer)` - if got != want { - t.Errorf("injection-guarded CREATE:\n got %q\nwant %q", got, want) - } - if strings.Contains(got, `evil"col" integer)`) && !strings.Contains(got, `evil""col`) { - t.Errorf("raw quote leaked into DDL: %q", got) - } -} - -func TestBuildInsertSQL_Postgres(t *testing.T) { - got, err := buildInsertSQL("postgres", "t", []string{"id", "name"}) - if err != nil { - t.Fatal(err) - } - want := `INSERT INTO "t" ("id","name") VALUES ($1,$2)` - if got != want { - t.Errorf("postgres INSERT:\n got %q\nwant %q", got, want) - } -} - -func TestBuildInsertSQL_MySQL(t *testing.T) { - got, err := buildInsertSQL("mysql", "t", []string{"id", "name"}) - if err != nil { - t.Fatal(err) - } - want := "INSERT INTO `t` (`id`,`name`) VALUES (?,?)" - if got != want { - t.Errorf("mysql INSERT:\n got %q\nwant %q", got, want) - } -} - -func TestBuildInsertSQL_EmptyColumns(t *testing.T) { - if _, err := buildInsertSQL("postgres", "t", nil); err == nil { - t.Fatal("buildInsertSQL with no columns: want error, got nil") - } -} - -func TestBuildInsertSQL_UnknownEngine(t *testing.T) { - if _, err := buildInsertSQL("oracle", "t", []string{"id"}); err == nil { - t.Fatal("buildInsertSQL unknown engine: want error, got nil") - } -} - -func TestBuildSelectSQL_Postgres(t *testing.T) { - got, err := buildSelectSQL("postgres", twoColTable()) - if err != nil { - t.Fatal(err) - } - want := `SELECT "id", "name" FROM "t"` - if got != want { - t.Errorf("postgres SELECT:\n got %q\nwant %q", got, want) - } -} - -func TestBuildSelectSQL_MySQL(t *testing.T) { - got, err := buildSelectSQL("mysql", twoColTable()) - if err != nil { - t.Fatal(err) - } - want := "SELECT `id`, `name` FROM `t`" - if got != want { - t.Errorf("mysql SELECT:\n got %q\nwant %q", got, want) - } -} - -// --- JSONL framing --------------------------------------------------------- - -func TestWriteJSONL_SchemaHeaderFirst(t *testing.T) { - var sb strings.Builder - schema := &CanonicalSchema{Tables: []CanonicalTable{twoColTable()}} - if err := writeJSONL(&sb, map[string]*CanonicalSchema{"schema": schema}); err != nil { - t.Fatal(err) - } - out := sb.String() - if !strings.HasPrefix(out, `{"schema":`) { - t.Errorf("first line must start with schema key, got %q", out) - } - if !strings.HasSuffix(out, "\n") { - t.Error("writeJSONL must terminate the line with a newline") - } -} - -func TestCanonicalRow_JSONRoundTrip(t *testing.T) { - orig := CanonicalRow{ - Table: "users", - Values: map[string]any{ - "id": float64(7), // JSON numbers decode as float64 - "name": "alice", - "ok": true, - "nil": nil, - }, - } - b, err := json.Marshal(orig) - if err != nil { - t.Fatal(err) - } - // Compact keys keep the stream small. - if !strings.Contains(string(b), `"t":"users"`) || !strings.Contains(string(b), `"v":`) { - t.Errorf("CanonicalRow uses compact keys t/v, got %s", b) - } - var back CanonicalRow - if err := json.Unmarshal(b, &back); err != nil { - t.Fatal(err) - } - if back.Table != orig.Table { - t.Errorf("table mismatch: got %q want %q", back.Table, orig.Table) - } - if len(back.Values) != len(orig.Values) { - t.Fatalf("values length mismatch: got %d want %d", len(back.Values), len(orig.Values)) - } - for k, v := range orig.Values { - if back.Values[k] != v { - t.Errorf("value %q mismatch: got %v want %v", k, back.Values[k], v) - } - } -} - -// TestSchemaHeader_RoundTrip confirms a schema header marshals and unmarshals -// through the same envelope ConsumeCanonical expects. -func TestSchemaHeader_RoundTrip(t *testing.T) { - schema := &CanonicalSchema{Tables: []CanonicalTable{ - {Name: "users", Columns: []CanonicalColumn{ - {Name: "id", Type: CTBigInt, Nullable: false}, - {Name: "email", Type: CTVarchar, Nullable: true, Precision: 320}, - }}, - }} - var sb strings.Builder - if err := writeJSONL(&sb, map[string]*CanonicalSchema{"schema": schema}); err != nil { - t.Fatal(err) - } - var header struct { - Schema *CanonicalSchema `json:"schema"` - } - if err := json.Unmarshal([]byte(strings.TrimSpace(sb.String())), &header); err != nil { - t.Fatal(err) - } - if header.Schema == nil || len(header.Schema.Tables) != 1 { - t.Fatalf("schema header did not round-trip: %+v", header.Schema) - } - got := header.Schema.Tables[0] - if got.Name != "users" || len(got.Columns) != 2 || got.Columns[1].Precision != 320 { - t.Errorf("schema fields lost in round-trip: %+v", got) - } -} - -// --- normalizeScanned ------------------------------------------------------ - -// TestNormalizeScanned_BytesBecomeString proves a []byte value scanned from -// database/sql is converted to a string so it marshals as a JSON string and -// round-trips intact, rather than base64-encoding (which would corrupt the -// value before ConsumeCanonical re-inserts it). -func TestNormalizeScanned_BytesBecomeString(t *testing.T) { - in := []byte("hello-world") - got := normalizeScanned(in) - s, ok := got.(string) - if !ok { - t.Fatalf("normalizeScanned([]byte) returned %T; want string", got) - } - if s != "hello-world" { - t.Fatalf("normalizeScanned = %q; want %q", s, "hello-world") - } - - // Round-trip through the row marshal: a []byte value must come back as the - // original text, NOT base64. - row := CanonicalRow{Table: "t", Values: map[string]any{"c": normalizeScanned([]byte("café"))}} - b, err := json.Marshal(row) - if err != nil { - t.Fatal(err) - } - if strings.Contains(string(b), "Y2Fmw") { // base64 prefix of "café" would leak in - t.Fatalf("[]byte was base64-encoded, not stringified: %s", b) - } - var back CanonicalRow - if err := json.Unmarshal(b, &back); err != nil { - t.Fatal(err) - } - if back.Values["c"] != "café" { - t.Fatalf("round-trip = %v; want %q", back.Values["c"], "café") - } -} - -// TestNormalizeScanned_NonBytesUnchanged confirms non-[]byte values pass -// through untouched. -func TestNormalizeScanned_NonBytesUnchanged(t *testing.T) { - if got := normalizeScanned(int64(42)); got != int64(42) { - t.Fatalf("normalizeScanned(int64) = %v; want 42", got) - } - if got := normalizeScanned(nil); got != nil { - t.Fatalf("normalizeScanned(nil) = %v; want nil", got) - } -} +// Tests for EmitCanonical and ConsumeCanonical (db-touching orchestration) will +// live here. Pure-function tests (types, builders, quote/placeholder) are in +// internal/canonical/canonical_test.go. diff --git a/internal/canonical/canonical.go b/internal/canonical/canonical.go new file mode 100644 index 0000000..08998e4 --- /dev/null +++ b/internal/canonical/canonical.go @@ -0,0 +1,265 @@ +package canonical + +import ( + "encoding/json" + "fmt" + "io" + "strconv" + "strings" +) + +// CanonicalType is the engine-independent column type vocabulary. A source +// engine's native types are normalized into these; a target engine maps them +// back to its own dialect via MapToNative. This is the pivot that lets data +// move between Postgres, MySQL, and MariaDB. +type CanonicalType string + +const ( + CTInt CanonicalType = "int" + CTBigInt CanonicalType = "bigint" + CTText CanonicalType = "text" + CTVarchar CanonicalType = "varchar" + CTBoolean CanonicalType = "boolean" + CTNumeric CanonicalType = "numeric" + CTUUID CanonicalType = "uuid" + CTTimestampTZ CanonicalType = "timestamptz" + CTJSON CanonicalType = "json" +) + +// CanonicalColumn describes one column in canonical terms. +// +// Precision/Scale are only meaningful for CTVarchar (Precision = length) and +// CTNumeric (Precision = total digits, Scale = fractional digits). When both +// are zero the type is emitted bare (e.g. VARCHAR, DECIMAL) — acceptable for +// v1; engines apply their own defaults. +type CanonicalColumn struct { + Name string `json:"name"` + Type CanonicalType `json:"type"` + Nullable bool `json:"nullable"` + Precision int `json:"precision,omitempty"` + Scale int `json:"scale,omitempty"` + PrimaryKey bool `json:"primary_key,omitempty"` +} + +// CanonicalTable is a table name plus its ordered columns. +type CanonicalTable struct { + Name string `json:"name"` + Columns []CanonicalColumn `json:"columns"` +} + +// CanonicalSchema is the full set of tables to transfer. +type CanonicalSchema struct { + Tables []CanonicalTable `json:"tables"` +} + +// CanonicalRow is one row of data, keyed by column name. The compact JSON keys +// keep the JSONL stream small across many rows. +type CanonicalRow struct { + Table string `json:"t"` + Values map[string]any `json:"v"` +} + +// ChangeOp is the kind of row change carried by a CanonicalChange. +type ChangeOp string + +const ( + OpInsert ChangeOp = "insert" + OpUpdate ChangeOp = "update" + OpDelete ChangeOp = "delete" +) + +// CanonicalChange is one engine-neutral row change. Key holds the primary-key +// column values (used to target UPDATE/DELETE; also populated for INSERT). +// Values holds the full post-image row for INSERT/UPDATE; it is empty for DELETE. +type CanonicalChange struct { + Op ChangeOp `json:"op"` + Table string `json:"table"` + Key map[string]any `json:"key,omitempty"` + Values map[string]any `json:"values,omitempty"` +} + +// Position is an engine-neutral stream cursor: a Postgres LSN, or a MySQL/MariaDB +// binlog file+offset. Serialized into the dump Envelope and the CDC state file. +type Position struct { + LSN string `json:"lsn,omitempty"` + BinlogFile string `json:"binlog_file,omitempty"` + BinlogPos uint64 `json:"binlog_pos,omitempty"` +} + +// MapToNative renders a canonical column as a native column type for the given +// target engine. Returns an error for an unknown engine or an unmappable type. +// +// VARCHAR/NUMERIC with Precision>0 carry their precision (and Scale for +// NUMERIC) into the rendered type; otherwise the type is emitted bare. +func MapToNative(engine string, col CanonicalColumn) (string, error) { + var m map[CanonicalType]string + switch engine { + case "postgres": + m = map[CanonicalType]string{ + CTInt: "integer", + CTBigInt: "bigint", + CTText: "text", + CTVarchar: "varchar", + CTBoolean: "boolean", + CTNumeric: "numeric", + CTUUID: "uuid", + CTTimestampTZ: "timestamptz", + CTJSON: "jsonb", + } + case "mysql", "mariadb": + m = map[CanonicalType]string{ + CTInt: "INT", + CTBigInt: "BIGINT", + CTText: "TEXT", + CTVarchar: "VARCHAR", + CTBoolean: "TINYINT(1)", + CTNumeric: "DECIMAL", + CTUUID: "CHAR(36)", + CTTimestampTZ: "TIMESTAMP", + CTJSON: "JSON", + } + default: + return "", fmt.Errorf("cross-engine: unknown engine %q", engine) + } + + native, ok := m[col.Type] + if !ok { + return "", fmt.Errorf("cross-engine: engine %q has no mapping for canonical type %q", engine, col.Type) + } + + // Decorate variable-length / fixed-precision types when precision is known. + switch col.Type { + case CTVarchar: + if col.Precision > 0 { + native = fmt.Sprintf("%s(%d)", native, col.Precision) + } + case CTNumeric: + if col.Precision > 0 { + native = fmt.Sprintf("%s(%d,%d)", native, col.Precision, col.Scale) + } + } + return native, nil +} + +// QuoteIdent quotes a SQL identifier for the given engine, escaping any quote +// characters inside it. Identifiers (table/column names) CANNOT be passed as +// bound parameters, so quoting+escaping per engine is the mitigation against a +// name that contains a space, keyword, quote, or `;` — whether malicious or +// merely awkward. Every identifier that reaches generated SQL must pass through +// here. +func QuoteIdent(engine, ident string) (string, error) { + switch engine { + case "postgres": + return `"` + strings.ReplaceAll(ident, `"`, `""`) + `"`, nil + case "mysql", "mariadb": + return "`" + strings.ReplaceAll(ident, "`", "``") + "`", nil + default: + return "", fmt.Errorf("cross-engine: unknown engine %q", engine) + } +} + +// Placeholder returns the bind-parameter placeholder for position n (1-based) +// in the given engine's dialect: $n for Postgres, ? for MySQL/MariaDB. +func Placeholder(engine string, n int) string { + switch engine { + case "postgres": + return "$" + strconv.Itoa(n) + default: + return "?" + } +} + +// BuildSelectSQL builds `SELECT FROM ` with every +// identifier quoted for the engine. Pure function — unit-testable without a DB. +func BuildSelectSQL(engine string, t CanonicalTable) (string, error) { + qt, err := QuoteIdent(engine, t.Name) + if err != nil { + return "", err + } + cols := make([]string, len(t.Columns)) + for i, c := range t.Columns { + qc, err := QuoteIdent(engine, c.Name) + if err != nil { + return "", err + } + cols[i] = qc + } + return fmt.Sprintf("SELECT %s FROM %s", strings.Join(cols, ", "), qt), nil +} + +// BuildCreateTableSQL renders a CREATE TABLE IF NOT EXISTS statement with every +// identifier quoted for the engine and each column mapped via MapToNative. Pure +// function — unit-testable without a DB. +func BuildCreateTableSQL(engine string, t CanonicalTable) (string, error) { + qt, err := QuoteIdent(engine, t.Name) + if err != nil { + return "", err + } + defs := make([]string, len(t.Columns)) + for i, c := range t.Columns { + qc, err := QuoteIdent(engine, c.Name) + if err != nil { + return "", err + } + native, err := MapToNative(engine, c) + if err != nil { + return "", err + } + def := qc + " " + native + if !c.Nullable { + def += " NOT NULL" + } + defs[i] = def + } + return fmt.Sprintf("CREATE TABLE IF NOT EXISTS %s (%s)", qt, strings.Join(defs, ", ")), nil +} + +// BuildInsertSQL renders an INSERT with quoted identifiers and per-engine +// placeholders. Pure function — unit-testable without a DB. Returns an error +// for an empty column set (an INSERT with no columns is malformed). +func BuildInsertSQL(engine, table string, cols []string) (string, error) { + if len(cols) == 0 { + return "", fmt.Errorf("cross-engine: insert into %s: no columns", table) + } + qt, err := QuoteIdent(engine, table) + if err != nil { + return "", err + } + qcols := make([]string, len(cols)) + phs := make([]string, len(cols)) + for i, c := range cols { + qc, err := QuoteIdent(engine, c) + if err != nil { + return "", err + } + qcols[i] = qc + phs[i] = Placeholder(engine, i+1) + } + return fmt.Sprintf("INSERT INTO %s (%s) VALUES (%s)", + qt, strings.Join(qcols, ","), strings.Join(phs, ",")), nil +} + +// WriteJSONL marshals v and writes it followed by a newline. +func WriteJSONL(w io.Writer, v any) error { + b, err := json.Marshal(v) + if err != nil { + return err + } + if _, err := w.Write(b); err != nil { + return err + } + _, err = w.Write([]byte("\n")) + return err +} + +// NormalizeScanned converts a value scanned from database/sql into a form that +// round-trips correctly through JSON. database/sql commonly yields []byte for +// text/numeric/json columns; marshaling []byte to JSON produces base64, which +// corrupts the value when ConsumeCanonical re-inserts it. Converting to string +// makes it marshal as a JSON string instead, preserving the original bytes. +func NormalizeScanned(v any) any { + if b, ok := v.([]byte); ok { + return string(b) + } + return v +} diff --git a/internal/canonical/canonical_test.go b/internal/canonical/canonical_test.go new file mode 100644 index 0000000..c35b368 --- /dev/null +++ b/internal/canonical/canonical_test.go @@ -0,0 +1,407 @@ +package canonical + +import ( + "encoding/json" + "strings" + "testing" +) + +// --- MapToNative ----------------------------------------------------------- + +func TestMapToNative_CoreTypes(t *testing.T) { + coreTypes := []CanonicalType{ + CTInt, CTBigInt, CTText, CTBoolean, CTUUID, + CTVarchar, CTNumeric, CTTimestampTZ, CTJSON, + } + for _, engine := range []string{"postgres", "mysql", "mariadb"} { + for _, ct := range coreTypes { + native, err := MapToNative(engine, CanonicalColumn{Name: "c", Type: ct}) + if err != nil { + t.Errorf("MapToNative(%q, %q): unexpected error: %v", engine, ct, err) + } + if native == "" { + t.Errorf("MapToNative(%q, %q): empty native type", engine, ct) + } + } + } +} + +func TestMapToNative_UnknownEngine(t *testing.T) { + if _, err := MapToNative("oracle", CanonicalColumn{Type: CTInt}); err == nil { + t.Fatal("MapToNative with unknown engine: want error, got nil") + } +} + +func TestMapToNative_UnknownType(t *testing.T) { + if _, err := MapToNative("postgres", CanonicalColumn{Type: CanonicalType("geometry")}); err == nil { + t.Fatal("MapToNative with unknown type: want error, got nil") + } +} + +func TestMapToNative_PrecisionDecoration(t *testing.T) { + // VARCHAR with length. + got, err := MapToNative("postgres", CanonicalColumn{Type: CTVarchar, Precision: 255}) + if err != nil { + t.Fatal(err) + } + if got != "varchar(255)" { + t.Errorf("varchar(255): got %q", got) + } + // NUMERIC with precision+scale (MySQL DECIMAL). + got, err = MapToNative("mysql", CanonicalColumn{Type: CTNumeric, Precision: 10, Scale: 2}) + if err != nil { + t.Fatal(err) + } + if got != "DECIMAL(10,2)" { + t.Errorf("DECIMAL(10,2): got %q", got) + } + // Bare when precision is zero (documented v1 behavior). + got, err = MapToNative("postgres", CanonicalColumn{Type: CTVarchar}) + if err != nil { + t.Fatal(err) + } + if got != "varchar" { + t.Errorf("bare varchar: got %q", got) + } +} + +// --- QuoteIdent (INJECTION GUARD) ------------------------------------------ + +func TestQuoteIdent_Postgres(t *testing.T) { + got, err := QuoteIdent("postgres", "users") + if err != nil { + t.Fatal(err) + } + if got != `"users"` { + t.Errorf("postgres QuoteIdent: got %q want %q", got, `"users"`) + } +} + +func TestQuoteIdent_PostgresEscapesQuote(t *testing.T) { + got, err := QuoteIdent("postgres", `we"ird`) + if err != nil { + t.Fatal(err) + } + if got != `"we""ird"` { + t.Errorf("postgres quote-doubling: got %q want %q", got, `"we""ird"`) + } +} + +func TestQuoteIdent_MySQL(t *testing.T) { + got, err := QuoteIdent("mysql", "users") + if err != nil { + t.Fatal(err) + } + if got != "`users`" { + t.Errorf("mysql QuoteIdent: got %q want %q", got, "`users`") + } +} + +func TestQuoteIdent_MySQLEscapesBacktick(t *testing.T) { + got, err := QuoteIdent("mysql", "we`ird") + if err != nil { + t.Fatal(err) + } + if got != "`we``ird`" { + t.Errorf("mysql backtick-doubling: got %q want %q", got, "`we``ird`") + } +} + +func TestQuoteIdent_UnknownEngine(t *testing.T) { + if _, err := QuoteIdent("oracle", "users"); err == nil { + t.Fatal("QuoteIdent with unknown engine: want error, got nil") + } +} + +// TestQuoteIdent_InjectionIsNeutralized is the core security assertion: a +// malicious identifier is wrapped+escaped, never passed through raw, so the +// embedded statement terminator and quote cannot break out of the identifier. +func TestQuoteIdent_InjectionIsNeutralized(t *testing.T) { + evil := `x"; DROP TABLE y; --` + + pg, err := QuoteIdent("postgres", evil) + if err != nil { + t.Fatal(err) + } + // The inner " must be doubled, and the whole thing wrapped in quotes. + want := `"x""; DROP TABLE y; --"` + if pg != want { + t.Errorf("postgres injection: got %q want %q", pg, want) + } + // The raw (un-doubled) attacker substring must NOT appear verbatim. + if strings.Contains(pg, `x"; DROP`) { + t.Errorf("postgres injection: raw quote leaked through: %q", pg) + } + + my, err := QuoteIdent("mysql", "x`; DROP TABLE y; --") + if err != nil { + t.Fatal(err) + } + wantMy := "`x``; DROP TABLE y; --`" + if my != wantMy { + t.Errorf("mysql injection: got %q want %q", my, wantMy) + } +} + +// --- Placeholder ----------------------------------------------------------- + +func TestPlaceholder(t *testing.T) { + cases := []struct { + engine string + n int + want string + }{ + {"postgres", 1, "$1"}, + {"postgres", 3, "$3"}, + {"mysql", 1, "?"}, + {"mariadb", 2, "?"}, + } + for _, c := range cases { + if got := Placeholder(c.engine, c.n); got != c.want { + t.Errorf("Placeholder(%q, %d): got %q want %q", c.engine, c.n, got, c.want) + } + } +} + +// --- SQL builders (pure, DB-free) ------------------------------------------ + +func twoColTable() CanonicalTable { + return CanonicalTable{ + Name: "t", + Columns: []CanonicalColumn{ + {Name: "id", Type: CTInt, Nullable: false}, + {Name: "name", Type: CTText, Nullable: true}, + }, + } +} + +func TestBuildCreateTableSQL_Postgres(t *testing.T) { + got, err := BuildCreateTableSQL("postgres", twoColTable()) + if err != nil { + t.Fatal(err) + } + want := `CREATE TABLE IF NOT EXISTS "t" ("id" integer NOT NULL, "name" text)` + if got != want { + t.Errorf("postgres CREATE:\n got %q\nwant %q", got, want) + } +} + +func TestBuildCreateTableSQL_MySQL(t *testing.T) { + got, err := BuildCreateTableSQL("mysql", twoColTable()) + if err != nil { + t.Fatal(err) + } + want := "CREATE TABLE IF NOT EXISTS `t` (`id` INT NOT NULL, `name` TEXT)" + if got != want { + t.Errorf("mysql CREATE:\n got %q\nwant %q", got, want) + } +} + +func TestBuildCreateTableSQL_UnknownEngine(t *testing.T) { + if _, err := BuildCreateTableSQL("oracle", twoColTable()); err == nil { + t.Fatal("BuildCreateTableSQL unknown engine: want error, got nil") + } +} + +// TestBuildCreateTableSQL_InjectionGuard proves a hostile column name flows +// through the builder quoted+escaped, not breaking out of the DDL. +func TestBuildCreateTableSQL_InjectionGuard(t *testing.T) { + tbl := CanonicalTable{ + Name: "t", + Columns: []CanonicalColumn{{Name: `evil"col`, Type: CTInt, Nullable: true}}, + } + got, err := BuildCreateTableSQL("postgres", tbl) + if err != nil { + t.Fatal(err) + } + want := `CREATE TABLE IF NOT EXISTS "t" ("evil""col" integer)` + if got != want { + t.Errorf("injection-guarded CREATE:\n got %q\nwant %q", got, want) + } + if strings.Contains(got, `evil"col" integer)`) && !strings.Contains(got, `evil""col`) { + t.Errorf("raw quote leaked into DDL: %q", got) + } +} + +func TestBuildInsertSQL_Postgres(t *testing.T) { + got, err := BuildInsertSQL("postgres", "t", []string{"id", "name"}) + if err != nil { + t.Fatal(err) + } + want := `INSERT INTO "t" ("id","name") VALUES ($1,$2)` + if got != want { + t.Errorf("postgres INSERT:\n got %q\nwant %q", got, want) + } +} + +func TestBuildInsertSQL_MySQL(t *testing.T) { + got, err := BuildInsertSQL("mysql", "t", []string{"id", "name"}) + if err != nil { + t.Fatal(err) + } + want := "INSERT INTO `t` (`id`,`name`) VALUES (?,?)" + if got != want { + t.Errorf("mysql INSERT:\n got %q\nwant %q", got, want) + } +} + +func TestBuildInsertSQL_EmptyColumns(t *testing.T) { + if _, err := BuildInsertSQL("postgres", "t", nil); err == nil { + t.Fatal("BuildInsertSQL with no columns: want error, got nil") + } +} + +func TestBuildInsertSQL_UnknownEngine(t *testing.T) { + if _, err := BuildInsertSQL("oracle", "t", []string{"id"}); err == nil { + t.Fatal("BuildInsertSQL unknown engine: want error, got nil") + } +} + +func TestBuildSelectSQL_Postgres(t *testing.T) { + got, err := BuildSelectSQL("postgres", twoColTable()) + if err != nil { + t.Fatal(err) + } + want := `SELECT "id", "name" FROM "t"` + if got != want { + t.Errorf("postgres SELECT:\n got %q\nwant %q", got, want) + } +} + +func TestBuildSelectSQL_MySQL(t *testing.T) { + got, err := BuildSelectSQL("mysql", twoColTable()) + if err != nil { + t.Fatal(err) + } + want := "SELECT `id`, `name` FROM `t`" + if got != want { + t.Errorf("mysql SELECT:\n got %q\nwant %q", got, want) + } +} + +// --- WriteJSONL framing ---------------------------------------------------- + +func TestWriteJSONL_SchemaHeaderFirst(t *testing.T) { + var sb strings.Builder + schema := &CanonicalSchema{Tables: []CanonicalTable{twoColTable()}} + if err := WriteJSONL(&sb, map[string]*CanonicalSchema{"schema": schema}); err != nil { + t.Fatal(err) + } + out := sb.String() + if !strings.HasPrefix(out, `{"schema":`) { + t.Errorf("first line must start with schema key, got %q", out) + } + if !strings.HasSuffix(out, "\n") { + t.Error("WriteJSONL must terminate the line with a newline") + } +} + +func TestCanonicalRow_JSONRoundTrip(t *testing.T) { + orig := CanonicalRow{ + Table: "users", + Values: map[string]any{ + "id": float64(7), // JSON numbers decode as float64 + "name": "alice", + "ok": true, + "nil": nil, + }, + } + b, err := json.Marshal(orig) + if err != nil { + t.Fatal(err) + } + // Compact keys keep the stream small. + if !strings.Contains(string(b), `"t":"users"`) || !strings.Contains(string(b), `"v":`) { + t.Errorf("CanonicalRow uses compact keys t/v, got %s", b) + } + var back CanonicalRow + if err := json.Unmarshal(b, &back); err != nil { + t.Fatal(err) + } + if back.Table != orig.Table { + t.Errorf("table mismatch: got %q want %q", back.Table, orig.Table) + } + if len(back.Values) != len(orig.Values) { + t.Fatalf("values length mismatch: got %d want %d", len(back.Values), len(orig.Values)) + } + for k, v := range orig.Values { + if back.Values[k] != v { + t.Errorf("value %q mismatch: got %v want %v", k, back.Values[k], v) + } + } +} + +// TestSchemaHeader_RoundTrip confirms a schema header marshals and unmarshals +// through the same envelope ConsumeCanonical expects. +func TestSchemaHeader_RoundTrip(t *testing.T) { + schema := &CanonicalSchema{Tables: []CanonicalTable{ + {Name: "users", Columns: []CanonicalColumn{ + {Name: "id", Type: CTBigInt, Nullable: false}, + {Name: "email", Type: CTVarchar, Nullable: true, Precision: 320}, + }}, + }} + var sb strings.Builder + if err := WriteJSONL(&sb, map[string]*CanonicalSchema{"schema": schema}); err != nil { + t.Fatal(err) + } + var header struct { + Schema *CanonicalSchema `json:"schema"` + } + if err := json.Unmarshal([]byte(strings.TrimSpace(sb.String())), &header); err != nil { + t.Fatal(err) + } + if header.Schema == nil || len(header.Schema.Tables) != 1 { + t.Fatalf("schema header did not round-trip: %+v", header.Schema) + } + got := header.Schema.Tables[0] + if got.Name != "users" || len(got.Columns) != 2 || got.Columns[1].Precision != 320 { + t.Errorf("schema fields lost in round-trip: %+v", got) + } +} + +// --- NormalizeScanned ------------------------------------------------------ + +// TestNormalizeScanned_BytesBecomeString proves a []byte value scanned from +// database/sql is converted to a string so it marshals as a JSON string and +// round-trips intact, rather than base64-encoding (which would corrupt the +// value before ConsumeCanonical re-inserts it). +func TestNormalizeScanned_BytesBecomeString(t *testing.T) { + in := []byte("hello-world") + got := NormalizeScanned(in) + s, ok := got.(string) + if !ok { + t.Fatalf("NormalizeScanned([]byte) returned %T; want string", got) + } + if s != "hello-world" { + t.Fatalf("NormalizeScanned = %q; want %q", s, "hello-world") + } + + // Round-trip through the row marshal: a []byte value must come back as the + // original text, NOT base64. + row := CanonicalRow{Table: "t", Values: map[string]any{"c": NormalizeScanned([]byte("café"))}} + b, err := json.Marshal(row) + if err != nil { + t.Fatal(err) + } + if strings.Contains(string(b), "Y2Fmw") { // base64 prefix of "café" would leak in + t.Fatalf("[]byte was base64-encoded, not stringified: %s", b) + } + var back CanonicalRow + if err := json.Unmarshal(b, &back); err != nil { + t.Fatal(err) + } + if back.Values["c"] != "café" { + t.Fatalf("round-trip = %v; want %q", back.Values["c"], "café") + } +} + +// TestNormalizeScanned_NonBytesUnchanged confirms non-[]byte values pass +// through untouched. +func TestNormalizeScanned_NonBytesUnchanged(t *testing.T) { + if got := NormalizeScanned(int64(42)); got != int64(42) { + t.Fatalf("NormalizeScanned(int64) = %v; want 42", got) + } + if got := NormalizeScanned(nil); got != nil { + t.Fatalf("NormalizeScanned(nil) = %v; want nil", got) + } +} From 0b6112bbfd21a283b57cb3ff7eac25049d972923 Mon Sep 17 00:00:00 2001 From: Nikhil Rajput Date: Mon, 22 Jun 2026 18:44:45 +0530 Subject: [PATCH 02/13] feat: add UPDATE/DELETE builders + ApplyChange + PK - Add BuildUpdateSQL and BuildDeleteSQL to internal/canonical. - Add PRIMARY KEY clause to BuildCreateTableSQL when PK columns exist. - Add changeColumns and ApplyChange to internal/app for CDC/restore. - Add CanonicalChange JSON round-trip test in canonical package. --- internal/app/canonical_consume.go | 103 +++++++++++++++++++++++ internal/app/canonical_test.go | 44 ++++++++++ internal/canonical/canonical.go | 56 +++++++++++++ internal/canonical/canonical_test.go | 120 +++++++++++++++++++++++++++ 4 files changed, 323 insertions(+) diff --git a/internal/app/canonical_consume.go b/internal/app/canonical_consume.go index e417363..f0ef929 100644 --- a/internal/app/canonical_consume.go +++ b/internal/app/canonical_consume.go @@ -7,8 +7,10 @@ import ( "errors" "fmt" "io" + "sort" "github.com/nixrajput/siphon/internal/canonical" + "github.com/nixrajput/siphon/internal/errs" ) // ConsumeCanonical reads a stream produced by EmitCanonical and replays it into @@ -89,3 +91,104 @@ func insertRow(ctx context.Context, db *sql.DB, engine string, row canonical.Can } return nil } + +// changeColumns returns the SET columns (Values minus Key, sorted) and the Key +// columns (sorted), giving deterministic statement shape and argument order. +func changeColumns(ch canonical.CanonicalChange) (setCols, keyCols []string) { + for k := range ch.Key { + keyCols = append(keyCols, k) + } + sort.Strings(keyCols) + keySet := make(map[string]bool, len(keyCols)) + for _, k := range keyCols { + keySet[k] = true + } + for c := range ch.Values { + if !keySet[c] { + setCols = append(setCols, c) + } + } + sort.Strings(setCols) + return setCols, keyCols +} + +// ApplyChange applies one CanonicalChange to db using engine's SQL dialect. +// UPDATE and DELETE require a non-empty Key (the row's primary key); passing +// an empty Key for those ops is a user error. +func ApplyChange(ctx context.Context, db *sql.DB, engine string, ch canonical.CanonicalChange) error { + switch ch.Op { + case canonical.OpInsert: + cols := make([]string, 0, len(ch.Values)) + for c := range ch.Values { + cols = append(cols, c) + } + sort.Strings(cols) + stmt, err := canonical.BuildInsertSQL(engine, ch.Table, cols) + if err != nil { + return err + } + args := make([]any, len(cols)) + for i, c := range cols { + args[i] = ch.Values[c] + } + _, err = db.ExecContext(ctx, stmt, args...) + return err + + case canonical.OpUpdate: + setCols, keyCols := changeColumns(ch) + if len(keyCols) == 0 { + return &errs.Error{ + Op: "canonical.apply", + Code: errs.CodeUser, + Cause: errs.ErrIncompatibleEngine, + Hint: "UPDATE on table " + ch.Table + " has no primary key", + } + } + stmt, err := canonical.BuildUpdateSQL(engine, ch.Table, setCols, keyCols) + if err != nil { + return err + } + args := make([]any, 0, len(setCols)+len(keyCols)) + for _, c := range setCols { + args = append(args, ch.Values[c]) + } + for _, c := range keyCols { + args = append(args, ch.Key[c]) + } + _, err = db.ExecContext(ctx, stmt, args...) + return err + + case canonical.OpDelete: + keyCols := make([]string, 0, len(ch.Key)) + for k := range ch.Key { + keyCols = append(keyCols, k) + } + sort.Strings(keyCols) + if len(keyCols) == 0 { + return &errs.Error{ + Op: "canonical.apply", + Code: errs.CodeUser, + Cause: errs.ErrIncompatibleEngine, + Hint: "DELETE on table " + ch.Table + " has no primary key", + } + } + stmt, err := canonical.BuildDeleteSQL(engine, ch.Table, keyCols) + if err != nil { + return err + } + args := make([]any, len(keyCols)) + for i, c := range keyCols { + args[i] = ch.Key[c] + } + _, err = db.ExecContext(ctx, stmt, args...) + return err + + default: + return &errs.Error{ + Op: "canonical.apply", + Code: errs.CodeSystem, + Cause: errs.ErrDumpCorrupt, + Hint: "unknown change op " + string(ch.Op), + } + } +} diff --git a/internal/app/canonical_test.go b/internal/app/canonical_test.go index 298283e..5dc539f 100644 --- a/internal/app/canonical_test.go +++ b/internal/app/canonical_test.go @@ -1,5 +1,49 @@ package app +import ( + "testing" + + "github.com/nixrajput/siphon/internal/canonical" +) + // Tests for EmitCanonical and ConsumeCanonical (db-touching orchestration) will // live here. Pure-function tests (types, builders, quote/placeholder) are in // internal/canonical/canonical_test.go. + +// TestApplyChange_BuildsCorrectSQLPerOp asserts the deterministic column +// ordering that changeColumns produces for UPDATE dispatch. +func TestApplyChange_BuildsCorrectSQLPerOp(t *testing.T) { + ch := canonical.CanonicalChange{ + Op: canonical.OpUpdate, + Table: "t", + Key: map[string]any{"id": 1}, + Values: map[string]any{"id": 1, "name": "x"}, + } + set, key := changeColumns(ch) + // Key column must not appear in SET. + if len(key) != 1 || key[0] != "id" { + t.Fatalf("key cols = %v", key) + } + if len(set) != 1 || set[0] != "name" { + t.Fatalf("set cols = %v want [name]", set) + } +} + +// TestChangeColumns_MultipleKeyAndSet confirms sorted, disjoint outputs. +func TestChangeColumns_MultipleKeyAndSet(t *testing.T) { + ch := canonical.CanonicalChange{ + Op: canonical.OpUpdate, + Table: "t", + Key: map[string]any{"b": 2, "a": 1}, + Values: map[string]any{"a": 1, "b": 2, "z": "v", "m": "w"}, + } + set, key := changeColumns(ch) + // Keys must be sorted. + if len(key) != 2 || key[0] != "a" || key[1] != "b" { + t.Fatalf("key cols = %v want [a b]", key) + } + // SET cols must be sorted, disjoint from key. + if len(set) != 2 || set[0] != "m" || set[1] != "z" { + t.Fatalf("set cols = %v want [m z]", set) + } +} diff --git a/internal/canonical/canonical.go b/internal/canonical/canonical.go index 08998e4..f6f3ef2 100644 --- a/internal/canonical/canonical.go +++ b/internal/canonical/canonical.go @@ -196,6 +196,7 @@ func BuildCreateTableSQL(engine string, t CanonicalTable) (string, error) { return "", err } defs := make([]string, len(t.Columns)) + var pkCols []string for i, c := range t.Columns { qc, err := QuoteIdent(engine, c.Name) if err != nil { @@ -210,10 +211,65 @@ func BuildCreateTableSQL(engine string, t CanonicalTable) (string, error) { def += " NOT NULL" } defs[i] = def + if c.PrimaryKey { + pkCols = append(pkCols, qc) + } + } + if len(pkCols) > 0 { + defs = append(defs, "PRIMARY KEY ("+strings.Join(pkCols, ", ")+")") } return fmt.Sprintf("CREATE TABLE IF NOT EXISTS %s (%s)", qt, strings.Join(defs, ", ")), nil } +// BuildUpdateSQL builds an UPDATE statement for the given engine: SET columns +// come first (1-based placeholders), then key columns. Identifiers are always +// quoted via QuoteIdent; values are bound, not interpolated. +func BuildUpdateSQL(engine, table string, setCols, keyCols []string) (string, error) { + qt, err := QuoteIdent(engine, table) + if err != nil { + return "", err + } + n := 1 + sets := make([]string, len(setCols)) + for i, c := range setCols { + qc, err := QuoteIdent(engine, c) + if err != nil { + return "", err + } + sets[i] = qc + " = " + Placeholder(engine, n) + n++ + } + wheres := make([]string, len(keyCols)) + for i, c := range keyCols { + qc, err := QuoteIdent(engine, c) + if err != nil { + return "", err + } + wheres[i] = qc + " = " + Placeholder(engine, n) + n++ + } + return "UPDATE " + qt + " SET " + strings.Join(sets, ", ") + " WHERE " + strings.Join(wheres, " AND "), nil +} + +// BuildDeleteSQL builds a DELETE FROM statement for the given engine with all +// key columns in the WHERE clause. Identifiers are always quoted; values are +// bound (1-based for Postgres, ? for MySQL/MariaDB). +func BuildDeleteSQL(engine, table string, keyCols []string) (string, error) { + qt, err := QuoteIdent(engine, table) + if err != nil { + return "", err + } + wheres := make([]string, len(keyCols)) + for i, c := range keyCols { + qc, err := QuoteIdent(engine, c) + if err != nil { + return "", err + } + wheres[i] = qc + " = " + Placeholder(engine, i+1) + } + return "DELETE FROM " + qt + " WHERE " + strings.Join(wheres, " AND "), nil +} + // BuildInsertSQL renders an INSERT with quoted identifiers and per-engine // placeholders. Pure function — unit-testable without a DB. Returns an error // for an empty column set (an INSERT with no columns is malformed). diff --git a/internal/canonical/canonical_test.go b/internal/canonical/canonical_test.go index c35b368..cb06102 100644 --- a/internal/canonical/canonical_test.go +++ b/internal/canonical/canonical_test.go @@ -405,3 +405,123 @@ func TestNormalizeScanned_NonBytesUnchanged(t *testing.T) { t.Fatalf("NormalizeScanned(nil) = %v; want nil", got) } } + +// --- CanonicalChange JSON round-trip ---------------------------------------- + +func TestCanonicalChange_JSONRoundTrip(t *testing.T) { + orig := CanonicalChange{ + Op: OpUpdate, + Table: "orders", + Key: map[string]any{"id": float64(42)}, + Values: map[string]any{ + "id": float64(42), + "status": "shipped", + }, + } + b, err := json.Marshal(orig) + if err != nil { + t.Fatal(err) + } + var back CanonicalChange + if err := json.Unmarshal(b, &back); err != nil { + t.Fatal(err) + } + if back.Op != orig.Op { + t.Errorf("Op: got %q want %q", back.Op, orig.Op) + } + if back.Table != orig.Table { + t.Errorf("Table: got %q want %q", back.Table, orig.Table) + } + if back.Key["id"] != orig.Key["id"] { + t.Errorf("Key[id]: got %v want %v", back.Key["id"], orig.Key["id"]) + } + if back.Values["status"] != orig.Values["status"] { + t.Errorf("Values[status]: got %v want %v", back.Values["status"], orig.Values["status"]) + } +} + +// --- BuildUpdateSQL ---------------------------------------------------------- + +func TestBuildUpdateSQL_Postgres(t *testing.T) { + got, err := BuildUpdateSQL("postgres", "users", []string{"name", "email"}, []string{"id"}) + if err != nil { + t.Fatal(err) + } + want := `UPDATE "users" SET "name" = $1, "email" = $2 WHERE "id" = $3` + if got != want { + t.Fatalf("got %q want %q", got, want) + } +} + +func TestBuildUpdateSQL_MySQL_CompositeKey(t *testing.T) { + got, err := BuildUpdateSQL("mysql", "t", []string{"v"}, []string{"a", "b"}) + if err != nil { + t.Fatal(err) + } + want := "UPDATE `t` SET `v` = ? WHERE `a` = ? AND `b` = ?" + if got != want { + t.Fatalf("got %q want %q", got, want) + } +} + +func TestBuildUpdateSQL_InjectionGuard(t *testing.T) { + got, err := BuildUpdateSQL("postgres", "t", []string{`evil"col`}, []string{"id"}) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(got, `"evil""col"`) { + t.Fatalf("identifier not quoted/escaped: %s", got) + } +} + +func TestBuildUpdateSQL_UnknownEngine(t *testing.T) { + if _, err := BuildUpdateSQL("oracle", "t", []string{"v"}, []string{"id"}); err == nil { + t.Fatal("BuildUpdateSQL unknown engine: want error, got nil") + } +} + +// --- BuildDeleteSQL ---------------------------------------------------------- + +func TestBuildDeleteSQL_Postgres(t *testing.T) { + got, err := BuildDeleteSQL("postgres", "users", []string{"id"}) + if err != nil { + t.Fatal(err) + } + want := `DELETE FROM "users" WHERE "id" = $1` + if got != want { + t.Fatalf("got %q want %q", got, want) + } +} + +func TestBuildDeleteSQL_MySQL_CompositeKey(t *testing.T) { + got, err := BuildDeleteSQL("mysql", "t", []string{"a", "b"}) + if err != nil { + t.Fatal(err) + } + want := "DELETE FROM `t` WHERE `a` = ? AND `b` = ?" + if got != want { + t.Fatalf("got %q want %q", got, want) + } +} + +func TestBuildDeleteSQL_UnknownEngine(t *testing.T) { + if _, err := BuildDeleteSQL("oracle", "t", []string{"id"}); err == nil { + t.Fatal("BuildDeleteSQL unknown engine: want error, got nil") + } +} + +// --- BuildCreateTableSQL with PRIMARY KEY ------------------------------------ + +func TestBuildCreateTableSQL_WithPrimaryKey(t *testing.T) { + tbl := CanonicalTable{Name: "users", Columns: []CanonicalColumn{ + {Name: "id", Type: CTInt, PrimaryKey: true}, + {Name: "name", Type: CTText}, + }} + got, err := BuildCreateTableSQL("postgres", tbl) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(got, `PRIMARY KEY ("id")`) { + t.Fatalf("missing PK clause: %s", got) + } +} From 256dd8da094cae5dde07d8dac654e3f6fa306157 Mon Sep 17 00:00:00 2001 From: Nikhil Rajput Date: Mon, 22 Jun 2026 19:03:35 +0530 Subject: [PATCH 03/13] fix(canonical): test keyless ApplyChange guard + accurate missing-PK sentinel --- internal/app/canonical_consume.go | 4 ++-- internal/app/canonical_test.go | 20 ++++++++++++++++++++ internal/errs/errs.go | 1 + 3 files changed, 23 insertions(+), 2 deletions(-) diff --git a/internal/app/canonical_consume.go b/internal/app/canonical_consume.go index f0ef929..4ab4a99 100644 --- a/internal/app/canonical_consume.go +++ b/internal/app/canonical_consume.go @@ -140,7 +140,7 @@ func ApplyChange(ctx context.Context, db *sql.DB, engine string, ch canonical.Ca return &errs.Error{ Op: "canonical.apply", Code: errs.CodeUser, - Cause: errs.ErrIncompatibleEngine, + Cause: errs.ErrMissingPrimaryKey, Hint: "UPDATE on table " + ch.Table + " has no primary key", } } @@ -168,7 +168,7 @@ func ApplyChange(ctx context.Context, db *sql.DB, engine string, ch canonical.Ca return &errs.Error{ Op: "canonical.apply", Code: errs.CodeUser, - Cause: errs.ErrIncompatibleEngine, + Cause: errs.ErrMissingPrimaryKey, Hint: "DELETE on table " + ch.Table + " has no primary key", } } diff --git a/internal/app/canonical_test.go b/internal/app/canonical_test.go index 5dc539f..36f0fb2 100644 --- a/internal/app/canonical_test.go +++ b/internal/app/canonical_test.go @@ -1,9 +1,12 @@ package app import ( + "context" + "errors" "testing" "github.com/nixrajput/siphon/internal/canonical" + "github.com/nixrajput/siphon/internal/errs" ) // Tests for EmitCanonical and ConsumeCanonical (db-touching orchestration) will @@ -29,6 +32,23 @@ func TestApplyChange_BuildsCorrectSQLPerOp(t *testing.T) { } } +// TestApplyChange_KeylessUpdateDelete_Rejected confirms that UPDATE and DELETE +// with an empty Key return a CodeUser *errs.Error before touching the db. +// A nil *sql.DB is intentionally passed to prove the guard fires first. +func TestApplyChange_KeylessUpdateDelete_Rejected(t *testing.T) { + for _, op := range []canonical.ChangeOp{canonical.OpUpdate, canonical.OpDelete} { + ch := canonical.CanonicalChange{Op: op, Table: "t", Values: map[string]any{"v": 1}} // no Key + err := ApplyChange(context.Background(), nil, "postgres", ch) + if err == nil { + t.Fatalf("op %s: expected error for empty Key, got nil", op) + } + var e *errs.Error + if !errors.As(err, &e) || e.Code != errs.CodeUser { + t.Fatalf("op %s: want *errs.Error CodeUser, got %v", op, err) + } + } +} + // TestChangeColumns_MultipleKeyAndSet confirms sorted, disjoint outputs. func TestChangeColumns_MultipleKeyAndSet(t *testing.T) { ch := canonical.CanonicalChange{ diff --git a/internal/errs/errs.go b/internal/errs/errs.go index b3abfc9..4367819 100644 --- a/internal/errs/errs.go +++ b/internal/errs/errs.go @@ -16,6 +16,7 @@ var ( ErrChecksumMismatch = errors.New("dump checksum mismatch") ErrDumpCorrupt = errors.New("dump file corrupt or incomplete") ErrIncompatibleEngine = errors.New("source/target engine incompatible") + ErrMissingPrimaryKey = errors.New("change has no primary key to target") Err2FARequired = errors.New("two-factor authentication required") ErrCancelled = errors.New("cancelled by user") ) From 76841a0e8d6ac3c3c19ef9e1b33409ab14cc0722 Mon Sep 17 00:00:00 2001 From: Nikhil Rajput Date: Mon, 22 Jun 2026 19:56:06 +0530 Subject: [PATCH 04/13] feat(driver): SchemaInspector + CanonicalTransfer; wire sync --cross-engine MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Add ChangeColumns + ValidateChangeKey exported fns to canonical pkg. - Add driver.SchemaInspector + driver.CanonicalTransfer optional interfaces. - Implement InspectSchema, EmitCanonical, ConsumeCanonical, ApplyChange on postgres.Conn and mysqlcommon.Conn (engine string field added to Conn). - Flip CrossEngineSource/Target = true in all three driver Capabilities. - Rewrite runCrossEngineSync to use the real inspect→emit→consume pipeline. - Delete app canonical_emit/consume/test; logic now lives in driver pkgs. --- internal/app/canonical_consume.go | 194 ------------------ internal/app/canonical_emit.go | 64 ------ internal/app/canonical_test.go | 69 ------- internal/app/sync.go | 113 +++++++--- internal/canonical/canonical.go | 29 +++ internal/canonical/canonical_test.go | 64 ++++++ internal/driver/_mysqlcommon/canonical.go | 188 +++++++++++++++++ internal/driver/_mysqlcommon/conn.go | 5 +- .../driver/_mysqlcommon/inspect_schema.go | 147 +++++++++++++ .../_mysqlcommon/inspect_schema_test.go | 39 ++++ internal/driver/driver.go | 16 ++ internal/driver/mariadb/driver.go | 6 +- internal/driver/mysql/driver.go | 6 +- internal/driver/postgres/canonical.go | 188 +++++++++++++++++ internal/driver/postgres/driver.go | 4 +- internal/driver/postgres/inspect_schema.go | 140 +++++++++++++ .../driver/postgres/inspect_schema_test.go | 34 +++ 17 files changed, 946 insertions(+), 360 deletions(-) delete mode 100644 internal/app/canonical_consume.go delete mode 100644 internal/app/canonical_emit.go delete mode 100644 internal/app/canonical_test.go create mode 100644 internal/driver/_mysqlcommon/canonical.go create mode 100644 internal/driver/_mysqlcommon/inspect_schema.go create mode 100644 internal/driver/_mysqlcommon/inspect_schema_test.go create mode 100644 internal/driver/postgres/canonical.go create mode 100644 internal/driver/postgres/inspect_schema.go create mode 100644 internal/driver/postgres/inspect_schema_test.go diff --git a/internal/app/canonical_consume.go b/internal/app/canonical_consume.go deleted file mode 100644 index 4ab4a99..0000000 --- a/internal/app/canonical_consume.go +++ /dev/null @@ -1,194 +0,0 @@ -package app - -import ( - "context" - "database/sql" - "encoding/json" - "errors" - "fmt" - "io" - "sort" - - "github.com/nixrajput/siphon/internal/canonical" - "github.com/nixrajput/siphon/internal/errs" -) - -// ConsumeCanonical reads a stream produced by EmitCanonical and replays it into -// db using engine's dialect: the first JSON value is the schema header (used to -// CREATE TABLE IF NOT EXISTS for each table), every subsequent JSON value is a -// CanonicalRow that is INSERTed. Values are always bound as parameters; only -// identifiers are interpolated (quoted via canonical.QuoteIdent). -// -// A json.Decoder reads successive JSON values directly, so there is no -// token-size cap: a single large row can no longer abort the stream mid-replay -// (which a bufio.Scanner's "token too long" would have done). -func ConsumeCanonical(ctx context.Context, db *sql.DB, engine string, r io.Reader) error { - dec := json.NewDecoder(r) - - var header struct { - Schema *canonical.CanonicalSchema `json:"schema"` - } - if err := dec.Decode(&header); err != nil { - if errors.Is(err, io.EOF) { - return fmt.Errorf("cross-engine: empty stream, schema header missing") - } - return fmt.Errorf("cross-engine: decode schema header: %w", err) - } - if header.Schema == nil { - return fmt.Errorf("cross-engine: schema header missing") - } - - if err := synthCreateTables(ctx, db, engine, header.Schema); err != nil { - return err - } - - for { - var row canonical.CanonicalRow - if err := dec.Decode(&row); err != nil { - if errors.Is(err, io.EOF) { - break - } - return fmt.Errorf("cross-engine: decode row: %w", err) - } - if err := insertRow(ctx, db, engine, row); err != nil { - return err - } - } - return nil -} - -// synthCreateTables issues a CREATE TABLE IF NOT EXISTS per table. -func synthCreateTables(ctx context.Context, db *sql.DB, engine string, schema *canonical.CanonicalSchema) error { - for _, t := range schema.Tables { - stmt, err := canonical.BuildCreateTableSQL(engine, t) - if err != nil { - return err - } - if _, err := db.ExecContext(ctx, stmt); err != nil { - return fmt.Errorf("cross-engine: create table %s: %w", t.Name, err) - } - } - return nil -} - -// insertRow inserts a single CanonicalRow, binding values as parameters. -func insertRow(ctx context.Context, db *sql.DB, engine string, row canonical.CanonicalRow) error { - // Build column list and value list in the SAME pass so col[i] and val[i] - // stay aligned even though map iteration order is unspecified. - cols := make([]string, 0, len(row.Values)) - vals := make([]any, 0, len(row.Values)) - for c, v := range row.Values { - cols = append(cols, c) - vals = append(vals, v) - } - - stmt, err := canonical.BuildInsertSQL(engine, row.Table, cols) - if err != nil { - return err - } - if _, err := db.ExecContext(ctx, stmt, vals...); err != nil { - return fmt.Errorf("cross-engine: insert into %s: %w", row.Table, err) - } - return nil -} - -// changeColumns returns the SET columns (Values minus Key, sorted) and the Key -// columns (sorted), giving deterministic statement shape and argument order. -func changeColumns(ch canonical.CanonicalChange) (setCols, keyCols []string) { - for k := range ch.Key { - keyCols = append(keyCols, k) - } - sort.Strings(keyCols) - keySet := make(map[string]bool, len(keyCols)) - for _, k := range keyCols { - keySet[k] = true - } - for c := range ch.Values { - if !keySet[c] { - setCols = append(setCols, c) - } - } - sort.Strings(setCols) - return setCols, keyCols -} - -// ApplyChange applies one CanonicalChange to db using engine's SQL dialect. -// UPDATE and DELETE require a non-empty Key (the row's primary key); passing -// an empty Key for those ops is a user error. -func ApplyChange(ctx context.Context, db *sql.DB, engine string, ch canonical.CanonicalChange) error { - switch ch.Op { - case canonical.OpInsert: - cols := make([]string, 0, len(ch.Values)) - for c := range ch.Values { - cols = append(cols, c) - } - sort.Strings(cols) - stmt, err := canonical.BuildInsertSQL(engine, ch.Table, cols) - if err != nil { - return err - } - args := make([]any, len(cols)) - for i, c := range cols { - args[i] = ch.Values[c] - } - _, err = db.ExecContext(ctx, stmt, args...) - return err - - case canonical.OpUpdate: - setCols, keyCols := changeColumns(ch) - if len(keyCols) == 0 { - return &errs.Error{ - Op: "canonical.apply", - Code: errs.CodeUser, - Cause: errs.ErrMissingPrimaryKey, - Hint: "UPDATE on table " + ch.Table + " has no primary key", - } - } - stmt, err := canonical.BuildUpdateSQL(engine, ch.Table, setCols, keyCols) - if err != nil { - return err - } - args := make([]any, 0, len(setCols)+len(keyCols)) - for _, c := range setCols { - args = append(args, ch.Values[c]) - } - for _, c := range keyCols { - args = append(args, ch.Key[c]) - } - _, err = db.ExecContext(ctx, stmt, args...) - return err - - case canonical.OpDelete: - keyCols := make([]string, 0, len(ch.Key)) - for k := range ch.Key { - keyCols = append(keyCols, k) - } - sort.Strings(keyCols) - if len(keyCols) == 0 { - return &errs.Error{ - Op: "canonical.apply", - Code: errs.CodeUser, - Cause: errs.ErrMissingPrimaryKey, - Hint: "DELETE on table " + ch.Table + " has no primary key", - } - } - stmt, err := canonical.BuildDeleteSQL(engine, ch.Table, keyCols) - if err != nil { - return err - } - args := make([]any, len(keyCols)) - for i, c := range keyCols { - args[i] = ch.Key[c] - } - _, err = db.ExecContext(ctx, stmt, args...) - return err - - default: - return &errs.Error{ - Op: "canonical.apply", - Code: errs.CodeSystem, - Cause: errs.ErrDumpCorrupt, - Hint: "unknown change op " + string(ch.Op), - } - } -} diff --git a/internal/app/canonical_emit.go b/internal/app/canonical_emit.go deleted file mode 100644 index 118e74c..0000000 --- a/internal/app/canonical_emit.go +++ /dev/null @@ -1,64 +0,0 @@ -package app - -import ( - "context" - "database/sql" - "fmt" - "io" - - "github.com/nixrajput/siphon/internal/canonical" -) - -// EmitCanonical writes a table-by-table snapshot of schema as JSONL to w. -// -// Line 1 is the schema header ({"schema": {...}}); each subsequent line is one -// CanonicalRow ({"t": table, "v": {col: val}}). The engine param names the -// SOURCE engine and is used only to quote identifiers in the SELECT — values -// are never interpolated. -func EmitCanonical(ctx context.Context, db *sql.DB, engine string, schema *canonical.CanonicalSchema, w io.Writer) error { - if err := canonical.WriteJSONL(w, map[string]*canonical.CanonicalSchema{"schema": schema}); err != nil { - return err - } - for _, t := range schema.Tables { - if err := emitTable(ctx, db, engine, t, w); err != nil { - return err - } - } - return nil -} - -// emitTable streams one table's rows as CanonicalRow JSONL lines. -func emitTable(ctx context.Context, db *sql.DB, engine string, t canonical.CanonicalTable, w io.Writer) error { - query, err := canonical.BuildSelectSQL(engine, t) - if err != nil { - return err - } - - rows, err := db.QueryContext(ctx, query) - if err != nil { - return fmt.Errorf("cross-engine: select %s: %w", t.Name, err) - } - defer func() { _ = rows.Close() }() - - for rows.Next() { - vals := make([]any, len(t.Columns)) - ptrs := make([]any, len(t.Columns)) - for i := range vals { - ptrs[i] = &vals[i] - } - if err := rows.Scan(ptrs...); err != nil { - return fmt.Errorf("cross-engine: scan %s: %w", t.Name, err) - } - m := make(map[string]any, len(t.Columns)) - for i, c := range t.Columns { - m[c.Name] = canonical.NormalizeScanned(vals[i]) - } - if err := canonical.WriteJSONL(w, canonical.CanonicalRow{Table: t.Name, Values: m}); err != nil { - return err - } - } - if err := rows.Err(); err != nil { - return fmt.Errorf("cross-engine: iterate %s: %w", t.Name, err) - } - return nil -} diff --git a/internal/app/canonical_test.go b/internal/app/canonical_test.go deleted file mode 100644 index 36f0fb2..0000000 --- a/internal/app/canonical_test.go +++ /dev/null @@ -1,69 +0,0 @@ -package app - -import ( - "context" - "errors" - "testing" - - "github.com/nixrajput/siphon/internal/canonical" - "github.com/nixrajput/siphon/internal/errs" -) - -// Tests for EmitCanonical and ConsumeCanonical (db-touching orchestration) will -// live here. Pure-function tests (types, builders, quote/placeholder) are in -// internal/canonical/canonical_test.go. - -// TestApplyChange_BuildsCorrectSQLPerOp asserts the deterministic column -// ordering that changeColumns produces for UPDATE dispatch. -func TestApplyChange_BuildsCorrectSQLPerOp(t *testing.T) { - ch := canonical.CanonicalChange{ - Op: canonical.OpUpdate, - Table: "t", - Key: map[string]any{"id": 1}, - Values: map[string]any{"id": 1, "name": "x"}, - } - set, key := changeColumns(ch) - // Key column must not appear in SET. - if len(key) != 1 || key[0] != "id" { - t.Fatalf("key cols = %v", key) - } - if len(set) != 1 || set[0] != "name" { - t.Fatalf("set cols = %v want [name]", set) - } -} - -// TestApplyChange_KeylessUpdateDelete_Rejected confirms that UPDATE and DELETE -// with an empty Key return a CodeUser *errs.Error before touching the db. -// A nil *sql.DB is intentionally passed to prove the guard fires first. -func TestApplyChange_KeylessUpdateDelete_Rejected(t *testing.T) { - for _, op := range []canonical.ChangeOp{canonical.OpUpdate, canonical.OpDelete} { - ch := canonical.CanonicalChange{Op: op, Table: "t", Values: map[string]any{"v": 1}} // no Key - err := ApplyChange(context.Background(), nil, "postgres", ch) - if err == nil { - t.Fatalf("op %s: expected error for empty Key, got nil", op) - } - var e *errs.Error - if !errors.As(err, &e) || e.Code != errs.CodeUser { - t.Fatalf("op %s: want *errs.Error CodeUser, got %v", op, err) - } - } -} - -// TestChangeColumns_MultipleKeyAndSet confirms sorted, disjoint outputs. -func TestChangeColumns_MultipleKeyAndSet(t *testing.T) { - ch := canonical.CanonicalChange{ - Op: canonical.OpUpdate, - Table: "t", - Key: map[string]any{"b": 2, "a": 1}, - Values: map[string]any{"a": 1, "b": 2, "z": "v", "m": "w"}, - } - set, key := changeColumns(ch) - // Keys must be sorted. - if len(key) != 2 || key[0] != "a" || key[1] != "b" { - t.Fatalf("key cols = %v want [a b]", key) - } - // SET cols must be sorted, disjoint from key. - if len(set) != 2 || set[0] != "m" || set[1] != "z" { - t.Fatalf("set cols = %v want [m z]", set) - } -} diff --git a/internal/app/sync.go b/internal/app/sync.go index ae64e8b..731f60c 100644 --- a/internal/app/sync.go +++ b/internal/app/sync.go @@ -29,8 +29,8 @@ type SyncOpts struct { // preventing a truncated dump from being committed as if clean. // // When opt.CrossEngine is set the work routes through runCrossEngineSync, which -// is capability-gated and today returns a clear unsupported error (no driver -// declares cross-engine source/target yet). +// is capability-gated and uses driver.SchemaInspector + driver.CanonicalTransfer +// for typed cross-engine snapshot transfer. func Sync(parent context.Context, d Deps, opt SyncOpts) (<-chan jobs.Event, string, error) { if opt.Continuous { return nil, "", &errs.Error{ @@ -107,19 +107,10 @@ func Sync(parent context.Context, d Deps, opt SyncOpts) (<-chan jobs.Event, stri } // runCrossEngineSync handles heterogeneous sync (e.g. postgres→mysql) by routing -// through the canonical-schema machinery (driver EmitCanonical/ConsumeCanonical). -// -// It is HONESTLY gated: it checks CapCrossEngineSource on the source driver and -// CapCrossEngineTarget on the target. No driver declares either today (all -// return false — see each driver.go), because cross-engine translation needs a -// typed CanonicalSchema and driver.Inspect carries no column types yet. So the -// capability gate rejects every cross-engine request with ErrDriverUnsupported. -// -// This is deliberate scaffolding: the canonical emit/consume machinery (Task 8) -// exists and is unit-tested, but wiring it requires typed schema introspection -// (a future task). We do NOT fabricate a CanonicalSchema here. Once Inspect -// emits column types and a driver flips its cross-engine caps to true, this -// function gains the real emit→translate→consume pipeline. +// through the canonical-schema machinery: InspectSchema on the source builds a +// CanonicalSchema, EmitCanonical streams it as JSONL through a jobs.Stream, and +// ConsumeCanonical on the target replays the rows. Both ends must implement +// driver.SchemaInspector and driver.CanonicalTransfer. func runCrossEngineSync(parent context.Context, d Deps, opt SyncOpts) (<-chan jobs.Event, string, error) { if err := RequireCapability(d, opt.From, CapCrossEngineSource); err != nil { return nil, "", err @@ -127,13 +118,89 @@ func runCrossEngineSync(parent context.Context, d Deps, opt SyncOpts) (<-chan jo if err := RequireCapability(d, opt.To, CapCrossEngineTarget); err != nil { return nil, "", err } - // Unreachable today (no driver advertises cross-engine caps), but kept as a - // clear, honest backstop should a cap ever flip true before the pipeline is - // actually wired. - return nil, "", &errs.Error{ - Op: "sync.cross_engine", - Code: errs.CodeUser, - Cause: errs.ErrDriverUnsupported, - Hint: "cross-engine sync requires typed schema introspection, not yet available", + + src, err := d.Profiles.Resolve(opt.From) + if err != nil { + return nil, "", err + } + dst, err := d.Profiles.Resolve(opt.To) + if err != nil { + return nil, "", err + } + srcDrv, err := d.Drivers.Get(src.Driver) + if err != nil { + return nil, "", err + } + dstDrv, err := d.Drivers.Get(dst.Driver) + if err != nil { + return nil, "", err } + + return d.Runner.Run(parent, jobs.Job{ + Stage: "sync.cross-engine", + Func: func(ctx context.Context, emit func(jobs.Event)) error { + srcConn, err := srcDrv.Connect(ctx, src) + if err != nil { + return err + } + defer func() { _ = srcConn.Close() }() + + dstConn, err := dstDrv.Connect(ctx, dst) + if err != nil { + return err + } + defer func() { _ = dstConn.Close() }() + + srcInsp, ok := srcConn.(driver.SchemaInspector) + if !ok { + return &errs.Error{ + Op: "sync.cross_engine", + Code: errs.CodeUser, + Cause: errs.ErrDriverUnsupported, + Hint: src.Driver + " driver does not implement SchemaInspector", + } + } + srcXfer, ok := srcConn.(driver.CanonicalTransfer) + if !ok { + return &errs.Error{ + Op: "sync.cross_engine", + Code: errs.CodeUser, + Cause: errs.ErrDriverUnsupported, + Hint: src.Driver + " driver does not implement CanonicalTransfer", + } + } + dstXfer, ok := dstConn.(driver.CanonicalTransfer) + if !ok { + return &errs.Error{ + Op: "sync.cross_engine", + Code: errs.CodeUser, + Cause: errs.ErrDriverUnsupported, + Hint: dst.Driver + " driver does not implement CanonicalTransfer", + } + } + + schema, err := srcInsp.InspectSchema(ctx) + if err != nil { + return err + } + + stream := jobs.NewStream(64) + errCh := make(chan error, 1) + + go func() { + emitErr := srcXfer.EmitCanonical(ctx, schema, stream) + _ = stream.CloseErr(emitErr) + errCh <- emitErr + }() + + consumeErr := dstXfer.ConsumeCanonical(ctx, stream) + _ = stream.Close() + emitErr := <-errCh + + if emitErr != nil { + return emitErr + } + return consumeErr + }, + }) } diff --git a/internal/canonical/canonical.go b/internal/canonical/canonical.go index f6f3ef2..c11d1a6 100644 --- a/internal/canonical/canonical.go +++ b/internal/canonical/canonical.go @@ -4,6 +4,7 @@ import ( "encoding/json" "fmt" "io" + "sort" "strconv" "strings" ) @@ -295,6 +296,34 @@ func BuildInsertSQL(engine, table string, cols []string) (string, error) { qt, strings.Join(qcols, ","), strings.Join(phs, ",")), nil } +// ChangeColumns returns the SET columns (Values minus Key, sorted) and the Key +// columns (sorted), giving deterministic statement shape and argument order. +func ChangeColumns(ch CanonicalChange) (setCols, keyCols []string) { + for k := range ch.Key { + keyCols = append(keyCols, k) + } + sort.Strings(keyCols) + keySet := make(map[string]bool, len(keyCols)) + for _, k := range keyCols { + keySet[k] = true + } + for c := range ch.Values { + if !keySet[c] { + setCols = append(setCols, c) + } + } + sort.Strings(setCols) + return setCols, keyCols +} + +// ValidateChangeKey returns an error if ch.Op is Update or Delete and ch.Key is empty. +func ValidateChangeKey(ch CanonicalChange) error { + if (ch.Op == OpUpdate || ch.Op == OpDelete) && len(ch.Key) == 0 { + return fmt.Errorf("change on %s has no primary key", ch.Table) + } + return nil +} + // WriteJSONL marshals v and writes it followed by a newline. func WriteJSONL(w io.Writer, v any) error { b, err := json.Marshal(v) diff --git a/internal/canonical/canonical_test.go b/internal/canonical/canonical_test.go index cb06102..526af33 100644 --- a/internal/canonical/canonical_test.go +++ b/internal/canonical/canonical_test.go @@ -525,3 +525,67 @@ func TestBuildCreateTableSQL_WithPrimaryKey(t *testing.T) { t.Fatalf("missing PK clause: %s", got) } } + +// --- ChangeColumns ----------------------------------------------------------- + +func TestChangeColumns_Basic(t *testing.T) { + ch := CanonicalChange{ + Op: OpUpdate, + Table: "t", + Key: map[string]any{"id": 1}, + Values: map[string]any{"id": 1, "name": "x"}, + } + set, key := ChangeColumns(ch) + if len(key) != 1 || key[0] != "id" { + t.Fatalf("key cols = %v want [id]", key) + } + if len(set) != 1 || set[0] != "name" { + t.Fatalf("set cols = %v want [name]", set) + } +} + +func TestChangeColumns_MultipleKeyAndSet(t *testing.T) { + ch := CanonicalChange{ + Op: OpUpdate, + Table: "t", + Key: map[string]any{"b": 2, "a": 1}, + Values: map[string]any{"a": 1, "b": 2, "z": "v", "m": "w"}, + } + set, key := ChangeColumns(ch) + if len(key) != 2 || key[0] != "a" || key[1] != "b" { + t.Fatalf("key cols = %v want [a b]", key) + } + if len(set) != 2 || set[0] != "m" || set[1] != "z" { + t.Fatalf("set cols = %v want [m z]", set) + } +} + +// --- ValidateChangeKey ------------------------------------------------------- + +func TestValidateChangeKey_InsertNoKey_OK(t *testing.T) { + ch := CanonicalChange{Op: OpInsert, Table: "t"} + if err := ValidateChangeKey(ch); err != nil { + t.Fatalf("INSERT with no key: got %v want nil", err) + } +} + +func TestValidateChangeKey_UpdateNoKey_Error(t *testing.T) { + ch := CanonicalChange{Op: OpUpdate, Table: "orders"} + if err := ValidateChangeKey(ch); err == nil { + t.Fatal("UPDATE with no key: want error, got nil") + } +} + +func TestValidateChangeKey_DeleteNoKey_Error(t *testing.T) { + ch := CanonicalChange{Op: OpDelete, Table: "orders"} + if err := ValidateChangeKey(ch); err == nil { + t.Fatal("DELETE with no key: want error, got nil") + } +} + +func TestValidateChangeKey_UpdateWithKey_OK(t *testing.T) { + ch := CanonicalChange{Op: OpUpdate, Table: "t", Key: map[string]any{"id": 1}} + if err := ValidateChangeKey(ch); err != nil { + t.Fatalf("UPDATE with key: got %v want nil", err) + } +} diff --git a/internal/driver/_mysqlcommon/canonical.go b/internal/driver/_mysqlcommon/canonical.go new file mode 100644 index 0000000..f4336a3 --- /dev/null +++ b/internal/driver/_mysqlcommon/canonical.go @@ -0,0 +1,188 @@ +package mysqlcommon + +import ( + "context" + "database/sql" + "encoding/json" + "errors" + "fmt" + "io" + "sort" + + "github.com/nixrajput/siphon/internal/canonical" + "github.com/nixrajput/siphon/internal/errs" +) + +// EmitCanonical writes a table-by-table snapshot of schema as JSONL to w. +func (c *Conn) EmitCanonical(ctx context.Context, schema *canonical.CanonicalSchema, w io.Writer) error { + if err := canonical.WriteJSONL(w, map[string]*canonical.CanonicalSchema{"schema": schema}); err != nil { + return err + } + for _, t := range schema.Tables { + if err := myEmitTable(ctx, c.db, c.engine, t, w); err != nil { + return err + } + } + return nil +} + +func myEmitTable(ctx context.Context, db *sql.DB, engine string, t canonical.CanonicalTable, w io.Writer) error { + query, err := canonical.BuildSelectSQL(engine, t) + if err != nil { + return err + } + rows, err := db.QueryContext(ctx, query) + if err != nil { + return fmt.Errorf("cross-engine: select %s: %w", t.Name, err) + } + defer func() { _ = rows.Close() }() + + for rows.Next() { + vals := make([]any, len(t.Columns)) + ptrs := make([]any, len(t.Columns)) + for i := range vals { + ptrs[i] = &vals[i] + } + if err := rows.Scan(ptrs...); err != nil { + return fmt.Errorf("cross-engine: scan %s: %w", t.Name, err) + } + m := make(map[string]any, len(t.Columns)) + for i, col := range t.Columns { + m[col.Name] = canonical.NormalizeScanned(vals[i]) + } + if err := canonical.WriteJSONL(w, canonical.CanonicalRow{Table: t.Name, Values: m}); err != nil { + return err + } + } + return rows.Err() +} + +// ConsumeCanonical reads a stream produced by EmitCanonical and replays it into the database. +func (c *Conn) ConsumeCanonical(ctx context.Context, r io.Reader) error { + dec := json.NewDecoder(r) + + var header struct { + Schema *canonical.CanonicalSchema `json:"schema"` + } + if err := dec.Decode(&header); err != nil { + if errors.Is(err, io.EOF) { + return fmt.Errorf("cross-engine: empty stream, schema header missing") + } + return fmt.Errorf("cross-engine: decode schema header: %w", err) + } + if header.Schema == nil { + return fmt.Errorf("cross-engine: schema header missing") + } + + for _, t := range header.Schema.Tables { + stmt, err := canonical.BuildCreateTableSQL(c.engine, t) + if err != nil { + return err + } + if _, err := c.db.ExecContext(ctx, stmt); err != nil { + return fmt.Errorf("cross-engine: create table %s: %w", t.Name, err) + } + } + + for { + var row canonical.CanonicalRow + if err := dec.Decode(&row); err != nil { + if errors.Is(err, io.EOF) { + break + } + return fmt.Errorf("cross-engine: decode row: %w", err) + } + if err := myInsertRow(ctx, c.db, c.engine, row); err != nil { + return err + } + } + return nil +} + +func myInsertRow(ctx context.Context, db *sql.DB, engine string, row canonical.CanonicalRow) error { + cols := make([]string, 0, len(row.Values)) + for col := range row.Values { + cols = append(cols, col) + } + sort.Strings(cols) + sortedVals := make([]any, len(cols)) + for i, col := range cols { + sortedVals[i] = row.Values[col] + } + stmt, err := canonical.BuildInsertSQL(engine, row.Table, cols) + if err != nil { + return err + } + if _, err := db.ExecContext(ctx, stmt, sortedVals...); err != nil { + return fmt.Errorf("cross-engine: insert into %s: %w", row.Table, err) + } + return nil +} + +// ApplyChange applies one CanonicalChange to the database. +func (c *Conn) ApplyChange(ctx context.Context, ch canonical.CanonicalChange) error { + if err := canonical.ValidateChangeKey(ch); err != nil { + return &errs.Error{ + Op: c.engine + ".apply", + Code: errs.CodeUser, + Cause: errs.ErrMissingPrimaryKey, + Hint: err.Error(), + } + } + + switch ch.Op { + case canonical.OpInsert: + cols := make([]string, 0, len(ch.Values)) + for col := range ch.Values { + cols = append(cols, col) + } + sort.Strings(cols) + stmt, err := canonical.BuildInsertSQL(c.engine, ch.Table, cols) + if err != nil { + return err + } + args := make([]any, len(cols)) + for i, col := range cols { + args[i] = ch.Values[col] + } + _, err = c.db.ExecContext(ctx, stmt, args...) + return err + + case canonical.OpUpdate: + setCols, keyCols := canonical.ChangeColumns(ch) + stmt, err := canonical.BuildUpdateSQL(c.engine, ch.Table, setCols, keyCols) + if err != nil { + return err + } + args := make([]any, 0, len(setCols)+len(keyCols)) + for _, col := range setCols { + args = append(args, ch.Values[col]) + } + for _, col := range keyCols { + args = append(args, ch.Key[col]) + } + _, err = c.db.ExecContext(ctx, stmt, args...) + return err + + case canonical.OpDelete: + _, keyCols := canonical.ChangeColumns(ch) + stmt, err := canonical.BuildDeleteSQL(c.engine, ch.Table, keyCols) + if err != nil { + return err + } + args := make([]any, len(keyCols)) + for i, col := range keyCols { + args[i] = ch.Key[col] + } + _, err = c.db.ExecContext(ctx, stmt, args...) + return err + + default: + return &errs.Error{ + Op: c.engine + ".apply", + Code: errs.CodeSystem, + Cause: errs.ErrDumpCorrupt, + Hint: "unknown change op " + string(ch.Op), + } + } +} diff --git a/internal/driver/_mysqlcommon/conn.go b/internal/driver/_mysqlcommon/conn.go index a291521..aa564e4 100644 --- a/internal/driver/_mysqlcommon/conn.go +++ b/internal/driver/_mysqlcommon/conn.go @@ -34,6 +34,7 @@ type Conn struct { dumpBinary string // "mysqldump" or "mariadb-dump" clientBinary string // "mysql" or "mariadb" binlogBinary string // "mysqlbinlog" or "mariadb-binlog" + engine string // "mysql" or "mariadb" } var _ driver.Conn = (*Conn)(nil) @@ -43,7 +44,7 @@ var _ driver.Conn = (*Conn)(nil) // Postgres driver (spec §4.3). connOp is the error-wrapping op label, e.g. // "mysql.connect" / "mariadb.connect", so errors name the right driver. The // three binary names (dump/client/binlog) are the only per-fork difference. -func NewConn(ctx context.Context, p driver.Profile, dumpBinary, clientBinary, binlogBinary, connOp string) (*Conn, error) { +func NewConn(ctx context.Context, p driver.Profile, dumpBinary, clientBinary, binlogBinary, connOp, engine string) (*Conn, error) { db, err := Open(p) if err != nil { return nil, connErr(connOp, err) @@ -52,7 +53,7 @@ func NewConn(ctx context.Context, p driver.Profile, dumpBinary, clientBinary, bi _ = db.Close() return nil, connErr(connOp, err) } - return &Conn{db: db, p: p, dumpBinary: dumpBinary, clientBinary: clientBinary, binlogBinary: binlogBinary}, nil + return &Conn{db: db, p: p, dumpBinary: dumpBinary, clientBinary: clientBinary, binlogBinary: binlogBinary, engine: engine}, nil } // connErr wraps a connection failure in the shape the harness asserts on diff --git a/internal/driver/_mysqlcommon/inspect_schema.go b/internal/driver/_mysqlcommon/inspect_schema.go new file mode 100644 index 0000000..e8186db --- /dev/null +++ b/internal/driver/_mysqlcommon/inspect_schema.go @@ -0,0 +1,147 @@ +package mysqlcommon + +import ( + "context" + "strings" + + "github.com/nixrajput/siphon/internal/canonical" +) + +// InspectSchema queries information_schema for tables and primary keys, +// returning a CanonicalSchema. +func (c *Conn) InspectSchema(ctx context.Context) (*canonical.CanonicalSchema, error) { + const colQuery = ` +SELECT TABLE_NAME, COLUMN_NAME, DATA_TYPE, COLUMN_TYPE, IS_NULLABLE, + CHARACTER_MAXIMUM_LENGTH, NUMERIC_PRECISION, NUMERIC_SCALE +FROM information_schema.COLUMNS +WHERE TABLE_SCHEMA = ? +ORDER BY TABLE_NAME, ORDINAL_POSITION +` + rows, err := c.db.QueryContext(ctx, colQuery, c.p.Database) + if err != nil { + return nil, err + } + defer func() { _ = rows.Close() }() + + tableMap := make(map[string]*canonical.CanonicalTable) + var tableOrder []string + + for rows.Next() { + var ( + tableName string + colName string + dataType string + columnType string + isNullable string + charMaxLen *int64 + numPrec *int64 + numScale *int64 + ) + if err := rows.Scan(&tableName, &colName, &dataType, &columnType, &isNullable, &charMaxLen, &numPrec, &numScale); err != nil { + return nil, err + } + if _, ok := tableMap[tableName]; !ok { + tableMap[tableName] = &canonical.CanonicalTable{Name: tableName} + tableOrder = append(tableOrder, tableName) + } + col := canonical.CanonicalColumn{ + Name: colName, + Type: mapMySQLType(dataType, columnType), + Nullable: strings.EqualFold(isNullable, "YES"), + } + if charMaxLen != nil { + col.Precision = int(*charMaxLen) + } + if numPrec != nil { + col.Precision = int(*numPrec) + } + if numScale != nil { + col.Scale = int(*numScale) + } + tableMap[tableName].Columns = append(tableMap[tableName].Columns, col) + } + if err := rows.Err(); err != nil { + return nil, err + } + + // Fetch primary keys + const pkQuery = ` +SELECT TABLE_NAME, COLUMN_NAME +FROM information_schema.KEY_COLUMN_USAGE +WHERE TABLE_SCHEMA = ? + AND CONSTRAINT_NAME = 'PRIMARY' +ORDER BY TABLE_NAME, ORDINAL_POSITION +` + pkRows, err := c.db.QueryContext(ctx, pkQuery, c.p.Database) + if err != nil { + return nil, err + } + defer func() { _ = pkRows.Close() }() + + pkSet := make(map[string]map[string]bool) + for pkRows.Next() { + var tbl, col string + if err := pkRows.Scan(&tbl, &col); err != nil { + return nil, err + } + if pkSet[tbl] == nil { + pkSet[tbl] = make(map[string]bool) + } + pkSet[tbl][col] = true + } + if err := pkRows.Err(); err != nil { + return nil, err + } + + // Mark PrimaryKey on columns + for tblName, tbl := range tableMap { + if pks, ok := pkSet[tblName]; ok { + for i := range tbl.Columns { + if pks[tbl.Columns[i].Name] { + tbl.Columns[i].PrimaryKey = true + } + } + } + } + + schema := &canonical.CanonicalSchema{} + for _, name := range tableOrder { + schema.Tables = append(schema.Tables, *tableMap[name]) + } + return schema, nil +} + +// mapMySQLType maps MySQL/MariaDB information_schema DATA_TYPE and COLUMN_TYPE +// to a CanonicalType. +func mapMySQLType(dataType, columnType string) canonical.CanonicalType { + dt := strings.ToLower(dataType) + ct := strings.ToLower(columnType) + switch dt { + case "int", "smallint", "mediumint": + return canonical.CTInt + case "tinyint": + if ct == "tinyint(1)" { + return canonical.CTBoolean + } + return canonical.CTInt + case "bigint": + return canonical.CTBigInt + case "varchar": + return canonical.CTVarchar + case "text", "longtext", "mediumtext": + return canonical.CTText + case "decimal": + return canonical.CTNumeric + case "char": + if ct == "char(36)" { + return canonical.CTUUID + } + return canonical.CTText + case "datetime", "timestamp": + return canonical.CTTimestampTZ + case "json": + return canonical.CTJSON + default: + return canonical.CTText + } +} diff --git a/internal/driver/_mysqlcommon/inspect_schema_test.go b/internal/driver/_mysqlcommon/inspect_schema_test.go new file mode 100644 index 0000000..32e4135 --- /dev/null +++ b/internal/driver/_mysqlcommon/inspect_schema_test.go @@ -0,0 +1,39 @@ +package mysqlcommon + +import ( + "testing" + + "github.com/nixrajput/siphon/internal/canonical" +) + +func TestMapMySQLType(t *testing.T) { + cases := []struct { + dataType string + columnType string + want canonical.CanonicalType + }{ + {"int", "int", canonical.CTInt}, + {"smallint", "smallint(6)", canonical.CTInt}, + {"mediumint", "mediumint(9)", canonical.CTInt}, + {"tinyint", "tinyint(1)", canonical.CTBoolean}, + {"tinyint", "tinyint(4)", canonical.CTInt}, + {"bigint", "bigint(20)", canonical.CTBigInt}, + {"varchar", "varchar(255)", canonical.CTVarchar}, + {"text", "text", canonical.CTText}, + {"longtext", "longtext", canonical.CTText}, + {"mediumtext", "mediumtext", canonical.CTText}, + {"decimal", "decimal(10,2)", canonical.CTNumeric}, + {"char", "char(36)", canonical.CTUUID}, + {"char", "char(10)", canonical.CTText}, + {"datetime", "datetime", canonical.CTTimestampTZ}, + {"timestamp", "timestamp", canonical.CTTimestampTZ}, + {"json", "json", canonical.CTJSON}, + {"blob", "blob", canonical.CTText}, // fallback + } + for _, tc := range cases { + got := mapMySQLType(tc.dataType, tc.columnType) + if got != tc.want { + t.Errorf("mapMySQLType(%q, %q) = %q; want %q", tc.dataType, tc.columnType, got, tc.want) + } + } +} diff --git a/internal/driver/driver.go b/internal/driver/driver.go index 9401e99..c6fa841 100644 --- a/internal/driver/driver.go +++ b/internal/driver/driver.go @@ -6,6 +6,8 @@ import ( "context" "io" "time" + + "github.com/nixrajput/siphon/internal/canonical" ) // Driver is the protocol-level abstraction for a database engine. @@ -28,6 +30,20 @@ type Conn interface { Close() error } +// SchemaInspector is an optional interface a Conn may implement to expose typed +// schema information for cross-engine transfers. +type SchemaInspector interface { + InspectSchema(ctx context.Context) (*canonical.CanonicalSchema, error) +} + +// CanonicalTransfer is an optional interface a Conn may implement to support +// cross-engine data transfer via the canonical JSONL format. +type CanonicalTransfer interface { + EmitCanonical(ctx context.Context, schema *canonical.CanonicalSchema, w io.Writer) error + ConsumeCanonical(ctx context.Context, r io.Reader) error + ApplyChange(ctx context.Context, ch canonical.CanonicalChange) error +} + // Capabilities describes what an engine supports. Each flag gates a UI // affordance or feature path. Drivers must declare honestly. type Capabilities struct { diff --git a/internal/driver/mariadb/driver.go b/internal/driver/mariadb/driver.go index 0ae124d..4c10f3d 100644 --- a/internal/driver/mariadb/driver.go +++ b/internal/driver/mariadb/driver.go @@ -26,8 +26,8 @@ func (Driver) Capabilities() driver.Capabilities { Parallel: false, // mariadb-dump is single-threaded Compression: true, BinaryFormat: false, // SQL text dump, not a binary archive - CrossEngineSource: false, // Phase F - CrossEngineTarget: false, // Phase F + CrossEngineSource: true, + CrossEngineTarget: true, CDC: false, // Phase F NativeBackpressure: true, // CrossVersionIncremental defaults false @@ -35,7 +35,7 @@ func (Driver) Capabilities() driver.Capabilities { } func (Driver) Connect(ctx context.Context, p driver.Profile) (driver.Conn, error) { - return mysqlcommon.NewConn(ctx, p, "mariadb-dump", "mariadb", "mariadb-binlog", "mariadb.connect") + return mysqlcommon.NewConn(ctx, p, "mariadb-dump", "mariadb", "mariadb-binlog", "mariadb.connect", "mariadb") } // Compile-time check. diff --git a/internal/driver/mysql/driver.go b/internal/driver/mysql/driver.go index 25ef55f..d9aad48 100644 --- a/internal/driver/mysql/driver.go +++ b/internal/driver/mysql/driver.go @@ -26,8 +26,8 @@ func (Driver) Capabilities() driver.Capabilities { Parallel: false, // mysqldump is single-threaded Compression: true, BinaryFormat: false, // SQL text dump, not a binary archive - CrossEngineSource: false, // Phase F - CrossEngineTarget: false, // Phase F + CrossEngineSource: true, + CrossEngineTarget: true, CDC: false, // Phase F NativeBackpressure: true, // CrossVersionIncremental defaults false @@ -35,7 +35,7 @@ func (Driver) Capabilities() driver.Capabilities { } func (Driver) Connect(ctx context.Context, p driver.Profile) (driver.Conn, error) { - return mysqlcommon.NewConn(ctx, p, "mysqldump", "mysql", "mysqlbinlog", "mysql.connect") + return mysqlcommon.NewConn(ctx, p, "mysqldump", "mysql", "mysqlbinlog", "mysql.connect", "mysql") } // Compile-time check. diff --git a/internal/driver/postgres/canonical.go b/internal/driver/postgres/canonical.go new file mode 100644 index 0000000..bec0668 --- /dev/null +++ b/internal/driver/postgres/canonical.go @@ -0,0 +1,188 @@ +package postgres + +import ( + "context" + "database/sql" + "encoding/json" + "errors" + "fmt" + "io" + "sort" + + "github.com/nixrajput/siphon/internal/canonical" + "github.com/nixrajput/siphon/internal/errs" +) + +// EmitCanonical writes a table-by-table snapshot of schema as JSONL to w. +func (c *Conn) EmitCanonical(ctx context.Context, schema *canonical.CanonicalSchema, w io.Writer) error { + if err := canonical.WriteJSONL(w, map[string]*canonical.CanonicalSchema{"schema": schema}); err != nil { + return err + } + for _, t := range schema.Tables { + if err := pgEmitTable(ctx, c.db, t, w); err != nil { + return err + } + } + return nil +} + +func pgEmitTable(ctx context.Context, db *sql.DB, t canonical.CanonicalTable, w io.Writer) error { + query, err := canonical.BuildSelectSQL("postgres", t) + if err != nil { + return err + } + rows, err := db.QueryContext(ctx, query) + if err != nil { + return fmt.Errorf("cross-engine: select %s: %w", t.Name, err) + } + defer func() { _ = rows.Close() }() + + for rows.Next() { + vals := make([]any, len(t.Columns)) + ptrs := make([]any, len(t.Columns)) + for i := range vals { + ptrs[i] = &vals[i] + } + if err := rows.Scan(ptrs...); err != nil { + return fmt.Errorf("cross-engine: scan %s: %w", t.Name, err) + } + m := make(map[string]any, len(t.Columns)) + for i, col := range t.Columns { + m[col.Name] = canonical.NormalizeScanned(vals[i]) + } + if err := canonical.WriteJSONL(w, canonical.CanonicalRow{Table: t.Name, Values: m}); err != nil { + return err + } + } + return rows.Err() +} + +// ConsumeCanonical reads a stream produced by EmitCanonical and replays it into the database. +func (c *Conn) ConsumeCanonical(ctx context.Context, r io.Reader) error { + dec := json.NewDecoder(r) + + var header struct { + Schema *canonical.CanonicalSchema `json:"schema"` + } + if err := dec.Decode(&header); err != nil { + if errors.Is(err, io.EOF) { + return fmt.Errorf("cross-engine: empty stream, schema header missing") + } + return fmt.Errorf("cross-engine: decode schema header: %w", err) + } + if header.Schema == nil { + return fmt.Errorf("cross-engine: schema header missing") + } + + for _, t := range header.Schema.Tables { + stmt, err := canonical.BuildCreateTableSQL("postgres", t) + if err != nil { + return err + } + if _, err := c.db.ExecContext(ctx, stmt); err != nil { + return fmt.Errorf("cross-engine: create table %s: %w", t.Name, err) + } + } + + for { + var row canonical.CanonicalRow + if err := dec.Decode(&row); err != nil { + if errors.Is(err, io.EOF) { + break + } + return fmt.Errorf("cross-engine: decode row: %w", err) + } + if err := pgInsertRow(ctx, c.db, row); err != nil { + return err + } + } + return nil +} + +func pgInsertRow(ctx context.Context, db *sql.DB, row canonical.CanonicalRow) error { + cols := make([]string, 0, len(row.Values)) + for col := range row.Values { + cols = append(cols, col) + } + sort.Strings(cols) + sortedVals := make([]any, len(cols)) + for i, col := range cols { + sortedVals[i] = row.Values[col] + } + stmt, err := canonical.BuildInsertSQL("postgres", row.Table, cols) + if err != nil { + return err + } + if _, err := db.ExecContext(ctx, stmt, sortedVals...); err != nil { + return fmt.Errorf("cross-engine: insert into %s: %w", row.Table, err) + } + return nil +} + +// ApplyChange applies one CanonicalChange to the database. +func (c *Conn) ApplyChange(ctx context.Context, ch canonical.CanonicalChange) error { + if err := canonical.ValidateChangeKey(ch); err != nil { + return &errs.Error{ + Op: "postgres.apply", + Code: errs.CodeUser, + Cause: errs.ErrMissingPrimaryKey, + Hint: err.Error(), + } + } + + switch ch.Op { + case canonical.OpInsert: + cols := make([]string, 0, len(ch.Values)) + for col := range ch.Values { + cols = append(cols, col) + } + sort.Strings(cols) + stmt, err := canonical.BuildInsertSQL("postgres", ch.Table, cols) + if err != nil { + return err + } + args := make([]any, len(cols)) + for i, col := range cols { + args[i] = ch.Values[col] + } + _, err = c.db.ExecContext(ctx, stmt, args...) + return err + + case canonical.OpUpdate: + setCols, keyCols := canonical.ChangeColumns(ch) + stmt, err := canonical.BuildUpdateSQL("postgres", ch.Table, setCols, keyCols) + if err != nil { + return err + } + args := make([]any, 0, len(setCols)+len(keyCols)) + for _, col := range setCols { + args = append(args, ch.Values[col]) + } + for _, col := range keyCols { + args = append(args, ch.Key[col]) + } + _, err = c.db.ExecContext(ctx, stmt, args...) + return err + + case canonical.OpDelete: + _, keyCols := canonical.ChangeColumns(ch) + stmt, err := canonical.BuildDeleteSQL("postgres", ch.Table, keyCols) + if err != nil { + return err + } + args := make([]any, len(keyCols)) + for i, col := range keyCols { + args[i] = ch.Key[col] + } + _, err = c.db.ExecContext(ctx, stmt, args...) + return err + + default: + return &errs.Error{ + Op: "postgres.apply", + Code: errs.CodeSystem, + Cause: errs.ErrDumpCorrupt, + Hint: "unknown change op " + string(ch.Op), + } + } +} diff --git a/internal/driver/postgres/driver.go b/internal/driver/postgres/driver.go index e9024dd..873fe5f 100644 --- a/internal/driver/postgres/driver.go +++ b/internal/driver/postgres/driver.go @@ -32,8 +32,8 @@ func (Driver) Capabilities() driver.Capabilities { Parallel: true, // capability exists in the engine, but not wired in Phase B: pg_dump -j needs -Fd (see backup.go) and RestoreOpts carries no Parallel field yet — both land in a later phase Compression: true, BinaryFormat: true, - CrossEngineSource: false, // Phase F - CrossEngineTarget: false, // Phase F + CrossEngineSource: true, + CrossEngineTarget: true, CDC: false, // Phase F NativeBackpressure: true, } diff --git a/internal/driver/postgres/inspect_schema.go b/internal/driver/postgres/inspect_schema.go new file mode 100644 index 0000000..c2d022c --- /dev/null +++ b/internal/driver/postgres/inspect_schema.go @@ -0,0 +1,140 @@ +package postgres + +import ( + "context" + "strings" + + "github.com/nixrajput/siphon/internal/canonical" +) + +// InspectSchema queries information_schema for tables (public schema) and their +// primary key columns, returning a CanonicalSchema. +func (c *Conn) InspectSchema(ctx context.Context) (*canonical.CanonicalSchema, error) { + const colQuery = ` +SELECT table_name, column_name, data_type, is_nullable, character_maximum_length, numeric_precision, numeric_scale +FROM information_schema.columns +WHERE table_schema = 'public' +ORDER BY table_name, ordinal_position +` + rows, err := c.db.QueryContext(ctx, colQuery) + if err != nil { + return nil, err + } + defer func() { _ = rows.Close() }() + + tableMap := make(map[string]*canonical.CanonicalTable) + var tableOrder []string + + for rows.Next() { + var ( + tableName string + colName string + dataType string + isNullable string + charMaxLen *int + numPrec *int + numScale *int + ) + if err := rows.Scan(&tableName, &colName, &dataType, &isNullable, &charMaxLen, &numPrec, &numScale); err != nil { + return nil, err + } + if _, ok := tableMap[tableName]; !ok { + tableMap[tableName] = &canonical.CanonicalTable{Name: tableName} + tableOrder = append(tableOrder, tableName) + } + col := canonical.CanonicalColumn{ + Name: colName, + Type: mapPGType(dataType), + Nullable: isNullable == "YES", + } + if charMaxLen != nil { + col.Precision = *charMaxLen + } + if numPrec != nil { + col.Precision = *numPrec + } + if numScale != nil { + col.Scale = *numScale + } + tableMap[tableName].Columns = append(tableMap[tableName].Columns, col) + } + if err := rows.Err(); err != nil { + return nil, err + } + + // Fetch primary key columns + const pkQuery = ` +SELECT tc.table_name, kcu.column_name +FROM information_schema.table_constraints tc +JOIN information_schema.key_column_usage kcu + ON kcu.constraint_name = tc.constraint_name + AND kcu.table_schema = tc.table_schema + AND kcu.table_name = tc.table_name +WHERE tc.constraint_type = 'PRIMARY KEY' + AND tc.table_schema = 'public' +ORDER BY tc.table_name, kcu.ordinal_position +` + pkRows, err := c.db.QueryContext(ctx, pkQuery) + if err != nil { + return nil, err + } + defer func() { _ = pkRows.Close() }() + + pkSet := make(map[string]map[string]bool) + for pkRows.Next() { + var tbl, col string + if err := pkRows.Scan(&tbl, &col); err != nil { + return nil, err + } + if pkSet[tbl] == nil { + pkSet[tbl] = make(map[string]bool) + } + pkSet[tbl][col] = true + } + if err := pkRows.Err(); err != nil { + return nil, err + } + + // Mark PrimaryKey on columns + for tblName, tbl := range tableMap { + if pks, ok := pkSet[tblName]; ok { + for i := range tbl.Columns { + if pks[tbl.Columns[i].Name] { + tbl.Columns[i].PrimaryKey = true + } + } + } + } + + schema := &canonical.CanonicalSchema{} + for _, name := range tableOrder { + schema.Tables = append(schema.Tables, *tableMap[name]) + } + return schema, nil +} + +// mapPGType maps a Postgres information_schema data_type string to a CanonicalType. +func mapPGType(dataType string) canonical.CanonicalType { + switch strings.ToLower(dataType) { + case "integer", "smallint": + return canonical.CTInt + case "bigint": + return canonical.CTBigInt + case "boolean": + return canonical.CTBoolean + case "character varying": + return canonical.CTVarchar + case "text": + return canonical.CTText + case "numeric", "decimal": + return canonical.CTNumeric + case "uuid": + return canonical.CTUUID + case "timestamp with time zone": + return canonical.CTTimestampTZ + case "json", "jsonb": + return canonical.CTJSON + default: + return canonical.CTText + } +} diff --git a/internal/driver/postgres/inspect_schema_test.go b/internal/driver/postgres/inspect_schema_test.go new file mode 100644 index 0000000..1ad4acd --- /dev/null +++ b/internal/driver/postgres/inspect_schema_test.go @@ -0,0 +1,34 @@ +package postgres + +import ( + "testing" + + "github.com/nixrajput/siphon/internal/canonical" +) + +func TestMapPGType(t *testing.T) { + cases := []struct { + input string + want canonical.CanonicalType + }{ + {"integer", canonical.CTInt}, + {"smallint", canonical.CTInt}, + {"bigint", canonical.CTBigInt}, + {"boolean", canonical.CTBoolean}, + {"character varying", canonical.CTVarchar}, + {"text", canonical.CTText}, + {"numeric", canonical.CTNumeric}, + {"decimal", canonical.CTNumeric}, + {"uuid", canonical.CTUUID}, + {"timestamp with time zone", canonical.CTTimestampTZ}, + {"json", canonical.CTJSON}, + {"jsonb", canonical.CTJSON}, + {"bytea", canonical.CTText}, // fallback + } + for _, tc := range cases { + got := mapPGType(tc.input) + if got != tc.want { + t.Errorf("mapPGType(%q) = %q; want %q", tc.input, got, tc.want) + } + } +} From 72907951b8406f1fa0b7f3826bac888b5632f1da Mon Sep 17 00:00:00 2001 From: Nikhil Rajput Date: Tue, 23 Jun 2026 02:58:50 +0530 Subject: [PATCH 05/13] fix(driver): cross-engine integration test + interface assertions + ApplyChange guard test --- internal/app/cross_engine_integration_test.go | 172 ++++++++++++++++++ .../driver/_mysqlcommon/canonical_test.go | 66 +++++++ internal/driver/_mysqlcommon/conn.go | 2 + internal/driver/postgres/driver.go | 4 + 4 files changed, 244 insertions(+) create mode 100644 internal/app/cross_engine_integration_test.go create mode 100644 internal/driver/_mysqlcommon/canonical_test.go diff --git a/internal/app/cross_engine_integration_test.go b/internal/app/cross_engine_integration_test.go new file mode 100644 index 0000000..18ab357 --- /dev/null +++ b/internal/app/cross_engine_integration_test.go @@ -0,0 +1,172 @@ +//go:build integration + +package app + +import ( + "context" + "database/sql" + "fmt" + "testing" + "time" + + mysqlctr "github.com/testcontainers/testcontainers-go/modules/mysql" + pgctr "github.com/testcontainers/testcontainers-go/modules/postgres" + + mysqlcommon "github.com/nixrajput/siphon/internal/driver/_mysqlcommon" + + _ "github.com/jackc/pgx/v5/stdlib" + + "github.com/nixrajput/siphon/internal/config" + "github.com/nixrajput/siphon/internal/driver" + "github.com/nixrajput/siphon/internal/dumps" + "github.com/nixrajput/siphon/internal/jobs" + "github.com/nixrajput/siphon/internal/profile" + "github.com/nixrajput/siphon/internal/secrets" +) + +// startIntegPG starts a Postgres 16-alpine testcontainer and returns its +// host, mapped port, and a cleanup function. +func startIntegPG(t *testing.T) (host string, port int, cleanup func()) { + t.Helper() + ctx := context.Background() + c, err := pgctr.Run(ctx, "postgres:16-alpine", + pgctr.WithDatabase("test"), + pgctr.WithUsername("postgres"), + pgctr.WithPassword("postgres"), + pgctr.BasicWaitStrategies(), + ) + if err != nil { + t.Fatalf("start postgres container: %v", err) + } + h, err := c.Host(ctx) + if err != nil { + _ = c.Terminate(ctx) + t.Fatalf("postgres container host: %v", err) + } + p, err := c.MappedPort(ctx, "5432/tcp") + if err != nil { + _ = c.Terminate(ctx) + t.Fatalf("postgres container port: %v", err) + } + return h, int(p.Num()), func() { _ = c.Terminate(ctx) } +} + +// startIntegMySQL starts a MySQL 8.0 testcontainer and returns its host, +// mapped port, and a cleanup function. +func startIntegMySQL(t *testing.T) (host string, port int, cleanup func()) { + t.Helper() + ctx := context.Background() + c, err := mysqlctr.Run(ctx, "mysql:8.0", + mysqlctr.WithDatabase("test"), + mysqlctr.WithUsername("root"), + mysqlctr.WithPassword("rootpass"), + ) + if err != nil { + t.Fatalf("start mysql container: %v", err) + } + h, err := c.Host(ctx) + if err != nil { + _ = c.Terminate(ctx) + t.Fatalf("mysql container host: %v", err) + } + p, err := c.MappedPort(ctx, "3306/tcp") + if err != nil { + _ = c.Terminate(ctx) + t.Fatalf("mysql container port: %v", err) + } + return h, int(p.Num()), func() { _ = c.Terminate(ctx) } +} + +// TestCrossEngineSync_PostgresToMySQL starts real Postgres and MySQL +// testcontainers, seeds a primary-keyed table on the Postgres side, runs +// app.Sync with CrossEngine:true, and asserts the rows arrive in MySQL. +func TestCrossEngineSync_PostgresToMySQL(t *testing.T) { + pgHost, pgPort, pgCleanup := startIntegPG(t) + defer pgCleanup() + + myHost, myPort, myCleanup := startIntegMySQL(t) + defer myCleanup() + + // Seed the Postgres source table. + pgDSN := fmt.Sprintf("host=%s port=%d user=postgres password=postgres dbname=test sslmode=disable", pgHost, pgPort) + pgDB, err := sql.Open("pgx", pgDSN) + if err != nil { + t.Fatalf("open postgres: %v", err) + } + defer func() { _ = pgDB.Close() }() + + ctx := context.Background() + if _, err := pgDB.ExecContext(ctx, + `DROP TABLE IF EXISTS widgets; + CREATE TABLE widgets(id int PRIMARY KEY, name text); + INSERT INTO widgets VALUES (1,'wrench'),(2,'hammer'),(3,'pliers');`); err != nil { + t.Fatalf("seed postgres: %v", err) + } + + // Build Deps with both profiles pointing at the containers. + cfg := &config.Config{Profiles: map[string]config.ProfileConfig{ + "pg": {Driver: "postgres", Host: pgHost, Port: pgPort, User: "postgres", Password: "postgres", Database: "test", SSLMode: "disable"}, + "my": {Driver: "mysql", Host: myHost, Port: myPort, User: "root", Password: "rootpass", Database: "test"}, + }} + res := secrets.NewResolver(secrets.Passthrough{}) + ps := profile.New(cfg, res, func(*config.Config) error { return nil }) + cat, err := dumps.NewCatalog(t.TempDir()) + if err != nil { + t.Fatalf("catalog: %v", err) + } + runner := jobs.NewRunner() + deps := Deps{ + Profiles: ps, + Dumps: cat, + Runner: runner, + Drivers: DefaultDrivers(), + } + + // Run the cross-engine sync. + ch, _, err := Sync(ctx, deps, SyncOpts{From: "pg", To: "my", CrossEngine: true}) + if err != nil { + t.Fatalf("Sync setup: %v", err) + } + + // Drain the event channel. Any Error event is a test failure. + timer := time.NewTimer(120 * time.Second) + defer timer.Stop() + for { + select { + case ev, ok := <-ch: + if !ok { + goto done + } + if ev.Err != nil { + t.Fatalf("Sync job error: %v", ev.Err) + } + case <-timer.C: + t.Fatal("Sync did not complete within 120 s") + } + } +done: + + // Open MySQL and count the transferred rows. + myProf := driver.Profile{ + Driver: "mysql", + Host: myHost, + Port: myPort, + User: "root", + Password: "rootpass", + Database: "test", + SSLMode: "disable", + } + myDB, err := mysqlcommon.Open(myProf) + if err != nil { + t.Fatalf("open mysql: %v", err) + } + defer func() { _ = myDB.Close() }() + + var count int + if err := myDB.QueryRowContext(ctx, "SELECT COUNT(*) FROM widgets").Scan(&count); err != nil { + t.Fatalf("count mysql widgets: %v", err) + } + if count != 3 { + t.Errorf("mysql widgets: got %d rows, want 3", count) + } +} diff --git a/internal/driver/_mysqlcommon/canonical_test.go b/internal/driver/_mysqlcommon/canonical_test.go new file mode 100644 index 0000000..fcc7a51 --- /dev/null +++ b/internal/driver/_mysqlcommon/canonical_test.go @@ -0,0 +1,66 @@ +package mysqlcommon + +import ( + "context" + "errors" + "testing" + + "github.com/nixrajput/siphon/internal/canonical" + "github.com/nixrajput/siphon/internal/errs" +) + +// TestApplyChange_KeylessUpdateWrapsCodeUser verifies that ApplyChange +// returns *errs.Error with Code==CodeUser and Cause==ErrMissingPrimaryKey +// when called with an UPDATE that has an empty Key map. The guard runs +// before any database access, so a zero *Conn suffices here. +func TestApplyChange_KeylessUpdateWrapsCodeUser(t *testing.T) { + c := &Conn{engine: "mysql"} + ch := canonical.CanonicalChange{ + Op: canonical.OpUpdate, + Table: "widgets", + Key: nil, // no primary key — should be rejected + Values: map[string]any{"name": "hammer"}, + } + + err := c.ApplyChange(context.Background(), ch) + if err == nil { + t.Fatal("ApplyChange: expected error, got nil") + } + + var appErr *errs.Error + if !errors.As(err, &appErr) { + t.Fatalf("ApplyChange: error is %T, want *errs.Error", err) + } + if appErr.Code != errs.CodeUser { + t.Errorf("ApplyChange: Code = %v, want CodeUser (%v)", appErr.Code, errs.CodeUser) + } + if !errors.Is(err, errs.ErrMissingPrimaryKey) { + t.Errorf("ApplyChange: errors.Is(err, ErrMissingPrimaryKey) = false; err = %v", err) + } +} + +// TestApplyChange_KeylessDeleteWrapsCodeUser mirrors the UPDATE case for DELETE. +func TestApplyChange_KeylessDeleteWrapsCodeUser(t *testing.T) { + c := &Conn{engine: "mysql"} + ch := canonical.CanonicalChange{ + Op: canonical.OpDelete, + Table: "widgets", + Key: map[string]any{}, // empty key — same guard + } + + err := c.ApplyChange(context.Background(), ch) + if err == nil { + t.Fatal("ApplyChange: expected error, got nil") + } + + var appErr *errs.Error + if !errors.As(err, &appErr) { + t.Fatalf("ApplyChange: error is %T, want *errs.Error", err) + } + if appErr.Code != errs.CodeUser { + t.Errorf("ApplyChange: Code = %v, want CodeUser (%v)", appErr.Code, errs.CodeUser) + } + if !errors.Is(err, errs.ErrMissingPrimaryKey) { + t.Errorf("ApplyChange: errors.Is(err, ErrMissingPrimaryKey) = false; err = %v", err) + } +} diff --git a/internal/driver/_mysqlcommon/conn.go b/internal/driver/_mysqlcommon/conn.go index aa564e4..ad14745 100644 --- a/internal/driver/_mysqlcommon/conn.go +++ b/internal/driver/_mysqlcommon/conn.go @@ -38,6 +38,8 @@ type Conn struct { } var _ driver.Conn = (*Conn)(nil) +var _ driver.SchemaInspector = (*Conn)(nil) +var _ driver.CanonicalTransfer = (*Conn)(nil) // NewConn opens + pings the database and returns a ready Conn. The ping is // wrapped in a bounded retry (jobs.Retry, 3 attempts) — same policy as the diff --git a/internal/driver/postgres/driver.go b/internal/driver/postgres/driver.go index 873fe5f..95cfff9 100644 --- a/internal/driver/postgres/driver.go +++ b/internal/driver/postgres/driver.go @@ -113,4 +113,8 @@ type Conn struct { p driver.Profile } +var _ driver.Conn = (*Conn)(nil) +var _ driver.SchemaInspector = (*Conn)(nil) +var _ driver.CanonicalTransfer = (*Conn)(nil) + func (c *Conn) Close() error { return c.db.Close() } From b36d0e48d5bafee1107ffb4e02e820a546eb215d Mon Sep 17 00:00:00 2001 From: Nikhil Rajput Date: Tue, 23 Jun 2026 03:23:09 +0530 Subject: [PATCH 06/13] =?UTF-8?q?feat(driver):=20ChangeStreamer=20?= =?UTF-8?q?=E2=80=94=20pglogrepl=20+=20MySQL=20binlog=20decode?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Add driver.ChangeStreamer interface: stream engine-neutral CanonicalChanges from a Position, returning the final Position. - Postgres: pglogrepl pgoutput (proto v1) logical decoding over a replication-mode conn; ensure siphon_pub publication + siphon_logical slot; decode Insert/Update/Delete with PK-flag key extraction; periodic standby acks; ctx cancel is a clean stop. - MySQL/MariaDB: shell out to the fork binlog tool with --verbose --base64-output=DECODE-ROWS and parse the ### ROW pseudo-SQL into CanonicalChanges; map @N to column names + PK via information_schema. - Tests: offline binlog-parser unit tests; bounded Postgres logical stream integration test (wal_level=logical container). - Capabilities unchanged: verb wiring + cap flips belong to Task 5/6. --- go.mod | 8 +- go.sum | 12 +- internal/driver/_mysqlcommon/changestream.go | 418 ++++++++++++++++++ .../driver/_mysqlcommon/changestream_test.go | 120 +++++ internal/driver/driver.go | 8 + internal/driver/postgres/changestream.go | 394 +++++++++++++++++ .../postgres/changestream_integration_test.go | 139 ++++++ 7 files changed, 1092 insertions(+), 7 deletions(-) create mode 100644 internal/driver/_mysqlcommon/changestream.go create mode 100644 internal/driver/_mysqlcommon/changestream_test.go create mode 100644 internal/driver/postgres/changestream.go create mode 100644 internal/driver/postgres/changestream_integration_test.go diff --git a/go.mod b/go.mod index bc76031..d092437 100644 --- a/go.mod +++ b/go.mod @@ -9,10 +9,12 @@ require ( github.com/charmbracelet/lipgloss v1.1.0 github.com/charmbracelet/x/exp/teatest v0.0.0-20260527151214-009e6338d40d github.com/go-sql-driver/mysql v1.10.0 + github.com/jackc/pglogrepl v0.0.0-20260401131349-e37c41485510 github.com/jackc/pgx/v5 v5.9.2 github.com/muesli/termenv v0.16.0 github.com/oklog/ulid/v2 v2.1.1 github.com/spf13/cobra v1.10.2 + github.com/testcontainers/testcontainers-go v0.42.0 github.com/testcontainers/testcontainers-go/modules/mariadb v0.42.0 github.com/testcontainers/testcontainers-go/modules/mysql v0.42.0 github.com/testcontainers/testcontainers-go/modules/postgres v0.42.0 @@ -58,6 +60,7 @@ require ( github.com/go-ole/go-ole v1.2.6 // indirect github.com/google/uuid v1.6.0 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect + github.com/jackc/pgio v1.0.0 // indirect github.com/jackc/pgpassfile v1.0.0 // indirect github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect github.com/jackc/puddle/v2 v2.2.2 // indirect @@ -91,7 +94,6 @@ require ( github.com/sirupsen/logrus v1.9.4 // indirect github.com/spf13/pflag v1.0.9 // indirect github.com/stretchr/testify v1.11.1 // indirect - github.com/testcontainers/testcontainers-go v0.42.0 // indirect github.com/tklauser/go-sysconf v0.3.16 // indirect github.com/tklauser/numcpus v0.11.0 // indirect github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect @@ -102,7 +104,7 @@ require ( go.opentelemetry.io/otel/metric v1.41.0 // indirect go.opentelemetry.io/otel/trace v1.41.0 // indirect golang.org/x/crypto v0.48.0 // indirect - golang.org/x/sync v0.19.0 // indirect + golang.org/x/sync v0.20.0 // indirect golang.org/x/sys v0.42.0 // indirect - golang.org/x/text v0.34.0 // indirect + golang.org/x/text v0.35.0 // indirect ) diff --git a/go.sum b/go.sum index 2c2ab9c..da9bbc7 100644 --- a/go.sum +++ b/go.sum @@ -106,6 +106,10 @@ github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= +github.com/jackc/pgio v1.0.0 h1:g12B9UwVnzGhueNavwioyEEpAmqMe1E/BN9ES+8ovkE= +github.com/jackc/pgio v1.0.0/go.mod h1:oP+2QK2wFfUWgr+gxjoBH9KGBb31Eio69xUb0w5bYf8= +github.com/jackc/pglogrepl v0.0.0-20260401131349-e37c41485510 h1:+PJCokZ2BhyDKlncScmiNzBwqOx+yH1i8xRlWN/wn6A= +github.com/jackc/pglogrepl v0.0.0-20260401131349-e37c41485510/go.mod h1:UzTJ5Jjuf4O9hYWW+HYVwVldYz9J7CaePW0iuNJkrPQ= github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7UlwOQYpKFhBabPMi4aNAfoODPEFNiAnClxo= @@ -232,8 +236,8 @@ golang.org/x/crypto v0.48.0 h1:/VRzVqiRSggnhY7gNRxPauEQ5Drw9haKdM0jqfcCFts= golang.org/x/crypto v0.48.0/go.mod h1:r0kV5h3qnFPlQnBSrULhlsRfryS2pmewsg+XfMgkVos= golang.org/x/exp v0.0.0-20231006140011-7918f672742d h1:jtJma62tbqLibJ5sFQz8bKtEM8rJBtfilJ2qTU199MI= golang.org/x/exp v0.0.0-20231006140011-7918f672742d/go.mod h1:ldy0pHrwJyGW56pPQzzkH36rKxoZW1tw7ZJpeKx+hdo= -golang.org/x/sync v0.19.0 h1:vV+1eWNmZ5geRlYjzm2adRgW2/mcpevXNg50YZtPCE4= -golang.org/x/sync v0.19.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= +golang.org/x/sync v0.20.0 h1:e0PTpb7pjO8GAtTs2dQ6jYa5BWYlMuX047Dco/pItO4= +golang.org/x/sync v0.20.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201204225414-ed752295db88/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -243,8 +247,8 @@ golang.org/x/sys v0.42.0 h1:omrd2nAlyT5ESRdCLYdm3+fMfNFE/+Rf4bDIQImRJeo= golang.org/x/sys v0.42.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= golang.org/x/term v0.40.0 h1:36e4zGLqU4yhjlmxEaagx2KuYbJq3EwY8K943ZsHcvg= golang.org/x/term v0.40.0/go.mod h1:w2P8uVp06p2iyKKuvXIm7N/y0UCRt3UfJTfZ7oOpglM= -golang.org/x/text v0.34.0 h1:oL/Qq0Kdaqxa1KbNeMKwQq0reLCCaFtqu2eNuSeNHbk= -golang.org/x/text v0.34.0/go.mod h1:homfLqTYRFyVYemLBFl5GgL/DWEiH5wcsQ5gSh1yziA= +golang.org/x/text v0.35.0 h1:JOVx6vVDFokkpaq1AEptVzLTpDe9KGpj5tR4/X+ybL8= +golang.org/x/text v0.35.0/go.mod h1:khi/HExzZJ2pGnjenulevKNX1W67CUy0AsXcNubPGCA= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= diff --git a/internal/driver/_mysqlcommon/changestream.go b/internal/driver/_mysqlcommon/changestream.go new file mode 100644 index 0000000..1ef6194 --- /dev/null +++ b/internal/driver/_mysqlcommon/changestream.go @@ -0,0 +1,418 @@ +package mysqlcommon + +import ( + "bufio" + "context" + "errors" + "io" + "os" + "os/exec" + "strconv" + "strings" + + "github.com/nixrajput/siphon/internal/canonical" + "github.com/nixrajput/siphon/internal/driver" + "github.com/nixrajput/siphon/internal/errs" +) + +var _ driver.ChangeStreamer = (*Conn)(nil) + +// StreamChanges streams binlog ROW events from `from` as engine-neutral +// CanonicalChanges, decoding the fork's binlog tool's --verbose pseudo-SQL +// (### INSERT/UPDATE/DELETE … ### @N=…). Bounded callers cancel ctx at a target +// position; unbounded callers stream until ctx cancel. ctx cancellation is the +// normal stop signal and is NOT reported as an error. +// +// Column positions (@1,@2,…) are mapped to names via information_schema (cached +// per table); the key is the table's primary-key columns. UPDATE events carry a +// WHERE (old image) block for the key and a SET (new image) block for Values. +// +// NOTE: this parser is structurally complete but UNPROVEN locally (no MySQL +// here). The pseudo-SQL grammar is stable across MySQL/MariaDB --verbose output, +// but value typing (everything decodes as a string) and edge cases (multi-row +// events, NULL rendering, quoting) need validation against a live +// log_bin=ON, binlog_format=ROW server in CI. +func (c *Conn) StreamChanges(ctx context.Context, from canonical.Position, emit func(canonical.CanonicalChange) error) (canonical.Position, error) { + if err := c.ValidateBinlogFormat(ctx); err != nil { + return canonical.Position{}, err + } + if from.BinlogFile == "" { + pos, err := c.CaptureBinlogPosition(ctx) + if err != nil { + return canonical.Position{}, err + } + from = canonical.Position{BinlogFile: pos.File, BinlogPos: pos.Position} + } + + meta := newTableMetaCache(ctx, c) + + since := BinlogPosition{File: from.BinlogFile, Position: from.BinlogPos} + args := append(binlogArgs(c.p, since, c.binlogBinary), + "--verbose", "--base64-output=DECODE-ROWS") + // binlogArgs places the starting binlog file last (positional); the flags + // above must precede it, so rebuild with the file kept last. + args = reorderBinlogFileLast(args, since.File) + + cmd := exec.CommandContext(ctx, c.binlogBinary, args...) + cmd.Env = withMySQLPwd(os.Environ(), c.p.Password) + cmd.Stderr = os.Stderr + stdout, err := cmd.StdoutPipe() + if err != nil { + return canonical.Position{}, &errs.Error{Op: c.binlogBinary + ".stream", Code: errs.CodeSystem, Cause: err} + } + if err := cmd.Start(); err != nil { + return canonical.Position{}, toolErr(c.binlogBinary, c.binlogBinary+".stream", err) + } + + endPos, parseErr := parseBinlogRows(stdout, meta, emit, since) + + // Kill the (possibly unbounded) tool and reap it. On ctx cancel the kill is + // expected; we only surface a parse error, not the resulting exec error. + _ = cmd.Process.Kill() + _ = cmd.Wait() + + final := canonical.Position{BinlogFile: endPos.File, BinlogPos: endPos.Position} + if parseErr != nil && ctx.Err() == nil { + return final, parseErr + } + return final, nil +} + +// reorderBinlogFileLast removes any occurrence of the positional binlog file +// from args and re-appends it at the end, so flags appended after binlogArgs +// (which already placed the file last) stay before the positional argument. +func reorderBinlogFileLast(args []string, file string) []string { + out := make([]string, 0, len(args)) + for _, a := range args { + if a == file { + continue + } + out = append(out, a) + } + return append(out, file) +} + +// rowEvent is the in-progress decode of a single ### INSERT/UPDATE/DELETE block. +type rowEvent struct { + op canonical.ChangeOp + table string // unqualified table name + whereM map[int]string + setM map[int]string + section string // "where" or "set", controls which map @N= lines fill +} + +// parseBinlogRows scans the binlog tool's --verbose output line by line, +// assembling ### blocks into CanonicalChanges and calling emit per change. It +// returns the final binlog position seen (from `# at ` comments) so the +// caller can stamp the envelope / CDC state. +func parseBinlogRows(r io.Reader, meta *tableMetaCache, emit func(canonical.CanonicalChange) error, start BinlogPosition) (BinlogPosition, error) { + sc := bufio.NewScanner(r) + sc.Buffer(make([]byte, 0, 64*1024), 4*1024*1024) + + pos := start + var ev *rowEvent + + flush := func() error { + if ev == nil { + return nil + } + ch, err := meta.toChange(ev) + ev = nil + if err != nil { + return err + } + if ch == nil { + return nil + } + return emit(*ch) + } + + for sc.Scan() { + line := sc.Text() + + // Track the current binlog offset from "# at " markers. + if p, ok := parseAtMarker(line); ok { + pos.Position = p + continue + } + // Track the active binlog file from rotate events. + if f, ok := parseRotateFile(line); ok { + pos.File = f + continue + } + + t := strings.TrimSpace(line) + if !strings.HasPrefix(t, "###") { + continue + } + body := strings.TrimSpace(strings.TrimPrefix(t, "###")) + + switch { + case strings.HasPrefix(body, "INSERT INTO "): + if err := flush(); err != nil { + return pos, err + } + ev = &rowEvent{op: canonical.OpInsert, table: tableFromRef(body[len("INSERT INTO "):]), setM: map[int]string{}, section: "set"} + case strings.HasPrefix(body, "DELETE FROM "): + if err := flush(); err != nil { + return pos, err + } + ev = &rowEvent{op: canonical.OpDelete, table: tableFromRef(body[len("DELETE FROM "):]), whereM: map[int]string{}, section: "where"} + case strings.HasPrefix(body, "UPDATE "): + if err := flush(); err != nil { + return pos, err + } + ev = &rowEvent{op: canonical.OpUpdate, table: tableFromRef(body[len("UPDATE "):]), whereM: map[int]string{}, setM: map[int]string{}} + case body == "WHERE": + if ev != nil { + ev.section = "where" + if ev.whereM == nil { + ev.whereM = map[int]string{} + } + } + case body == "SET": + if ev != nil { + ev.section = "set" + if ev.setM == nil { + ev.setM = map[int]string{} + } + } + case strings.HasPrefix(body, "@"): + if ev != nil { + if n, val, ok := parseColAssign(body); ok { + if ev.section == "where" { + ev.whereM[n] = val + } else { + ev.setM[n] = val + } + } + } + } + } + if err := flush(); err != nil { + return pos, err + } + if err := sc.Err(); err != nil { + return pos, &errs.Error{Op: "mysql.stream", Code: errs.CodeSystem, Cause: err} + } + return pos, nil +} + +// parseAtMarker extracts the offset from a "# at 12345" binlog comment. +func parseAtMarker(line string) (uint64, bool) { + t := strings.TrimSpace(line) + const prefix = "# at " + if !strings.HasPrefix(t, prefix) { + return 0, false + } + n, err := strconv.ParseUint(strings.TrimSpace(t[len(prefix):]), 10, 64) + if err != nil { + return 0, false + } + return n, true +} + +// parseRotateFile extracts the next binlog file name from a Rotate-event +// comment, e.g. "#... Rotate to mysql-bin.000124 pos: 4". +func parseRotateFile(line string) (string, bool) { + i := strings.Index(line, "Rotate to ") + if i < 0 { + return "", false + } + rest := strings.TrimSpace(line[i+len("Rotate to "):]) + if j := strings.Index(rest, " "); j >= 0 { + rest = rest[:j] + } + rest = strings.TrimSuffix(strings.TrimSpace(rest), ";") + if rest == "" { + return "", false + } + return rest, true +} + +// tableFromRef extracts the unqualified table name from a "`db`.`tbl`" or +// "db.tbl" reference, stripping backticks and any trailing tokens. +func tableFromRef(ref string) string { + ref = strings.TrimSpace(ref) + if i := strings.IndexAny(ref, " \t"); i >= 0 { + ref = ref[:i] + } + if i := strings.LastIndex(ref, "."); i >= 0 { + ref = ref[i+1:] + } + return strings.Trim(ref, "`") +} + +// parseColAssign parses a "@3='foo'" assignment into its 1-based column index +// and unquoted value. Values mysqlbinlog wraps in single quotes are unquoted; +// NULL renders as the literal NULL and maps to a nil value (returned as the +// empty string flagged via the caller's NULL handling — here we keep "NULL" +// literal-stripped to nil at change-build time). +func parseColAssign(body string) (int, string, bool) { + // body looks like "@3=value" (mysqlbinlog) optionally with a trailing + // "/* ... */" type comment. + eq := strings.IndexByte(body, '=') + if eq < 0 || !strings.HasPrefix(body, "@") { + return 0, "", false + } + n, err := strconv.Atoi(strings.TrimSpace(body[1:eq])) + if err != nil { + return 0, "", false + } + val := strings.TrimSpace(body[eq+1:]) + if i := strings.Index(val, "/*"); i >= 0 { // strip mysqlbinlog type comment + val = strings.TrimSpace(val[:i]) + } + val = unquoteBinlogValue(val) + return n, val, true +} + +// unquoteBinlogValue strips the surrounding single quotes mysqlbinlog adds to +// string values and unescapes the doubled/backslash escapes it emits. +func unquoteBinlogValue(v string) string { + if len(v) >= 2 && v[0] == '\'' && v[len(v)-1] == '\'' { + inner := v[1 : len(v)-1] + inner = strings.ReplaceAll(inner, `\'`, `'`) + inner = strings.ReplaceAll(inner, `''`, `'`) + inner = strings.ReplaceAll(inner, `\\`, `\`) + return inner + } + return v +} + +// tableMetaCache resolves @N column positions to names and identifies primary +// keys, caching the per-table layout (the binlog stream repeats events for the +// same tables, so one information_schema lookup per table suffices). +type tableMetaCache struct { + ctx context.Context + conn *Conn + cache map[string]*tableLayout +} + +type tableLayout struct { + cols []string // ordered column names; index 0 == @1 + pk map[string]bool // primary-key column names +} + +func newTableMetaCache(ctx context.Context, c *Conn) *tableMetaCache { + return &tableMetaCache{ctx: ctx, conn: c, cache: map[string]*tableLayout{}} +} + +// layout returns the cached (or freshly queried) column order + PK set for a +// table in the connection's database. +func (m *tableMetaCache) layout(table string) (*tableLayout, error) { + if l, ok := m.cache[table]; ok { + return l, nil + } + const colQ = `SELECT COLUMN_NAME FROM information_schema.COLUMNS +WHERE TABLE_SCHEMA = ? AND TABLE_NAME = ? ORDER BY ORDINAL_POSITION` + rows, err := m.conn.db.QueryContext(m.ctx, colQ, m.conn.p.Database, table) + if err != nil { + return nil, &errs.Error{Op: "mysql.stream", Code: errs.CodeSystem, Cause: err} + } + var cols []string + for rows.Next() { + var name string + if err := rows.Scan(&name); err != nil { + _ = rows.Close() + return nil, &errs.Error{Op: "mysql.stream", Code: errs.CodeSystem, Cause: err} + } + cols = append(cols, name) + } + _ = rows.Close() + if err := rows.Err(); err != nil { + return nil, &errs.Error{Op: "mysql.stream", Code: errs.CodeSystem, Cause: err} + } + + const pkQ = `SELECT COLUMN_NAME FROM information_schema.KEY_COLUMN_USAGE +WHERE TABLE_SCHEMA = ? AND TABLE_NAME = ? AND CONSTRAINT_NAME = 'PRIMARY'` + pkRows, err := m.conn.db.QueryContext(m.ctx, pkQ, m.conn.p.Database, table) + if err != nil { + return nil, &errs.Error{Op: "mysql.stream", Code: errs.CodeSystem, Cause: err} + } + pk := map[string]bool{} + for pkRows.Next() { + var name string + if err := pkRows.Scan(&name); err != nil { + _ = pkRows.Close() + return nil, &errs.Error{Op: "mysql.stream", Code: errs.CodeSystem, Cause: err} + } + pk[name] = true + } + _ = pkRows.Close() + if err := pkRows.Err(); err != nil { + return nil, &errs.Error{Op: "mysql.stream", Code: errs.CodeSystem, Cause: err} + } + + l := &tableLayout{cols: cols, pk: pk} + m.cache[table] = l + return l, nil +} + +// toChange converts a fully-parsed rowEvent into a CanonicalChange, mapping +// @N positions to column names and selecting key columns from the PK. INSERT +// uses the SET image for both Values and Key; UPDATE uses WHERE (old) for the +// key and SET (new) for Values; DELETE uses WHERE (old) for the key. +func (m *tableMetaCache) toChange(ev *rowEvent) (*canonical.CanonicalChange, error) { + l, err := m.layout(ev.table) + if err != nil { + return nil, err + } + nameOf := func(idx1 int) (string, bool) { // 1-based @N → name + if idx1 < 1 || idx1 > len(l.cols) { + return "", false + } + return l.cols[idx1-1], true + } + toMap := func(byPos map[int]string) map[string]any { + if byPos == nil { + return nil + } + out := make(map[string]any, len(byPos)) + for n, v := range byPos { + if name, ok := nameOf(n); ok { + out[name] = binlogValue(v) + } + } + return out + } + keyFrom := func(vals map[string]any) map[string]any { + key := map[string]any{} + for name := range vals { + if l.pk[name] { + key[name] = vals[name] + } + } + if len(key) == 0 { // no PK: fall back to the full row as the key + for k, v := range vals { + key[k] = v + } + } + return key + } + + switch ev.op { + case canonical.OpInsert: + vals := toMap(ev.setM) + return &canonical.CanonicalChange{Op: canonical.OpInsert, Table: ev.table, Key: keyFrom(vals), Values: vals}, nil + case canonical.OpUpdate: + vals := toMap(ev.setM) + old := toMap(ev.whereM) + return &canonical.CanonicalChange{Op: canonical.OpUpdate, Table: ev.table, Key: keyFrom(old), Values: vals}, nil + case canonical.OpDelete: + old := toMap(ev.whereM) + return &canonical.CanonicalChange{Op: canonical.OpDelete, Table: ev.table, Key: keyFrom(old)}, nil + default: + return nil, &errs.Error{Op: "mysql.stream", Code: errs.CodeSystem, Cause: errors.New("unknown row op")} + } +} + +// binlogValue maps the parsed string token to a Go value. A bare NULL token +// becomes nil; everything else stays a string (v1 keeps binlog values as text, +// matching the snapshot path's NormalizeScanned behavior). +func binlogValue(v string) any { + if v == "NULL" { + return nil + } + return v +} diff --git a/internal/driver/_mysqlcommon/changestream_test.go b/internal/driver/_mysqlcommon/changestream_test.go new file mode 100644 index 0000000..8626af4 --- /dev/null +++ b/internal/driver/_mysqlcommon/changestream_test.go @@ -0,0 +1,120 @@ +package mysqlcommon + +import ( + "reflect" + "strings" + "testing" + + "github.com/nixrajput/siphon/internal/canonical" +) + +func TestParseColAssign(t *testing.T) { + tests := []struct { + body string + wantN int + wantVal string + wantOK bool + }{ + {"@1=42", 1, "42", true}, + {"@2='wrench'", 2, "wrench", true}, + {"@3='a''b'", 3, "a'b", true}, + {"@4=NULL", 4, "NULL", true}, + {"@5='x' /* VARSTRING(60) meta=60 nullable=1 is_null=0 */", 5, "x", true}, + {"not an assign", 0, "", false}, + } + for _, tt := range tests { + n, val, ok := parseColAssign(tt.body) + if ok != tt.wantOK || n != tt.wantN || val != tt.wantVal { + t.Errorf("parseColAssign(%q) = (%d, %q, %v), want (%d, %q, %v)", + tt.body, n, val, ok, tt.wantN, tt.wantVal, tt.wantOK) + } + } +} + +func TestTableFromRef(t *testing.T) { + tests := map[string]string{ + "`test`.`widgets`": "widgets", + "test.widgets": "widgets", + "`widgets`": "widgets", + "`db`.`tbl` WHERE something": "tbl", + } + for in, want := range tests { + if got := tableFromRef(in); got != want { + t.Errorf("tableFromRef(%q) = %q, want %q", in, got, want) + } + } +} + +func TestParseAtMarker(t *testing.T) { + if p, ok := parseAtMarker("# at 1234"); !ok || p != 1234 { + t.Errorf("parseAtMarker = (%d,%v), want (1234,true)", p, ok) + } + if _, ok := parseAtMarker("### INSERT INTO x"); ok { + t.Error("parseAtMarker matched a ### line") + } +} + +// fakeRowsReader feeds a canned mysqlbinlog --verbose transcript through the +// parser. It builds CanonicalChanges WITHOUT a DB by pre-seeding the layout +// cache, so column names and PKs resolve offline. +func TestParseBinlogRows(t *testing.T) { + transcript := `# at 4 +#250101 12:00:00 server id 1 end_log_pos 120 Write_rows: table id 10 flags: STMT_END_F +### INSERT INTO ` + "`test`.`widgets`" + ` +### SET +### @1=1 +### @2='wrench' +# at 200 +#250101 12:00:01 server id 1 end_log_pos 280 Update_rows: table id 10 flags: STMT_END_F +### UPDATE ` + "`test`.`widgets`" + ` +### WHERE +### @1=1 +### @2='wrench' +### SET +### @1=1 +### @2='spanner' +# at 360 +#250101 12:00:02 server id 1 end_log_pos 420 Delete_rows: table id 10 flags: STMT_END_F +### DELETE FROM ` + "`test`.`widgets`" + ` +### WHERE +### @1=1 +### @2='spanner' +` + meta := &tableMetaCache{cache: map[string]*tableLayout{ + "widgets": {cols: []string{"id", "name"}, pk: map[string]bool{"id": true}}, + }} + + var got []canonical.CanonicalChange + emit := func(ch canonical.CanonicalChange) error { got = append(got, ch); return nil } + + pos, err := parseBinlogRows(strings.NewReader(transcript), meta, emit, BinlogPosition{File: "mysql-bin.000001"}) + if err != nil { + t.Fatalf("parseBinlogRows: %v", err) + } + if pos.Position != 360 { + t.Errorf("final pos = %d, want 360 (last # at marker)", pos.Position) + } + if len(got) != 3 { + t.Fatalf("got %d changes, want 3: %+v", len(got), got) + } + + want := []canonical.CanonicalChange{ + {Op: canonical.OpInsert, Table: "widgets", Key: map[string]any{"id": "1"}, Values: map[string]any{"id": "1", "name": "wrench"}}, + {Op: canonical.OpUpdate, Table: "widgets", Key: map[string]any{"id": "1"}, Values: map[string]any{"id": "1", "name": "spanner"}}, + {Op: canonical.OpDelete, Table: "widgets", Key: map[string]any{"id": "1"}}, + } + for i := range want { + if !reflect.DeepEqual(got[i], want[i]) { + t.Errorf("change[%d] = %+v, want %+v", i, got[i], want[i]) + } + } +} + +func TestBinlogValueNull(t *testing.T) { + if binlogValue("NULL") != nil { + t.Error("NULL should map to nil") + } + if binlogValue("x") != "x" { + t.Error("non-NULL should pass through") + } +} diff --git a/internal/driver/driver.go b/internal/driver/driver.go index c6fa841..6ebe230 100644 --- a/internal/driver/driver.go +++ b/internal/driver/driver.go @@ -44,6 +44,14 @@ type CanonicalTransfer interface { ApplyChange(ctx context.Context, ch canonical.CanonicalChange) error } +// ChangeStreamer is an optional Conn capability: stream logical row changes as +// engine-neutral CanonicalChanges, starting after `from`. Bounded callers cancel +// ctx at a target end position; unbounded (CDC) callers stream until ctx cancel. +// Returns the final Position reached (for envelope stamping / CDC state persistence). +type ChangeStreamer interface { + StreamChanges(ctx context.Context, from canonical.Position, emit func(canonical.CanonicalChange) error) (canonical.Position, error) +} + // Capabilities describes what an engine supports. Each flag gates a UI // affordance or feature path. Drivers must declare honestly. type Capabilities struct { diff --git a/internal/driver/postgres/changestream.go b/internal/driver/postgres/changestream.go new file mode 100644 index 0000000..8f85cdb --- /dev/null +++ b/internal/driver/postgres/changestream.go @@ -0,0 +1,394 @@ +package postgres + +import ( + "context" + "errors" + "strconv" + "strings" + "time" + + "github.com/jackc/pglogrepl" + "github.com/jackc/pgx/v5/pgconn" + "github.com/jackc/pgx/v5/pgproto3" + "github.com/jackc/pgx/v5/pgtype" + + "github.com/nixrajput/siphon/internal/canonical" + "github.com/nixrajput/siphon/internal/driver" + "github.com/nixrajput/siphon/internal/errs" +) + +// Stable, siphon-owned names for the publication and logical slot. Both are +// created on first use with IF-NOT-EXISTS semantics so repeated streaming +// (bounded incremental + unbounded CDC) reuses the same objects. +const ( + siphonPublication = "siphon_pub" + siphonSlot = "siphon_logical" + outputPlugin = "pgoutput" +) + +// standbyInterval bounds how long we block on a single ReceiveMessage and how +// often we ack the server. The server drops a logical replication connection +// that goes silent, so we must send a Standby Status Update at least this +// often even when no rows are flowing. +const standbyInterval = 10 * time.Second + +var _ driver.ChangeStreamer = (*Conn)(nil) + +// StreamChanges streams logical row changes as engine-neutral CanonicalChanges, +// starting after `from`. It uses pgoutput logical decoding over a dedicated +// replication-mode connection. Bounded callers cancel ctx at a target end +// position; unbounded (CDC) callers stream until ctx cancel. ctx cancellation +// is the normal stop signal and is NOT reported as an error — the final +// Position reached is returned for envelope stamping / CDC state persistence. +func (c *Conn) StreamChanges(ctx context.Context, from canonical.Position, emit func(canonical.CanonicalChange) error) (canonical.Position, error) { + if err := c.requireLogicalWAL(ctx); err != nil { + return canonical.Position{}, err + } + if err := c.ensurePublication(ctx); err != nil { + return canonical.Position{}, err + } + + repl, err := pgconn.Connect(ctx, c.replicationDSN()) + if err != nil { + return canonical.Position{}, &errs.Error{ + Op: "postgres.stream", + Code: errs.CodeSystem, + Cause: errs.ErrConnectionFailed, + Hint: "logical replication connect: " + err.Error(), + } + } + defer func() { _ = repl.Close(context.Background()) }() + + if err := ensureLogicalSlot(ctx, repl); err != nil { + return canonical.Position{}, err + } + + startLSN, err := resolveStartLSN(ctx, repl, from) + if err != nil { + return canonical.Position{}, err + } + + pluginArgs := []string{ + "proto_version '1'", + "publication_names '" + siphonPublication + "'", + } + if err := pglogrepl.StartReplication(ctx, repl, siphonSlot, startLSN, + pglogrepl.StartReplicationOptions{PluginArgs: pluginArgs}); err != nil { + return canonical.Position{}, &errs.Error{Op: "postgres.stream", Code: errs.CodeSystem, Cause: err} + } + + return receiveLoop(ctx, repl, startLSN, emit) +} + +// receiveLoop runs the pgoutput receive/decode/ack cycle until ctx is cancelled +// or emit returns an error. It returns the final committed LSN as a Position. +func receiveLoop(ctx context.Context, repl *pgconn.PgConn, startLSN pglogrepl.LSN, emit func(canonical.CanonicalChange) error) (canonical.Position, error) { + relations := map[uint32]*pglogrepl.RelationMessage{} + typeMap := pgtype.NewMap() + clientXLogPos := startLSN + nextDeadline := time.Now().Add(standbyInterval) + + finalPos := func() canonical.Position { return canonical.Position{LSN: clientXLogPos.String()} } + + for { + // ctx cancel is the normal bounded/unbounded stop signal: ack the last + // LSN best-effort and return cleanly. + if ctx.Err() != nil { + _ = pglogrepl.SendStandbyStatusUpdate(context.Background(), repl, + pglogrepl.StandbyStatusUpdate{WALWritePosition: clientXLogPos}) + return finalPos(), nil + } + + if time.Now().After(nextDeadline) { + if err := pglogrepl.SendStandbyStatusUpdate(ctx, repl, + pglogrepl.StandbyStatusUpdate{WALWritePosition: clientXLogPos}); err != nil { + return finalPos(), &errs.Error{Op: "postgres.stream", Code: errs.CodeSystem, Cause: err} + } + nextDeadline = time.Now().Add(standbyInterval) + } + + recvCtx, cancel := context.WithDeadline(ctx, nextDeadline) + rawMsg, err := repl.ReceiveMessage(recvCtx) + cancel() + if err != nil { + if pgconn.Timeout(err) { + continue // deadline hit with no data: loop to send a standby ack + } + if ctx.Err() != nil { + return finalPos(), nil // cancelled mid-receive: clean stop + } + return finalPos(), &errs.Error{Op: "postgres.stream", Code: errs.CodeSystem, Cause: err} + } + + if errMsg, ok := rawMsg.(*pgproto3.ErrorResponse); ok { + return finalPos(), &errs.Error{ + Op: "postgres.stream", + Code: errs.CodeSystem, + Cause: errors.New("replication stream error: " + + errMsg.Severity + " " + errMsg.Message), + } + } + + cd, ok := rawMsg.(*pgproto3.CopyData) + if !ok { + continue + } + + switch cd.Data[0] { + case pglogrepl.PrimaryKeepaliveMessageByteID: + pkm, err := pglogrepl.ParsePrimaryKeepaliveMessage(cd.Data[1:]) + if err != nil { + return finalPos(), &errs.Error{Op: "postgres.stream", Code: errs.CodeSystem, Cause: err} + } + if pkm.ServerWALEnd > clientXLogPos { + clientXLogPos = pkm.ServerWALEnd + } + if pkm.ReplyRequested { + nextDeadline = time.Time{} // force an immediate ack next iteration + } + + case pglogrepl.XLogDataByteID: + xld, err := pglogrepl.ParseXLogData(cd.Data[1:]) + if err != nil { + return finalPos(), &errs.Error{Op: "postgres.stream", Code: errs.CodeSystem, Cause: err} + } + if err := decodeWALData(xld.WALData, relations, typeMap, emit); err != nil { + return finalPos(), err + } + end := xld.WALStart + pglogrepl.LSN(len(xld.WALData)) + if end > clientXLogPos { + clientXLogPos = end + } + } + } +} + +// decodeWALData parses one pgoutput message and, for Insert/Update/Delete, +// builds a CanonicalChange and passes it to emit. Relation messages are cached +// so later DML can resolve column names and key flags. An emit error stops the +// stream and is propagated verbatim. +func decodeWALData(walData []byte, relations map[uint32]*pglogrepl.RelationMessage, typeMap *pgtype.Map, emit func(canonical.CanonicalChange) error) error { + msg, err := pglogrepl.Parse(walData) + if err != nil { + return &errs.Error{Op: "postgres.stream", Code: errs.CodeSystem, Cause: err} + } + + switch m := msg.(type) { + case *pglogrepl.RelationMessage: + relations[m.RelationID] = m + + case *pglogrepl.InsertMessage: + rel, ok := relations[m.RelationID] + if !ok { + return relErr(m.RelationID) + } + vals := tupleValues(rel, m.Tuple, typeMap) + return emit(canonical.CanonicalChange{ + Op: canonical.OpInsert, + Table: rel.RelationName, + Key: keyFromValues(rel, vals), + Values: vals, + }) + + case *pglogrepl.UpdateMessage: + rel, ok := relations[m.RelationID] + if !ok { + return relErr(m.RelationID) + } + vals := tupleValues(rel, m.NewTuple, typeMap) + // Prefer the old-tuple image for the key (REPLICA IDENTITY); fall back to + // the new image when the server sent no old tuple (key unchanged). + key := keyFromValues(rel, vals) + if m.OldTuple != nil { + if k := keyFromValues(rel, tupleValues(rel, m.OldTuple, typeMap)); len(k) > 0 { + key = k + } + } + return emit(canonical.CanonicalChange{ + Op: canonical.OpUpdate, + Table: rel.RelationName, + Key: key, + Values: vals, + }) + + case *pglogrepl.DeleteMessage: + rel, ok := relations[m.RelationID] + if !ok { + return relErr(m.RelationID) + } + old := tupleValues(rel, m.OldTuple, typeMap) + return emit(canonical.CanonicalChange{ + Op: canonical.OpDelete, + Table: rel.RelationName, + Key: keyFromValues(rel, old), + }) + } + // Begin/Commit/Type/Truncate/Origin/streaming markers carry no row change. + return nil +} + +func relErr(id uint32) error { + return &errs.Error{ + Op: "postgres.stream", + Code: errs.CodeSystem, + Cause: errors.New("pgoutput: unknown relation ID " + strconv.FormatUint(uint64(id), 10)), + Hint: "a DML message arrived before its relation definition", + } +} + +// tupleValues decodes a pgoutput tuple into a column-name → Go-value map. NULL +// columns map to nil; unchanged-TOAST columns ('u') are omitted (their value +// was not transmitted). Text-format columns are decoded via the pgtype map for +// the column's OID, falling back to the raw string. +func tupleValues(rel *pglogrepl.RelationMessage, tuple *pglogrepl.TupleData, typeMap *pgtype.Map) map[string]any { + if tuple == nil { + return nil + } + out := make(map[string]any, len(tuple.Columns)) + for i, col := range tuple.Columns { + if i >= len(rel.Columns) { + break + } + name := rel.Columns[i].Name + switch col.DataType { + case 'n': // null + out[name] = nil + case 'u': // unchanged TOAST: value not sent, leave it out + continue + case 't': // text + out[name] = decodeText(typeMap, col.Data, rel.Columns[i].DataType) + default: // binary ('b') or unexpected: keep the raw bytes as a string + out[name] = string(col.Data) + } + } + return out +} + +// decodeText decodes a text-format column using the pgtype codec for its OID, +// returning the original string when the OID is unknown or decoding fails. +func decodeText(typeMap *pgtype.Map, data []byte, oid uint32) any { + if dt, ok := typeMap.TypeForOID(oid); ok { + if v, err := dt.Codec.DecodeValue(typeMap, oid, pgtype.TextFormatCode, data); err == nil { + return v + } + } + return string(data) +} + +// keyFromValues extracts the primary-key columns (RelationMessageColumn.Flags +// bit 1 = part of the key, set under REPLICA IDENTITY DEFAULT/USING INDEX) from +// a decoded value map. When the relation declares no key columns (e.g. REPLICA +// IDENTITY FULL with no PK), it falls back to the full value set so UPDATE and +// DELETE still have something to target. +func keyFromValues(rel *pglogrepl.RelationMessage, vals map[string]any) map[string]any { + key := map[string]any{} + hasKeyCol := false + for _, col := range rel.Columns { + if col.Flags&1 == 1 { + hasKeyCol = true + if v, ok := vals[col.Name]; ok { + key[col.Name] = v + } + } + } + if !hasKeyCol { + // REPLICA IDENTITY FULL / no declared key: use every column as the key. + full := make(map[string]any, len(vals)) + for k, v := range vals { + full[k] = v + } + return full + } + return key +} + +// requireLogicalWAL fails fast with an actionable hint when the server is not +// configured for logical decoding (wal_level must be 'logical'). +func (c *Conn) requireLogicalWAL(ctx context.Context) error { + var level string + if err := c.db.QueryRowContext(ctx, "SHOW wal_level").Scan(&level); err != nil { + return &errs.Error{Op: "postgres.stream", Code: errs.CodeSystem, Cause: err} + } + if !strings.EqualFold(level, "logical") { + return &errs.Error{ + Op: "postgres.stream", + Code: errs.CodeUser, + Cause: errors.New("wal_level is " + level), + Hint: "set wal_level = logical in postgresql.conf and restart the server", + } + } + return nil +} + +// ensurePublication creates the siphon publication FOR ALL TABLES if it does +// not already exist. CREATE PUBLICATION has no IF NOT EXISTS clause (pre-PG13), +// so we query pg_publication first and skip creation when present. +func (c *Conn) ensurePublication(ctx context.Context) error { + var exists bool + if err := c.db.QueryRowContext(ctx, + "SELECT EXISTS (SELECT 1 FROM pg_publication WHERE pubname = $1)", siphonPublication).Scan(&exists); err != nil { + return &errs.Error{Op: "postgres.stream", Code: errs.CodeSystem, Cause: err} + } + if exists { + return nil + } + // The publication name is a fixed constant, so direct interpolation is safe. + if _, err := c.db.ExecContext(ctx, "CREATE PUBLICATION "+siphonPublication+" FOR ALL TABLES"); err != nil { + // Tolerate a concurrent creator (duplicate_object, SQLSTATE 42710). + if strings.Contains(err.Error(), "already exists") { + return nil + } + return &errs.Error{ + Op: "postgres.stream", + Code: errs.CodeUser, + Cause: err, + Hint: "creating publication requires a superuser or a role with CREATE on the database", + } + } + return nil +} + +// ensureLogicalSlot creates the pgoutput logical slot on the replication +// connection if absent. CreateReplicationSlot errors with "already exists" when +// the slot is present; that is tolerated so repeated streams reuse the slot. +func ensureLogicalSlot(ctx context.Context, repl *pgconn.PgConn) error { + _, err := pglogrepl.CreateReplicationSlot(ctx, repl, siphonSlot, outputPlugin, + pglogrepl.CreateReplicationSlotOptions{Temporary: false}) + if err != nil { + if strings.Contains(err.Error(), "already exists") { + return nil + } + return &errs.Error{Op: "postgres.stream", Code: errs.CodeSystem, Cause: err} + } + return nil +} + +// resolveStartLSN returns the LSN to start replication from. A non-empty +// from.LSN is parsed and used verbatim; otherwise we start from the server's +// current WAL position via IdentifySystem. +func resolveStartLSN(ctx context.Context, repl *pgconn.PgConn, from canonical.Position) (pglogrepl.LSN, error) { + if from.LSN != "" { + lsn, err := pglogrepl.ParseLSN(from.LSN) + if err != nil { + return 0, &errs.Error{ + Op: "postgres.stream", + Code: errs.CodeUser, + Cause: err, + Hint: "invalid start LSN " + from.LSN, + } + } + return lsn, nil + } + sys, err := pglogrepl.IdentifySystem(ctx, repl) + if err != nil { + return 0, &errs.Error{Op: "postgres.stream", Code: errs.CodeSystem, Cause: err} + } + return sys.XLogPos, nil +} + +// replicationDSN builds a libpq keyword DSN like buildDSN but adds +// replication=database, which pgconn requires to open a logical replication +// connection (a plain connection rejects START_REPLICATION). +func (c *Conn) replicationDSN() string { + return buildDSN(c.p) + " replication=database" +} diff --git a/internal/driver/postgres/changestream_integration_test.go b/internal/driver/postgres/changestream_integration_test.go new file mode 100644 index 0000000..fbb8edd --- /dev/null +++ b/internal/driver/postgres/changestream_integration_test.go @@ -0,0 +1,139 @@ +//go:build integration + +package postgres + +import ( + "context" + "database/sql" + "sync" + "testing" + "time" + + tc "github.com/testcontainers/testcontainers-go" + pgctr "github.com/testcontainers/testcontainers-go/modules/postgres" + + "github.com/nixrajput/siphon/internal/canonical" + "github.com/nixrajput/siphon/internal/driver" +) + +// startLogicalPostgres starts a Postgres container configured for logical +// decoding (wal_level=logical), the prerequisite for StreamChanges. +func startLogicalPostgres(t *testing.T) (driver.Profile, func()) { + t.Helper() + ctx := context.Background() + c, err := pgctr.Run(ctx, "postgres:16-alpine", + pgctr.WithDatabase("test"), + pgctr.WithUsername("postgres"), + pgctr.WithPassword("postgres"), + pgctr.BasicWaitStrategies(), + tc.WithCmdArgs("-c", "wal_level=logical", "-c", "max_wal_senders=10", "-c", "max_replication_slots=10"), + ) + if err != nil { + t.Fatalf("start postgres container: %v", err) + } + host, err := c.Host(ctx) + if err != nil { + t.Fatalf("container host: %v", err) + } + port, err := c.MappedPort(ctx, "5432/tcp") + if err != nil { + t.Fatalf("container mapped port: %v", err) + } + prof := driver.Profile{ + Driver: "postgres", + Host: host, + Port: int(port.Num()), + User: "postgres", + Password: "postgres", + Database: "test", + SSLMode: "disable", + } + return prof, func() { _ = c.Terminate(ctx) } +} + +// TestStreamChanges_Bounded seeds a table, captures the current LSN, performs +// insert/update/delete, then streams changes bounded to those three events +// (emit cancels ctx after the third). It asserts op + table + key/values. +func TestStreamChanges_Bounded(t *testing.T) { + prof, cleanup := startLogicalPostgres(t) + defer cleanup() + + ctx := context.Background() + db, err := sql.Open("pgx", buildDSN(prof)) + if err != nil { + t.Fatalf("open db: %v", err) + } + defer func() { _ = db.Close() }() + + if _, err := db.ExecContext(ctx, + `CREATE TABLE widgets(id int primary key, name text);`); err != nil { + t.Fatalf("create table: %v", err) + } + + conn := &Conn{db: db, p: prof} + + // Capture the start LSN before any DML so the stream sees exactly our changes. + var startLSN string + if err := db.QueryRowContext(ctx, "SELECT pg_current_wal_lsn()::text").Scan(&startLSN); err != nil { + t.Fatalf("capture lsn: %v", err) + } + // The publication+slot must exist before the DML so the slot retains the WAL. + if err := conn.ensurePublication(ctx); err != nil { + t.Fatalf("ensure publication: %v", err) + } + + // Apply the three operations. + if _, err := db.ExecContext(ctx, `INSERT INTO widgets VALUES (1,'wrench')`); err != nil { + t.Fatalf("insert: %v", err) + } + if _, err := db.ExecContext(ctx, `UPDATE widgets SET name='spanner' WHERE id=1`); err != nil { + t.Fatalf("update: %v", err) + } + if _, err := db.ExecContext(ctx, `DELETE FROM widgets WHERE id=1`); err != nil { + t.Fatalf("delete: %v", err) + } + + streamCtx, cancel := context.WithTimeout(ctx, 30*time.Second) + defer cancel() + + var mu sync.Mutex + var got []canonical.CanonicalChange + emit := func(ch canonical.CanonicalChange) error { + mu.Lock() + got = append(got, ch) + n := len(got) + mu.Unlock() + if n >= 3 { + cancel() // bounded: stop after the three expected changes + } + return nil + } + + if _, err := conn.StreamChanges(streamCtx, canonical.Position{LSN: startLSN}, emit); err != nil { + t.Fatalf("StreamChanges: %v", err) + } + + if len(got) < 3 { + t.Fatalf("expected at least 3 changes, got %d: %+v", len(got), got) + } + + ins, upd, del := got[0], got[1], got[2] + if ins.Op != canonical.OpInsert || ins.Table != "widgets" { + t.Errorf("change[0] = %+v, want insert into widgets", ins) + } + if ins.Values["name"] != "wrench" { + t.Errorf("insert values name = %v, want wrench", ins.Values["name"]) + } + if upd.Op != canonical.OpUpdate || upd.Values["name"] != "spanner" { + t.Errorf("change[1] = %+v, want update name=spanner", upd) + } + if del.Op != canonical.OpDelete { + t.Errorf("change[2] = %+v, want delete", del) + } + // id is the PK; every op must carry it in Key. + for i, ch := range []canonical.CanonicalChange{ins, upd, del} { + if _, ok := ch.Key["id"]; !ok { + t.Errorf("change[%d] key missing id: %+v", i, ch.Key) + } + } +} From 93780aa12360cce3d8af075380a4db6af2004e46 Mon Sep 17 00:00:00 2001 From: Nikhil Rajput Date: Tue, 23 Jun 2026 03:44:21 +0530 Subject: [PATCH 07/13] feat: wire incremental backup and change replay - Add driver.IncrementalBackuper: bounded change capture from a base position to the engine's current end, emitted as JSONL CanonicalChange records. Postgres bounds the pgoutput stream by pg_current_wal_lsn(); MySQL/MariaDB by the current binlog offset. - Wire app.Backup --incremental: read the base envelope end position, capture to a temp body, then write envelope(end pos) + body and a Meta linking BaseID/ParentID. - Replay incremental restore links via ApplyChange; base links still restore natively. - Add Postgres SweepOrphanSlots: drop inactive siphon_ physical slots, preserving the persistent logical resume slot. - Un-gate the CLI and flip the Postgres Incremental capability. - Update restore-chain tests, README, CHANGELOG, INCREMENTAL.md. --- CHANGELOG.md | 4 +- README.md | 6 +- docs/INCREMENTAL.md | 72 +++++--- internal/app/backup.go | 166 ++++++++++++++++++ internal/app/restore.go | 50 ++++++ internal/app/restore_chain_test.go | 91 +++++++--- internal/cli/backup.go | 23 +-- internal/driver/_mysqlcommon/changestream.go | 28 ++- .../driver/_mysqlcommon/changestream_test.go | 2 +- internal/driver/_mysqlcommon/incremental.go | 43 +++-- internal/driver/driver.go | 14 ++ internal/driver/postgres/changestream.go | 28 ++- internal/driver/postgres/driver.go | 2 +- internal/driver/postgres/incremental.go | 80 --------- .../driver/postgres/incremental_change.go | 100 +++++++++++ .../postgres/incremental_integration_test.go | 139 +++++++++++++++ 16 files changed, 667 insertions(+), 181 deletions(-) create mode 100644 internal/driver/postgres/incremental_change.go create mode 100644 internal/driver/postgres/incremental_integration_test.go diff --git a/CHANGELOG.md b/CHANGELOG.md index f462a08..8d9837a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -24,5 +24,5 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - CI installs the Postgres/MySQL/MariaDB client tools and runs the full integration suite on the Ubuntu runner (where Docker is available); unit tests continue to run on Linux/macOS/Windows. - Phase F advanced-transfer machinery (some CLI paths gated pending follow-up wiring — see the `docs/INCREMENTAL.md`, `docs/CROSS_ENGINE.md`, `docs/CDC.md` Status sections): - **Working today:** every dump is prefixed with a 4 KB `SIPH` JSON envelope (type, base/parent IDs, WAL/binlog positions, checksum); `restore` resolves the base→incremental chain and applies it in order, with `--up-to ` to stop early (cycle + broken-chain detection). Sync now streams through a bounded `jobs.Stream` (default 64×1 MB) instead of `io.Pipe`, exposing a `FillPercent` backpressure metric and propagating backup failures to the restore side via `CloseErr` (no truncated dump committed as clean). - - **Machinery in place, CLI gated (follow-up):** incremental backup capture (Postgres WAL replication slot + LSN; MySQL/MariaDB binlog position) — `backup --incremental` returns a clear "not yet wired" error; cross-engine type-mapping (canonical schema + JSONL emit/consume with per-engine identifier quoting and placeholders) — `sync --cross-engine` is capability-gated off until typed schema introspection exists; CDC continuous mode (state-file persistence + resume + capability gating) ships as a polling scaffold, not real logical-replication streaming. - - Known follow-up gates before incremental backup ships to users: Postgres orphan-replication-slot cleanup, validation of the `pg_receivewal`/`mysqlbinlog` streaming invocations against live servers (compile-checked only here). + - **Incremental backup wired end-to-end (`backup --incremental --base `):** captures a **bounded** change set since the base dump's recorded end position, serialized as engine-neutral JSONL `CanonicalChange` records. Postgres bounds the pgoutput logical-decoding stream by `pg_current_wal_lsn()` captured at backup time; MySQL/MariaDB bound the binlog-tool decode by the current binlog file+offset. The incremental dump's envelope carries this capture's end position so the next incremental resumes exactly there. `restore` replays incremental links via `ApplyChange` (base links still restore natively). Postgres adds an orphan replication-slot sweep (`SweepOrphanSlots`) run before each capture — drops inactive `siphon_*` physical slots while preserving the persistent logical resume slot. `Incremental` capability is now `true` for all three drivers. Live-server behavior is integration-tested in CI (`wal_level=logical`); compile-checked locally. + - **Machinery in place, CLI gated (follow-up):** cross-engine type-mapping (canonical schema + JSONL emit/consume with per-engine identifier quoting and placeholders) — `sync --cross-engine` is capability-gated off until typed schema introspection exists; CDC continuous mode (state-file persistence + resume + capability gating) ships as a polling scaffold, not real logical-replication streaming. diff --git a/README.md b/README.md index fdd2840..8e00469 100644 --- a/README.md +++ b/README.md @@ -17,7 +17,7 @@ A single binary that turns the painful, error-prone sprawl of `pg_dump` → `pg_ --- > [!WARNING] -> **Pre-1.0 — active development.** Postgres, MySQL, and MariaDB backup/restore/sync/verify/inspect work end-to-end today (Phases B + E), and bare `siphon` opens an interactive multi-panel dashboard (Phase C). The driver layer is hardened with a shared cross-driver test harness, capability gating, and connection retry (Phase D). Incremental backups, cross-engine sync, and ops features are on the [roadmap](#roadmap). APIs, flags, and the on-disk dump format may change before 1.0. Track progress via the milestone tags (`phase-a`, `phase-b`, `phase-c`, `phase-d`, `phase-e`, …). +> **Pre-1.0 — active development.** Postgres, MySQL, and MariaDB backup/restore/sync/verify/inspect work end-to-end today (Phases B + E), and bare `siphon` opens an interactive multi-panel dashboard (Phase C). The driver layer is hardened with a shared cross-driver test harness, capability gating, and connection retry (Phase D). Incremental backup + restore work end-to-end (Phase F); cross-engine sync and ops features are on the [roadmap](#roadmap). APIs, flags, and the on-disk dump format may change before 1.0. Track progress via the milestone tags (`phase-a`, `phase-b`, `phase-c`, `phase-d`, `phase-e`, …). ## Table of contents @@ -54,7 +54,7 @@ A single binary that turns the painful, error-prone sprawl of `pg_dump` → `pg_ | **C** — TUI dashboard | Multi-panel Bubble Tea dashboard (profiles · dumps · jobs) with live job progress, backup/restore modal forms, and snapshot tests | ✅ Complete | | **D** — Driver hardening | Shared cross-driver test harness (`RunDriverSuite`), capability-gating helper (`RequireCapability`), Postgres connection-probe retry, and a `docs/DRIVERS.md` contributor guide | ✅ Complete | | **E** — MySQL + MariaDB | Both drivers via a shared `_mysqlcommon` package (shared `Conn`, `mysqldump`/`mariadb-dump` backup, client-pipe restore), exercised by the Phase D `RunDriverSuite` harness | ✅ Complete | -| **F** — Advanced transfer | Bounded-buffer streaming sync + chain-walking incremental restore work today; the dump envelope, incremental WAL/binlog capture, cross-engine type-mapping, and CDC state machinery are in place but their CLI paths (`backup --incremental`, `sync --cross-engine`, CDC) are gated pending follow-up wiring — see [docs/INCREMENTAL.md](docs/INCREMENTAL.md), [docs/CROSS_ENGINE.md](docs/CROSS_ENGINE.md), [docs/CDC.md](docs/CDC.md) | 🟡 Partial | +| **F** — Advanced transfer | Bounded-buffer streaming sync and incremental backup/restore work today: `backup --incremental --base ` captures a bounded change set (Postgres logical decoding / MySQL-MariaDB binlog) and `restore` replays the base→incremental chain, with Postgres orphan replication-slot sweep. Cross-engine type-mapping and CDC machinery are in place but their CLI paths (`sync --cross-engine`, CDC) remain gated pending follow-up wiring — see [docs/INCREMENTAL.md](docs/INCREMENTAL.md), [docs/CROSS_ENGINE.md](docs/CROSS_ENGINE.md), [docs/CDC.md](docs/CDC.md) | 🟡 Partial | | **G** — Ops features | Cloud storage, secret backends, profile groups + 2FA, team mode, audit log, retention, telemetry | ⏳ Planned | | **H** — Distribution | GoReleaser, Homebrew tap, Scoop bucket, install script, docs site | ⏳ Planned | @@ -222,7 +222,7 @@ Contributions are welcome. Please read [CONTRIBUTING.md](CONTRIBUTING.md), keep Adding a new database engine? See [docs/DRIVERS.md](docs/DRIVERS.md) for the driver contributor guide. -Concept docs (honest as-built status — several paths are documented follow-ups): [docs/INCREMENTAL.md](docs/INCREMENTAL.md) (incremental restore works; the backup path is a follow-up), [docs/CROSS_ENGINE.md](docs/CROSS_ENGINE.md) (type-map matrix; `--cross-engine` is gated off pending typed introspection), and [docs/CDC.md](docs/CDC.md) (scaffold only; not yet runnable). +Concept docs (honest as-built status — several paths are documented follow-ups): [docs/INCREMENTAL.md](docs/INCREMENTAL.md) (incremental backup + restore work end-to-end), [docs/CROSS_ENGINE.md](docs/CROSS_ENGINE.md) (type-map matrix; `--cross-engine` is gated off pending typed introspection), and [docs/CDC.md](docs/CDC.md) (scaffold only; not yet runnable). ## License diff --git a/docs/INCREMENTAL.md b/docs/INCREMENTAL.md index ca717b7..80caea9 100644 --- a/docs/INCREMENTAL.md +++ b/docs/INCREMENTAL.md @@ -27,14 +27,26 @@ looping or silently truncating. `Restore` then applies the resolved chain in order, base first (`internal/app/restore.go`). A plain, non-incremental dump resolves to a single-element chain, so the same restore path serves both. -The driver-level capture machinery also exists: - -- **Postgres** (`internal/driver/postgres/incremental.go`) creates a temporary - physical replication slot and records the start/end LSN around a base backup, - so a later incremental can resume from the correct WAL position. +An incremental backup is a **bounded change capture**: starting from the base +dump's recorded end position, siphon streams the row changes that committed since +then up to a fixed end position captured at backup time, and serializes each as a +JSONL `CanonicalChange` (insert/update/delete with primary key + post-image). The +incremental dump body is therefore engine-neutral change records, not raw +WAL/binlog bytes. At restore time those changes are **replayed** via +`ApplyChange` rather than fed to the native restore tool — base links restore +natively, incremental links replay change records. + +The driver-level capture (`driver.IncrementalBackuper`): + +- **Postgres** (`internal/driver/postgres/incremental_change.go`) captures the + current `pg_current_wal_lsn()` as the end bound, then drives the same pgoutput + logical-decoding loop as CDC with that LSN as a stop target — it returns + cleanly at the first message boundary past the bound, so every change committed + at or before it is captured and none after. - **MySQL/MariaDB** (`internal/driver/_mysqlcommon/incremental.go`) captures the - binlog file + position via `SHOW BINARY LOG STATUS` (MySQL 8.4+) or - `SHOW MASTER STATUS` (older MySQL / MariaDB). + current binlog file + offset via `SHOW BINARY LOG STATUS` (MySQL 8.4+) or + `SHOW MASTER STATUS` (older MySQL / MariaDB) as the end bound, then decodes the + fork's binlog tool output up to that offset. ## The CLI surface @@ -45,10 +57,13 @@ siphon restore --profile # Stop applying the chain after a specific dump (point-in-chain restore). siphon restore --profile --up-to -# Request an incremental backup (NOT yet wired — see Status). +# Take an incremental backup capturing changes since a base dump. siphon backup --incremental --base ``` +`--incremental` requires `--base `; `--base` without `--incremental` (or +`--incremental` without `--base`) is rejected with a clear error. + ## Status | Capability | Status | @@ -57,17 +72,17 @@ siphon backup --incremental --base | Chain resolution (`ResolveChain`) | ✅ Works | | Chain-walking restore (base → incrementals, in order) | ✅ Works | | `restore --up-to ` (stop chain early) | ✅ Works | -| Driver-level capture machinery (Postgres WAL slot/LSN, MySQL/MariaDB binlog pos) | ✅ Exists | -| `backup --incremental --base ` end-to-end | ⚠️ Not yet wired (follow-up) | - -The restore-side chain machinery and the envelope are in place and tested. The -`--incremental` backup path is a documented follow-up: the driver machinery and -the chain restore are both ready, but the wiring that ties them together — read -the base envelope for the parent WAL/binlog position, invoke the driver's -incremental capture method, and write an `incremental`-type catalog entry — is -not yet connected. Until then `siphon backup --incremental` returns a clear -error: *"incremental backup is not yet wired end-to-end (Phase F follow-up); the -driver-level machinery exists"* (`internal/cli/backup.go`). +| `backup --incremental --base ` (bounded change capture) | ✅ Works | +| Incremental restore (change replay via `ApplyChange`) | ✅ Works | +| Postgres orphan replication-slot sweep | ✅ Works | + +The full incremental path is wired end-to-end: `backup --incremental` reads the +base envelope's end position, captures the bounded change set via the driver's +`IncrementalBackuper`, and writes an `incremental`-type catalog entry whose +envelope carries this capture's end position (so the next incremental resumes +exactly here). Restore replays each incremental link's changes via `ApplyChange`. +The live-server behavior is exercised in CI (integration-tagged tests against a +`wal_level=logical` Postgres); it is compile-checked but not run locally. ## Examples @@ -88,21 +103,26 @@ siphon restore inc-2 --profile prod-replica --up-to inc-1 A typo'd `--up-to` is rejected (the dump isn't in the chain) rather than silently restoring more than asked. -Requesting an incremental backup (returns the not-wired error today): +Taking an incremental backup against a base dump: ```bash siphon backup prod --incremental --base base-0 -# Error: incremental backup is not yet wired end-to-end (Phase F follow-up); -# the driver-level machinery exists +# Captures changes committed since base-0's end position and writes a new +# incremental dump linked to base-0. Restoring it later replays base-0 then the +# captured changes. ``` ## Limitations and runtime gates -These apply to the incremental **backup** path once it is wired: +These apply to the incremental **backup** path: -- **Postgres** anchors WAL retention with a temporary physical replication slot. - Orphan-slot cleanup is **not yet built** — an orphaned slot pins WAL on the - server and must be dropped manually until automatic cleanup ships. +- **Postgres** uses a persistent logical replication slot (`siphon_logical`) as + the change-stream resume anchor. Per-base physical slots are swept + automatically: before each incremental capture, `SweepOrphanSlots` drops any + inactive `siphon_*` physical slot (an inactive siphon slot is by definition + orphaned — a completed backup drops its own, an active one is in use). The + persistent logical slot is excluded from the sweep so the resume position + survives between runs. - **MySQL/MariaDB** require `binlog_format=ROW` for usable incrementals. - **Cross-version incrementals are unsupported** (`CrossVersionIncremental: false`): a chain must be captured and restored against the same engine major diff --git a/internal/app/backup.go b/internal/app/backup.go index 3751879..1f3635e 100644 --- a/internal/app/backup.go +++ b/internal/app/backup.go @@ -13,6 +13,7 @@ import ( "github.com/oklog/ulid/v2" + "github.com/nixrajput/siphon/internal/canonical" "github.com/nixrajput/siphon/internal/driver" "github.com/nixrajput/siphon/internal/dumps" "github.com/nixrajput/siphon/internal/errs" @@ -44,6 +45,8 @@ type BackupOpts struct { DataOnly bool CompressionLevel int Parallel int + Incremental bool + BaseID string } // Backup dumps the source profile to a new entry in the catalog. @@ -67,6 +70,10 @@ func Backup(parent context.Context, d Deps, opt BackupOpts) (<-chan jobs.Event, } defer func() { _ = conn.Close() }() + if opt.Incremental { + return runIncrementalBackup(ctx, d, opt, resolved, conn, emit) + } + id := ulid.Make().String() tmpPath := filepath.Join(d.Dumps.Root(), id+".dump.tmp") finalPath := d.Dumps.Path(id) @@ -149,3 +156,162 @@ func Backup(parent context.Context, d Deps, opt BackupOpts) (<-chan jobs.Event, }, }) } + +// runIncrementalBackup captures the bounded change set since the base dump's end +// position and writes a new incremental dump linked to that base. +// +// Envelope write ordering: the incremental envelope must carry the END position +// of THIS capture (so the next incremental resumes from here), but that position +// is only known AFTER streaming. We therefore stream the JSONL change body into a +// temp file first, learn the end Position, then assemble the final dump as +// envelope(end position) followed by the temp body — checksumming the whole file +// as it is written, mirroring the full-backup tmp→rename + sha + WriteMeta flow. +func runIncrementalBackup(ctx context.Context, d Deps, opt BackupOpts, resolved driver.Profile, conn driver.Conn, emit func(jobs.Event)) error { + base, err := d.Dumps.ReadMeta(opt.BaseID) + if err != nil { + return &errs.Error{ + Op: "app.backup.incremental", + Code: errs.CodeUser, + Cause: err, + Hint: "base dump " + opt.BaseID + " not found in the catalog", + } + } + + since, err := basePosition(d, opt.BaseID) + if err != nil { + return err + } + + inc, ok := conn.(driver.IncrementalBackuper) + if !ok { + return &errs.Error{ + Op: "app.backup.incremental", + Code: errs.CodeUser, + Cause: errs.ErrDriverUnsupported, + Hint: resolved.Driver + " does not support incremental backup", + } + } + + // Stream the change body to a temp file so we learn the end Position before + // writing the envelope (which must carry that end Position). + bodyTmp, err := os.CreateTemp(d.Dumps.Root(), "siphon-inc-body-*") + if err != nil { + return err + } + bodyPath := bodyTmp.Name() + defer func() { _ = os.Remove(bodyPath) }() + + emit(jobs.Event{Phase: jobs.PhaseProgress, Message: "capturing changes since base"}) + endPos, capErr := inc.BackupIncremental(ctx, since, bodyTmp) + closeErr := bodyTmp.Close() + if capErr != nil { + return capErr + } + if closeErr != nil { + return &errs.Error{Op: "app.backup.incremental", Code: errs.CodeSystem, Cause: closeErr, Hint: "failed to flush change body to disk"} + } + + // Assemble the final dump: envelope(end position) + change body. + id := ulid.Make().String() + tmpPath := filepath.Join(d.Dumps.Root(), id+".dump.tmp") + finalPath := d.Dumps.Path(id) + + f, err := os.Create(tmpPath) + if err != nil { + return err + } + h := sha256.New() + tee := io.MultiWriter(f, h) + + // The chain root: if the base is itself an incremental, inherit its BaseID; + // otherwise the base IS the root. + root := base.BaseID + if root == "" { + root = base.ID + } + env := &dumps.Envelope{ + Type: dumps.EnvelopeIncremental, + Driver: resolved.Driver, + BaseID: root, + ParentID: opt.BaseID, + WALEnd: endPos.LSN, + Created: time.Now().UTC(), + } + if endPos.BinlogFile != "" { + env.BinlogFile = endPos.BinlogFile + env.BinlogEnd = endPos.BinlogPos + } + if _, err := dumps.WriteEnvelope(tee, env); err != nil { + _ = f.Close() + _ = os.Remove(tmpPath) + return err + } + + body, err := os.Open(bodyPath) + if err != nil { + _ = f.Close() + _ = os.Remove(tmpPath) + return err + } + if _, err := io.Copy(tee, body); err != nil { + _ = body.Close() + _ = f.Close() + _ = os.Remove(tmpPath) + return err + } + _ = body.Close() + + closeErr = f.Close() + if closeErr != nil { + _ = os.Remove(tmpPath) + return &errs.Error{Op: "app.backup.incremental", Code: errs.CodeSystem, Cause: closeErr, Hint: "failed to flush dump to disk (out of space?)"} + } + if err := os.Rename(tmpPath, finalPath); err != nil { + _ = os.Remove(tmpPath) + return err + } + + st, _ := os.Stat(finalPath) + size := int64(0) + if st != nil { + size = st.Size() + } + meta := &dumps.Meta{ + ID: id, + Profile: opt.Profile, + Driver: resolved.Driver, + SizeBytes: size, + Checksum: "sha256:" + hex.EncodeToString(h.Sum(nil)), + Created: time.Now(), + DumpFormat: "jsonl-changes", + BaseID: root, + ParentID: opt.BaseID, + } + if writeErr := d.Dumps.WriteMeta(meta); writeErr != nil { + _ = os.Remove(finalPath) + return writeErr + } + + emit(jobs.Event{Phase: jobs.PhaseProgress, Message: "wrote " + finalPath}) + return nil +} + +// basePosition reads the end Position recorded in the base dump's envelope. The +// next incremental resumes from here: WALEnd for Postgres, BinlogFile+BinlogEnd +// for MySQL/MariaDB. +func basePosition(d Deps, baseID string) (canonical.Position, error) { + f, err := os.Open(d.Dumps.Path(baseID)) + if err != nil { + return canonical.Position{}, err + } + defer func() { _ = f.Close() }() + env, _, err := dumps.ReadEnvelope(f) + if err != nil { + return canonical.Position{}, err + } + return canonical.Position{ + LSN: env.WALEnd, + BinlogFile: env.BinlogFile, + BinlogPos: env.BinlogEnd, + }, nil +} diff --git a/internal/app/restore.go b/internal/app/restore.go index fe7ba89..b7dc148 100644 --- a/internal/app/restore.go +++ b/internal/app/restore.go @@ -1,9 +1,13 @@ package app import ( + "bufio" "context" + "encoding/json" + "io" "os" + "github.com/nixrajput/siphon/internal/canonical" "github.com/nixrajput/siphon/internal/driver" "github.com/nixrajput/siphon/internal/dumps" "github.com/nixrajput/siphon/internal/errs" @@ -78,6 +82,27 @@ func Restore(parent context.Context, d Deps, opt RestoreOpts) (<-chan jobs.Event Hint: "dump was created by " + env.Driver + "; cannot restore into a " + resolved.Driver + " target", } } + // Incremental links carry a JSONL change body, not a native dump: + // replay each change via ApplyChange instead of conn.Restore. + if env.Type == dumps.EnvelopeIncremental { + applier, ok := conn.(driver.CanonicalTransfer) + if !ok { + _ = f.Close() + return &errs.Error{ + Op: "restore", + Code: errs.CodeUser, + Cause: errs.ErrDriverUnsupported, + Hint: resolved.Driver + " cannot replay incremental change links", + } + } + if err := replayChanges(ctx, applier, body); err != nil { + _ = f.Close() + return err + } + _ = f.Close() + continue + } + rOpts := driver.RestoreOpts{ TargetTables: opt.TargetTables, SchemaOnly: opt.SchemaOnly, @@ -95,6 +120,31 @@ func Restore(parent context.Context, d Deps, opt RestoreOpts) (<-chan jobs.Event }) } +// replayChanges decodes a JSONL stream of CanonicalChanges from r and applies +// each to the target via ApplyChange, in order. A malformed line aborts the +// replay so a corrupt incremental never partially applies undetected. +func replayChanges(ctx context.Context, applier driver.CanonicalTransfer, r io.Reader) error { + sc := bufio.NewScanner(r) + sc.Buffer(make([]byte, 0, 64*1024), 8*1024*1024) + for sc.Scan() { + line := sc.Bytes() + if len(line) == 0 { + continue + } + var ch canonical.CanonicalChange + if err := json.Unmarshal(line, &ch); err != nil { + return &errs.Error{Op: "restore.replay", Code: errs.CodeSystem, Cause: err, Hint: "incremental change body is corrupt"} + } + if err := applier.ApplyChange(ctx, ch); err != nil { + return err + } + } + if err := sc.Err(); err != nil { + return &errs.Error{Op: "restore.replay", Code: errs.CodeSystem, Cause: err} + } + return nil +} + // truncateChain returns chain up to and including the dump named upTo. Unlike // silently applying the full chain, an unknown upTo is an error so a typo'd // --up-to surfaces instead of restoring more than the user asked for. diff --git a/internal/app/restore_chain_test.go b/internal/app/restore_chain_test.go index 1dd43d7..54d9605 100644 --- a/internal/app/restore_chain_test.go +++ b/internal/app/restore_chain_test.go @@ -2,12 +2,14 @@ package app import ( "context" + "encoding/json" "errors" "io" "os" "testing" "time" + "github.com/nixrajput/siphon/internal/canonical" "github.com/nixrajput/siphon/internal/config" "github.com/nixrajput/siphon/internal/driver" "github.com/nixrajput/siphon/internal/dumps" @@ -25,9 +27,12 @@ type chainRestoreCall struct { } // chainFakeConn records every Restore call so a test can assert chain order, -// per-dump envelope stripping, and that Clean was set only for the base. +// per-dump envelope stripping, and that Clean was set only for the base. It also +// implements CanonicalTransfer so incremental (change-replay) links are recorded +// via ApplyChange. type chainFakeConn struct { - calls []chainRestoreCall + calls []chainRestoreCall + applied []canonical.CanonicalChange } func (c *chainFakeConn) Inspect(_ context.Context) (*driver.Schema, error) { @@ -53,6 +58,17 @@ func (c *chainFakeConn) Verify(_ context.Context, _ io.Reader) (*driver.VerifyRe func (c *chainFakeConn) Close() error { return nil } +// EmitCanonical/ConsumeCanonical satisfy CanonicalTransfer; only ApplyChange is +// exercised by the incremental restore path. +func (c *chainFakeConn) EmitCanonical(_ context.Context, _ *canonical.CanonicalSchema, _ io.Writer) error { + return nil +} +func (c *chainFakeConn) ConsumeCanonical(_ context.Context, _ io.Reader) error { return nil } +func (c *chainFakeConn) ApplyChange(_ context.Context, ch canonical.CanonicalChange) error { + c.applied = append(c.applied, ch) + return nil +} + // chainFakeDriver always returns the same chainFakeConn so the test inspects it. type chainFakeDriver struct{ conn driver.Conn } @@ -64,8 +80,11 @@ func (d *chainFakeDriver) Connect(_ context.Context, _ driver.Profile) (driver.C return d.conn, nil } -// writeChainDump writes a real catalog dump file (envelope header + native -// payload) at cat.Path(id) plus its sidecar Meta with the given lineage. +// writeChainDump writes a real catalog dump file (envelope header + body) at +// cat.Path(id) plus its sidecar Meta with the given lineage. A base dump's body +// is an opaque native payload (restored via conn.Restore); an incremental dump's +// body is the payload string wrapped as a single JSONL CanonicalChange (replayed +// via conn.ApplyChange), so the change's table records the payload. func writeChainDump(t *testing.T, cat *dumps.Catalog, id, baseID, parentID, payload string) { t.Helper() f, err := os.Create(cat.Path(id)) @@ -85,7 +104,17 @@ func writeChainDump(t *testing.T, cat *dumps.Catalog, id, baseID, parentID, payl _ = f.Close() t.Fatalf("write envelope %s: %v", id, err) } - if _, err := io.WriteString(f, payload); err != nil { + body := payload + if typ == dumps.EnvelopeIncremental { + // Encode the payload as a single JSONL change so the replay path records + // it via ApplyChange (the change's Table carries the payload marker). + js, err := json.Marshal(canonical.CanonicalChange{Op: canonical.OpInsert, Table: payload, Key: map[string]any{"id": 1}}) + if err != nil { + t.Fatalf("marshal change %s: %v", id, err) + } + body = string(js) + "\n" + } + if _, err := io.WriteString(f, body); err != nil { _ = f.Close() t.Fatalf("write payload %s: %v", id, err) } @@ -137,9 +166,9 @@ func seedThreeDumpChain(t *testing.T, dir string, conn driver.Conn) Deps { return deps } -// TestRestoreChain_AppliesInOrder proves the chain is applied base->inc1->inc2 -// and that each recorded body is the native payload with the 4 KB envelope -// stripped per dump (not the raw file bytes). +// TestRestoreChain_AppliesInOrder proves the chain is applied base->inc1->inc2: +// the base goes through conn.Restore (native body, envelope stripped) and each +// incremental link is replayed through conn.ApplyChange in order. func TestRestoreChain_AppliesInOrder(t *testing.T) { conn := &chainFakeConn{} deps := seedThreeDumpChain(t, t.TempDir(), conn) @@ -150,19 +179,28 @@ func TestRestoreChain_AppliesInOrder(t *testing.T) { } drain(t, ch) - want := []string{"base-data", "inc1-data", "inc2-data"} - if len(conn.calls) != len(want) { - t.Fatalf("Restore made %d calls; want %d", len(conn.calls), len(want)) + // Base: one native Restore call with the envelope stripped. + if len(conn.calls) != 1 { + t.Fatalf("Restore made %d native calls; want 1 (base only)", len(conn.calls)) + } + if conn.calls[0].body != "base-data" { + t.Fatalf("base body = %q; want base-data (envelope must be stripped)", conn.calls[0].body) + } + // Incrementals: replayed via ApplyChange, in order. + wantInc := []string{"inc1-data", "inc2-data"} + if len(conn.applied) != len(wantInc) { + t.Fatalf("ApplyChange called %d times; want %d", len(conn.applied), len(wantInc)) } - for i, w := range want { - if conn.calls[i].body != w { - t.Fatalf("call %d body = %q; want %q (envelope must be stripped per dump)", i, conn.calls[i].body, w) + for i, w := range wantInc { + if conn.applied[i].Table != w { + t.Fatalf("applied[%d].Table = %q; want %q", i, conn.applied[i].Table, w) } } } // TestRestoreChain_CleanOnlyBeforeBase proves Clean is set on the base restore -// only and false for every incremental, when Restore is called with Clean:true. +// only; incrementals never run conn.Restore (they replay via ApplyChange), so a +// destructive Clean cannot leak onto a later link. func TestRestoreChain_CleanOnlyBeforeBase(t *testing.T) { conn := &chainFakeConn{} deps := seedThreeDumpChain(t, t.TempDir(), conn) @@ -173,21 +211,19 @@ func TestRestoreChain_CleanOnlyBeforeBase(t *testing.T) { } drain(t, ch) - if len(conn.calls) != 3 { - t.Fatalf("Restore made %d calls; want 3", len(conn.calls)) + if len(conn.calls) != 1 { + t.Fatalf("Restore made %d native calls; want 1 (base only)", len(conn.calls)) } if !conn.calls[0].clean { t.Fatalf("base call clean = false; want true") } - for i := 1; i < len(conn.calls); i++ { - if conn.calls[i].clean { - t.Fatalf("incremental call %d clean = true; want false (clean once, base only)", i) - } + if len(conn.applied) != 2 { + t.Fatalf("ApplyChange called %d times; want 2 (inc1, inc2)", len(conn.applied)) } } // TestRestoreChain_UpToTruncates proves --up-to stops the chain early, applying -// only base and inc1 when targeting inc2 with UpTo=inc1. +// only base (native) and inc1 (replay) when targeting inc2 with UpTo=inc1. func TestRestoreChain_UpToTruncates(t *testing.T) { conn := &chainFakeConn{} deps := seedThreeDumpChain(t, t.TempDir(), conn) @@ -198,14 +234,11 @@ func TestRestoreChain_UpToTruncates(t *testing.T) { } drain(t, ch) - want := []string{"base-data", "inc1-data"} - if len(conn.calls) != len(want) { - t.Fatalf("Restore made %d calls; want %d", len(conn.calls), len(want)) + if len(conn.calls) != 1 || conn.calls[0].body != "base-data" { + t.Fatalf("native calls = %+v; want one base-data call", conn.calls) } - for i, w := range want { - if conn.calls[i].body != w { - t.Fatalf("call %d body = %q; want %q", i, conn.calls[i].body, w) - } + if len(conn.applied) != 1 || conn.applied[0].Table != "inc1-data" { + t.Fatalf("applied = %+v; want one inc1-data change", conn.applied) } } diff --git a/internal/cli/backup.go b/internal/cli/backup.go index 6daf6e1..3f3d95f 100644 --- a/internal/cli/backup.go +++ b/internal/cli/backup.go @@ -28,16 +28,6 @@ func newBackupCmd() *cobra.Command { if len(args) == 1 { profileName = args[0] } - // Incremental backup is scaffolded but not wired end-to-end. Full - // wiring needs: (a) reading --base's envelope for the parent - // WAL/binlog position, (b) an optional-interface dance to reach the - // driver's BackupIncremental (it is on the concrete *Conn types, not - // driver.Conn), and (c) writing an incremental-type envelope with - // BaseID/ParentID. It also needs a live DB to verify, plus the - // tracked runtime gates: orphan replication-slot cleanup for - // Postgres and binlog-position validation for MySQL/MariaDB. Tracked - // as a Phase F follow-up; rejected honestly here rather than - // half-wired into app.Backup. if baseID != "" && !incremental { return &errs.Error{ Op: "backup", @@ -45,12 +35,11 @@ func newBackupCmd() *cobra.Command { Hint: "--base requires --incremental", } } - if incremental { + if incremental && baseID == "" { return &errs.Error{ - Op: "backup.incremental", - Code: errs.CodeUser, - Cause: errs.ErrDriverUnsupported, - Hint: "incremental backup is not yet wired end-to-end (Phase F follow-up); the driver-level machinery exists", + Op: "backup", + Code: errs.CodeUser, + Hint: "--incremental requires --base ", } } deps, err := buildDeps() @@ -66,6 +55,8 @@ func newBackupCmd() *cobra.Command { DataOnly: dataOnly, Parallel: parallel, CompressionLevel: compressionLvl, + Incremental: incremental, + BaseID: baseID, }) if err != nil { return err @@ -81,7 +72,7 @@ func newBackupCmd() *cobra.Command { cmd.Flags().BoolVar(&dataOnly, "data-only", false, "Data, no schema") cmd.Flags().IntVar(¶llel, "jobs", 1, "Parallel workers (not yet effective for backup; Phase F)") cmd.Flags().IntVar(&compressionLvl, "compression", 1, "Compression level 0-9") - cmd.Flags().BoolVar(&incremental, "incremental", false, "Capture only changes since --base (not yet wired end-to-end; Phase F follow-up)") + cmd.Flags().BoolVar(&incremental, "incremental", false, "Capture only changes since --base (requires --base)") cmd.Flags().StringVar(&baseID, "base", "", "Base dump ID for an incremental backup (used with --incremental)") return cmd } diff --git a/internal/driver/_mysqlcommon/changestream.go b/internal/driver/_mysqlcommon/changestream.go index 1ef6194..705454a 100644 --- a/internal/driver/_mysqlcommon/changestream.go +++ b/internal/driver/_mysqlcommon/changestream.go @@ -33,6 +33,15 @@ var _ driver.ChangeStreamer = (*Conn)(nil) // events, NULL rendering, quoting) need validation against a live // log_bin=ON, binlog_format=ROW server in CI. func (c *Conn) StreamChanges(ctx context.Context, from canonical.Position, emit func(canonical.CanonicalChange) error) (canonical.Position, error) { + return c.streamChangesWithStop(ctx, from, nil, emit) +} + +// streamChangesWithStop is the shared binlog decode driver behind StreamChanges +// (stopAt==nil, unbounded) and BackupIncremental (stopAt!=nil, bounded). When +// stopAt is non-nil, parsing returns cleanly at the first row event whose binlog +// position reaches or passes stopAt — every change up to that point has been +// emitted, and none past it. +func (c *Conn) streamChangesWithStop(ctx context.Context, from canonical.Position, stopAt *BinlogPosition, emit func(canonical.CanonicalChange) error) (canonical.Position, error) { if err := c.ValidateBinlogFormat(ctx); err != nil { return canonical.Position{}, err } @@ -64,10 +73,11 @@ func (c *Conn) StreamChanges(ctx context.Context, from canonical.Position, emit return canonical.Position{}, toolErr(c.binlogBinary, c.binlogBinary+".stream", err) } - endPos, parseErr := parseBinlogRows(stdout, meta, emit, since) + endPos, parseErr := parseBinlogRows(stdout, meta, emit, since, stopAt) - // Kill the (possibly unbounded) tool and reap it. On ctx cancel the kill is - // expected; we only surface a parse error, not the resulting exec error. + // Kill the (possibly unbounded) tool and reap it. On ctx cancel — or on a + // bounded stop where we stopped reading before EOF — the kill is expected; we + // only surface a parse error, not the resulting exec error. _ = cmd.Process.Kill() _ = cmd.Wait() @@ -105,7 +115,7 @@ type rowEvent struct { // assembling ### blocks into CanonicalChanges and calling emit per change. It // returns the final binlog position seen (from `# at ` comments) so the // caller can stamp the envelope / CDC state. -func parseBinlogRows(r io.Reader, meta *tableMetaCache, emit func(canonical.CanonicalChange) error, start BinlogPosition) (BinlogPosition, error) { +func parseBinlogRows(r io.Reader, meta *tableMetaCache, emit func(canonical.CanonicalChange) error, start BinlogPosition, stopAt *BinlogPosition) (BinlogPosition, error) { sc := bufio.NewScanner(r) sc.Buffer(make([]byte, 0, 64*1024), 4*1024*1024) @@ -133,6 +143,16 @@ func parseBinlogRows(r io.Reader, meta *tableMetaCache, emit func(canonical.Cano // Track the current binlog offset from "# at " markers. if p, ok := parseAtMarker(line); ok { pos.Position = p + // Bounded (incremental) stop: an "# at" marker precedes each event, so + // once the marker reaches/passes the captured end position in the end + // file, every earlier event has been parsed. Flush the pending event + // and return at this clean boundary. + if stopAt != nil && pos.File == stopAt.File && pos.Position >= stopAt.Position { + if err := flush(); err != nil { + return pos, err + } + return pos, nil + } continue } // Track the active binlog file from rotate events. diff --git a/internal/driver/_mysqlcommon/changestream_test.go b/internal/driver/_mysqlcommon/changestream_test.go index 8626af4..bec4952 100644 --- a/internal/driver/_mysqlcommon/changestream_test.go +++ b/internal/driver/_mysqlcommon/changestream_test.go @@ -87,7 +87,7 @@ func TestParseBinlogRows(t *testing.T) { var got []canonical.CanonicalChange emit := func(ch canonical.CanonicalChange) error { got = append(got, ch); return nil } - pos, err := parseBinlogRows(strings.NewReader(transcript), meta, emit, BinlogPosition{File: "mysql-bin.000001"}) + pos, err := parseBinlogRows(strings.NewReader(transcript), meta, emit, BinlogPosition{File: "mysql-bin.000001"}, nil) if err != nil { t.Fatalf("parseBinlogRows: %v", err) } diff --git a/internal/driver/_mysqlcommon/incremental.go b/internal/driver/_mysqlcommon/incremental.go index 02bc509..567dcc5 100644 --- a/internal/driver/_mysqlcommon/incremental.go +++ b/internal/driver/_mysqlcommon/incremental.go @@ -5,11 +5,10 @@ import ( "database/sql" "errors" "io" - "os" - "os/exec" "strconv" "strings" + "github.com/nixrajput/siphon/internal/canonical" "github.com/nixrajput/siphon/internal/driver" "github.com/nixrajput/siphon/internal/errs" ) @@ -150,21 +149,31 @@ func binlogSSLArgs(sslMode, binlogBinary string) []string { } } -// BackupIncremental streams binlog events from `since` to current end-of-binlog -// into w, using the fork's binlog tool (mysqlbinlog / mariadb-binlog). +var _ driver.IncrementalBackuper = (*Conn)(nil) + +// BackupIncremental captures the BOUNDED change set from `since` to the current +// end-of-binlog, serializing each CanonicalChange to w as JSONL, and returns the +// end Position reached. +// +// Bounding mechanism: the end binlog coordinates are captured up front via +// CaptureBinlogPosition and passed to the shared binlog decode loop as a stop +// target. Parsing returns cleanly at the first "# at" marker that reaches or +// passes the captured end offset in the end file, so every event up to it is +// emitted and none past it. This reuses StreamChanges' decode machinery so the +// incremental body is engine-neutral JSONL that the restore path replays via +// ApplyChange (rather than raw binlog bytes). // -// NOTE: the --read-from-remote-server + --start-position invocation is -// structurally complete but UNPROVEN locally (no Docker/MySQL here). The exact -// remote-auth flags and whether a single starting binlog file suffices vs. -// needing --to-last-log need validation against a live log_bin=ON, -// binlog_format=ROW server (see CI / the incremental wiring task). -func (c *Conn) BackupIncremental(ctx context.Context, since BinlogPosition, w io.Writer) error { - cmd := exec.CommandContext(ctx, c.binlogBinary, binlogArgs(c.p, since, c.binlogBinary)...) - cmd.Env = withMySQLPwd(os.Environ(), c.p.Password) - cmd.Stdout = w - cmd.Stderr = os.Stderr - if err := cmd.Run(); err != nil { - return &errs.Error{Op: c.binlogBinary + ".backup_incremental", Code: errs.CodeSystem, Cause: err} +// This path is exercised against a live log_bin=ON, binlog_format=ROW server only +// in CI; it is not validated locally (no MySQL/MariaDB here). +func (c *Conn) BackupIncremental(ctx context.Context, since canonical.Position, w io.Writer) (canonical.Position, error) { + end, err := c.CaptureBinlogPosition(ctx) + if err != nil { + return canonical.Position{}, err } - return nil + stopAt := &BinlogPosition{File: end.File, Position: end.Position} + + emit := func(ch canonical.CanonicalChange) error { + return canonical.WriteJSONL(w, ch) + } + return c.streamChangesWithStop(ctx, since, stopAt, emit) } diff --git a/internal/driver/driver.go b/internal/driver/driver.go index 6ebe230..30471b6 100644 --- a/internal/driver/driver.go +++ b/internal/driver/driver.go @@ -52,6 +52,20 @@ type ChangeStreamer interface { StreamChanges(ctx context.Context, from canonical.Position, emit func(canonical.CanonicalChange) error) (canonical.Position, error) } +// IncrementalBackuper is an optional Conn capability: capture the BOUNDED change +// set from `since` to the engine's current end position, serializing each +// CanonicalChange to w as JSONL, and return the end Position reached. +// +// "Bounded" means the capture stops at a fixed end position captured at the +// start of the call (Postgres: pg_current_wal_lsn(); MySQL/MariaDB: the current +// binlog file+offset), unlike CDC's unbounded StreamChanges. It is implemented +// in terms of StreamChanges' decode machinery with that end position as a stop +// target. The returned Position is stamped into the incremental dump's Envelope +// so the NEXT incremental resumes from exactly here. +type IncrementalBackuper interface { + BackupIncremental(ctx context.Context, since canonical.Position, w io.Writer) (canonical.Position, error) +} + // Capabilities describes what an engine supports. Each flag gates a UI // affordance or feature path. Drivers must declare honestly. type Capabilities struct { diff --git a/internal/driver/postgres/changestream.go b/internal/driver/postgres/changestream.go index 8f85cdb..1c38678 100644 --- a/internal/driver/postgres/changestream.go +++ b/internal/driver/postgres/changestream.go @@ -41,6 +41,14 @@ var _ driver.ChangeStreamer = (*Conn)(nil) // is the normal stop signal and is NOT reported as an error — the final // Position reached is returned for envelope stamping / CDC state persistence. func (c *Conn) StreamChanges(ctx context.Context, from canonical.Position, emit func(canonical.CanonicalChange) error) (canonical.Position, error) { + return c.streamWithStop(ctx, from, 0, emit) +} + +// streamWithStop is the shared pgoutput decode driver behind StreamChanges +// (stopLSN==0, unbounded) and BackupIncremental (stopLSN!=0, bounded). It opens +// the replication connection, ensures the slot, resolves the start LSN, starts +// replication, and runs receiveLoop with the given stop bound. +func (c *Conn) streamWithStop(ctx context.Context, from canonical.Position, stopLSN pglogrepl.LSN, emit func(canonical.CanonicalChange) error) (canonical.Position, error) { if err := c.requireLogicalWAL(ctx); err != nil { return canonical.Position{}, err } @@ -77,12 +85,18 @@ func (c *Conn) StreamChanges(ctx context.Context, from canonical.Position, emit return canonical.Position{}, &errs.Error{Op: "postgres.stream", Code: errs.CodeSystem, Cause: err} } - return receiveLoop(ctx, repl, startLSN, emit) + return receiveLoop(ctx, repl, startLSN, stopLSN, emit) } // receiveLoop runs the pgoutput receive/decode/ack cycle until ctx is cancelled // or emit returns an error. It returns the final committed LSN as a Position. -func receiveLoop(ctx context.Context, repl *pgconn.PgConn, startLSN pglogrepl.LSN, emit func(canonical.CanonicalChange) error) (canonical.Position, error) { +// +// stopLSN bounds a BOUNDED (incremental) capture: when non-zero, the loop returns +// cleanly once the client position reaches or passes it — but only at a message +// boundary AFTER all changes up to that point have been decoded and emitted, so +// the bound never truncates a change that committed at or before stopLSN. A zero +// stopLSN means unbounded (CDC): stream until ctx cancel. +func receiveLoop(ctx context.Context, repl *pgconn.PgConn, startLSN, stopLSN pglogrepl.LSN, emit func(canonical.CanonicalChange) error) (canonical.Position, error) { relations := map[uint32]*pglogrepl.RelationMessage{} typeMap := pgtype.NewMap() clientXLogPos := startLSN @@ -99,6 +113,16 @@ func receiveLoop(ctx context.Context, repl *pgconn.PgConn, startLSN pglogrepl.LS return finalPos(), nil } + // Bounded (incremental) stop: once the client has caught up to the target + // end LSN, every change committed at or before it has already been decoded + // and emitted (XLogData advances clientXLogPos only after decodeWALData), + // so it is safe to ack and return the end position. + if stopLSN != 0 && clientXLogPos >= stopLSN { + _ = pglogrepl.SendStandbyStatusUpdate(context.Background(), repl, + pglogrepl.StandbyStatusUpdate{WALWritePosition: clientXLogPos}) + return finalPos(), nil + } + if time.Now().After(nextDeadline) { if err := pglogrepl.SendStandbyStatusUpdate(ctx, repl, pglogrepl.StandbyStatusUpdate{WALWritePosition: clientXLogPos}); err != nil { diff --git a/internal/driver/postgres/driver.go b/internal/driver/postgres/driver.go index 95cfff9..b95ccb7 100644 --- a/internal/driver/postgres/driver.go +++ b/internal/driver/postgres/driver.go @@ -24,7 +24,7 @@ func (Driver) Name() string { return "postgres" } func (Driver) Capabilities() driver.Capabilities { return driver.Capabilities{ - Incremental: false, // arrives in Phase F (WAL) + Incremental: true, // Phase F: bounded change capture via logical decoding NativeStream: true, PerTable: true, SchemaOnly: true, diff --git a/internal/driver/postgres/incremental.go b/internal/driver/postgres/incremental.go index 7532bae..9f36d72 100644 --- a/internal/driver/postgres/incremental.go +++ b/internal/driver/postgres/incremental.go @@ -3,17 +3,10 @@ package postgres import ( "context" "fmt" - "io" - "os" - "os/exec" - "path/filepath" - "sort" - "strconv" "strings" "github.com/oklog/ulid/v2" - "github.com/nixrajput/siphon/internal/driver" "github.com/nixrajput/siphon/internal/errs" ) @@ -57,79 +50,6 @@ func (c *Conn) CaptureBaseEnd(ctx context.Context, info *IncrementalBaseInfo) er return nil } -// incrementalArgs builds the pg_receivewal argv to stream WAL from the slot -// into a destination DIRECTORY. pg_receivewal requires -D ; it does NOT -// support "-D -" (stdout). The slot (created at base time) anchors retention, -// so streaming from it captures WAL accumulated since the base. --endpos stops -// the stream at a defined LSN so the invocation terminates deterministically; -// --no-loop controls connection-RETRY behavior (don't loop reconnecting on a -// dropped connection), NOT WAL-end termination. This path needs validation -// against a live wal_level>=replica server (see incremental_test.go) — it is -// structurally complete but unproven locally (no Docker in this environment). -func incrementalArgs(p driver.Profile, slotName, dir, endpos string) []string { - return []string{ - "-h", p.Host, - "-p", strconv.Itoa(p.Port), - "-U", p.User, - "-D", dir, // pg_receivewal writes WAL segments into this directory - "--slot=" + slotName, - "--endpos=" + endpos, // stop at this LSN instead of streaming forever - "--synchronous", - "--no-loop", // do not retry the connection if it drops - "--verbose", - } -} - -// BackupIncremental streams WAL from the base's slot into a temp directory up -// to the server's current end LSN, then concatenates the resulting WAL segment -// files into w in name order. The caller prepends the dump Envelope at the app -// layer. -// -// pg_receivewal cannot stream to stdout (no "-D -"), so it must write segment -// files to a directory; we capture the current end LSN via pg_current_wal_lsn() -// and pass it as --endpos so the stream terminates at a defined point. This -// streaming path is exercised only in CI / against a live server — it is not -// validated locally (no Docker in this environment). -func (c *Conn) BackupIncremental(ctx context.Context, info IncrementalBaseInfo, w io.Writer) error { - var endpos string - if err := c.db.QueryRowContext(ctx, "SELECT pg_current_wal_lsn()::text").Scan(&endpos); err != nil { - return &errs.Error{Op: "postgres.backup_incremental", Code: errs.CodeSystem, Cause: err} - } - - tmpDir, err := os.MkdirTemp("", "siphon-wal-*") - if err != nil { - return &errs.Error{Op: "postgres.backup_incremental", Code: errs.CodeSystem, Cause: err} - } - defer func() { _ = os.RemoveAll(tmpDir) }() - - cmd := exec.CommandContext(ctx, "pg_receivewal", incrementalArgs(c.p, info.SlotName, tmpDir, endpos)...) - cmd.Env = append(os.Environ(), "PGPASSWORD="+c.p.Password) - cmd.Stderr = os.Stderr // surface pg_receivewal diagnostics (matches restore.go convention) - if err := cmd.Run(); err != nil { - return &errs.Error{Op: "postgres.backup_incremental", Code: errs.CodeSystem, Cause: err} - } - - // Copy the captured WAL segments to w in name order. Segment file names are - // fixed-width hex, so lexical sort is chronological order. - segments, err := filepath.Glob(filepath.Join(tmpDir, "*")) - if err != nil { - return &errs.Error{Op: "postgres.backup_incremental", Code: errs.CodeSystem, Cause: err} - } - sort.Strings(segments) - for _, seg := range segments { - f, err := os.Open(seg) - if err != nil { - return &errs.Error{Op: "postgres.backup_incremental", Code: errs.CodeSystem, Cause: err} - } - if _, err := io.Copy(w, f); err != nil { - _ = f.Close() - return &errs.Error{Op: "postgres.backup_incremental", Code: errs.CodeSystem, Cause: err} - } - _ = f.Close() - } - return nil -} - // DropSlot removes the replication slot once a chain is sealed (or via an // orphan scan on startup). Safe to call best-effort. func (c *Conn) DropSlot(ctx context.Context, slotName string) error { diff --git a/internal/driver/postgres/incremental_change.go b/internal/driver/postgres/incremental_change.go new file mode 100644 index 0000000..1844a70 --- /dev/null +++ b/internal/driver/postgres/incremental_change.go @@ -0,0 +1,100 @@ +package postgres + +import ( + "context" + "io" + + "github.com/jackc/pglogrepl" + + "github.com/nixrajput/siphon/internal/canonical" + "github.com/nixrajput/siphon/internal/driver" + "github.com/nixrajput/siphon/internal/errs" +) + +var _ driver.IncrementalBackuper = (*Conn)(nil) + +// BackupIncremental captures the BOUNDED change set from `since` to the server's +// current end LSN, serializing each CanonicalChange to w as JSONL, and returns +// the end Position reached. +// +// Bounding mechanism: the end LSN is captured up front via pg_current_wal_lsn() +// and passed to the shared pgoutput decode loop as a stop target. The loop +// advances its client position only AFTER decoding+emitting each XLogData +// message, so it returns cleanly at the first message boundary where the client +// position reaches or passes the captured end LSN — every change committed at or +// before that LSN has been emitted, and none past it. (Once the live stream is +// caught up, a server keepalive carries ServerWALEnd, which crosses the bound and +// triggers the stop.) This reuses StreamChanges' decode machinery rather than +// streaming raw WAL bytes, so the incremental body is engine-neutral JSONL that +// the restore path replays via ApplyChange. +// +// Before streaming, orphaned siphon replication slots are swept (best-effort) to +// keep WAL retention bounded. +// +// This path is exercised against a live wal_level=logical server only in CI (see +// incremental_integration_test.go); it is not validated locally (no Docker here). +func (c *Conn) BackupIncremental(ctx context.Context, since canonical.Position, w io.Writer) (canonical.Position, error) { + // Best-effort orphan-slot sweep before we (re)use the logical slot. A failure + // here must not abort the backup, so the count/err is logged-by-return only + // where the caller surfaces it; here we ignore a sweep error and proceed. + _, _ = c.SweepOrphanSlots(ctx) + + var endLSNText string + if err := c.db.QueryRowContext(ctx, "SELECT pg_current_wal_lsn()::text").Scan(&endLSNText); err != nil { + return canonical.Position{}, &errs.Error{Op: "postgres.backup_incremental", Code: errs.CodeSystem, Cause: err} + } + stopLSN, err := pglogrepl.ParseLSN(endLSNText) + if err != nil { + return canonical.Position{}, &errs.Error{Op: "postgres.backup_incremental", Code: errs.CodeSystem, Cause: err} + } + + emit := func(ch canonical.CanonicalChange) error { + return canonical.WriteJSONL(w, ch) + } + return c.streamWithStop(ctx, since, stopLSN, emit) +} + +// SweepOrphanSlots drops inactive siphon-owned PHYSICAL base slots and returns +// the count dropped. +// +// Policy: a base backup creates a non-temporary physical slot named +// siphon_ (see CreateBaseSlot) to pin WAL for a future incremental; that +// slot must be dropped when the chain is sealed. An active such slot is in use by +// a running backup, but an inactive one is orphaned — a normal run drops its own +// slot, so any inactive siphon_ left behind belongs to a crashed/aborted +// run and is only pinning WAL. We drop every inactive slot matching the prefix, +// EXCEPT the persistent logical CDC slot (siphonSlot), which is the resume anchor +// for change streaming and is legitimately inactive between runs — sweeping it +// would discard the resume position. Each drop is best-effort: a concurrent run +// that re-activates a slot between the scan and the drop makes +// pg_drop_replication_slot fail with "in use", which we skip. +func (c *Conn) SweepOrphanSlots(ctx context.Context) (int, error) { + rows, err := c.db.QueryContext(ctx, + `SELECT slot_name FROM pg_replication_slots + WHERE slot_name LIKE 'siphon\_%' AND active = false AND slot_name <> $1`, siphonSlot) + if err != nil { + return 0, &errs.Error{Op: "postgres.sweep_slots", Code: errs.CodeSystem, Cause: err} + } + var names []string + for rows.Next() { + var name string + if scanErr := rows.Scan(&name); scanErr != nil { + _ = rows.Close() + return 0, &errs.Error{Op: "postgres.sweep_slots", Code: errs.CodeSystem, Cause: scanErr} + } + names = append(names, name) + } + _ = rows.Close() + if err := rows.Err(); err != nil { + return 0, &errs.Error{Op: "postgres.sweep_slots", Code: errs.CodeSystem, Cause: err} + } + + dropped := 0 + for _, name := range names { + if _, err := c.db.ExecContext(ctx, "SELECT pg_drop_replication_slot($1)", name); err != nil { + continue // raced with a re-activation, or already gone: skip + } + dropped++ + } + return dropped, nil +} diff --git a/internal/driver/postgres/incremental_integration_test.go b/internal/driver/postgres/incremental_integration_test.go new file mode 100644 index 0000000..4f74cfb --- /dev/null +++ b/internal/driver/postgres/incremental_integration_test.go @@ -0,0 +1,139 @@ +//go:build integration + +package postgres + +import ( + "bufio" + "bytes" + "context" + "database/sql" + "encoding/json" + "testing" + "time" + + "github.com/nixrajput/siphon/internal/canonical" +) + +// TestBackupIncremental_BoundedCaptureAndReplay exercises the full bounded +// incremental path against a live wal_level=logical server: +// +// base state → record base-end LSN → insert/update/delete → +// BackupIncremental (bounded JSONL capture) → drop & recreate the table → +// replay the captured changes via ApplyChange → assert final row state. +// +// It proves (a) BackupIncremental stops at the captured end position and returns +// it, and (b) the JSONL change body, replayed via ApplyChange, reconstructs the +// post-change state — the same machinery app.Restore uses for incremental links. +func TestBackupIncremental_BoundedCaptureAndReplay(t *testing.T) { + prof, cleanup := startLogicalPostgres(t) + defer cleanup() + + ctx := context.Background() + db, err := sql.Open("pgx", buildDSN(prof)) + if err != nil { + t.Fatalf("open db: %v", err) + } + defer func() { _ = db.Close() }() + + if _, err := db.ExecContext(ctx, + `CREATE TABLE widgets(id int primary key, name text)`); err != nil { + t.Fatalf("create table: %v", err) + } + // Base state: one row that will be updated, one that will be deleted. + if _, err := db.ExecContext(ctx, `INSERT INTO widgets VALUES (1,'wrench'),(2,'doomed')`); err != nil { + t.Fatalf("seed base: %v", err) + } + + conn := &Conn{db: db, p: prof} + + // Publication + slot must exist before the DML so the slot retains the WAL. + if err := conn.ensurePublication(ctx); err != nil { + t.Fatalf("ensure publication: %v", err) + } + + // Capture the base-end LSN: the incremental resumes from exactly here. + var baseEnd string + if err := db.QueryRowContext(ctx, "SELECT pg_current_wal_lsn()::text").Scan(&baseEnd); err != nil { + t.Fatalf("capture base-end lsn: %v", err) + } + + // Post-base changes: one insert, one update, one delete. + if _, err := db.ExecContext(ctx, `INSERT INTO widgets VALUES (3,'fresh')`); err != nil { + t.Fatalf("insert: %v", err) + } + if _, err := db.ExecContext(ctx, `UPDATE widgets SET name='spanner' WHERE id=1`); err != nil { + t.Fatalf("update: %v", err) + } + if _, err := db.ExecContext(ctx, `DELETE FROM widgets WHERE id=2`); err != nil { + t.Fatalf("delete: %v", err) + } + + // Bounded incremental capture from the base-end LSN to the current end. + capCtx, cancel := context.WithTimeout(ctx, 60*time.Second) + defer cancel() + var buf bytes.Buffer + endPos, err := conn.BackupIncremental(capCtx, canonical.Position{LSN: baseEnd}, &buf) + if err != nil { + t.Fatalf("BackupIncremental: %v", err) + } + if endPos.LSN == "" { + t.Fatalf("BackupIncremental returned empty end position") + } + + // Reset the table to its base state, then replay the captured changes; the + // final state must reflect the insert/update/delete. + if _, err := db.ExecContext(ctx, `TRUNCATE widgets`); err != nil { + t.Fatalf("truncate: %v", err) + } + if _, err := db.ExecContext(ctx, `INSERT INTO widgets VALUES (1,'wrench'),(2,'doomed')`); err != nil { + t.Fatalf("reseed base: %v", err) + } + + sc := bufio.NewScanner(&buf) + var applied int + for sc.Scan() { + if len(sc.Bytes()) == 0 { + continue + } + var ch canonical.CanonicalChange + if err := json.Unmarshal(sc.Bytes(), &ch); err != nil { + t.Fatalf("decode change: %v", err) + } + if err := conn.ApplyChange(ctx, ch); err != nil { + t.Fatalf("ApplyChange %+v: %v", ch, err) + } + applied++ + } + if err := sc.Err(); err != nil { + t.Fatalf("scan changes: %v", err) + } + if applied < 3 { + t.Fatalf("expected at least 3 captured changes, applied %d", applied) + } + + // Assert final state: id=1 updated to spanner, id=2 deleted, id=3 inserted. + rows := map[int]string{} + r, err := db.QueryContext(ctx, `SELECT id, name FROM widgets ORDER BY id`) + if err != nil { + t.Fatalf("query final: %v", err) + } + for r.Next() { + var id int + var name string + if err := r.Scan(&id, &name); err != nil { + t.Fatalf("scan final: %v", err) + } + rows[id] = name + } + _ = r.Close() + + if rows[1] != "spanner" { + t.Errorf("id=1 = %q, want spanner (update not applied)", rows[1]) + } + if _, ok := rows[2]; ok { + t.Errorf("id=2 still present (delete not applied): %q", rows[2]) + } + if rows[3] != "fresh" { + t.Errorf("id=3 = %q, want fresh (insert not applied)", rows[3]) + } +} From fcf66e9faef2034ff1fed1a48183ff3a763c2cb4 Mon Sep 17 00:00:00 2001 From: Nikhil Rajput Date: Tue, 23 Jun 2026 03:57:18 +0530 Subject: [PATCH 08/13] fix(app,driver): stamp base position on full backups so first incremental resumes correctly - Add driver.BasePositioner; implement CurrentPosition on Postgres (pg_current_wal_lsn) and MySQL/MariaDB (binlog file+offset) Conns with compile assertions. - app.Backup full path now streams the dump body to a temp file, captures the engine position just after Backup returns, and stamps WALEnd / BinlogFile+BinlogEnd into the base envelope before assembling envelope+body. Previously full base envelopes carried no position, so the first incremental off a full base started from "now" and silently dropped changes committed between the base dump and the incremental run. - Make incremental-replay INSERTs idempotent via canonical.BuildIdempotentInsertSQL (ON CONFLICT DO NOTHING / INSERT IGNORE) so a change captured in both base and first incremental re-applies harmlessly. - Add covering unit test that a full backup whose driver implements BasePositioner stamps a non-empty WALEnd into the base envelope, read back via basePosition; plus canonical idempotent-INSERT tests. - Document the base-position stamping and capture-after-dump rationale in docs/INCREMENTAL.md. --- docs/INCREMENTAL.md | 21 ++++ internal/app/backup.go | 102 ++++++++++++++---- internal/app/backup_test.go | 70 ++++++++++++ internal/canonical/canonical.go | 21 ++++ internal/canonical/canonical_test.go | 28 +++++ internal/driver/_mysqlcommon/canonical.go | 2 +- internal/driver/_mysqlcommon/incremental.go | 13 +++ internal/driver/driver.go | 12 +++ internal/driver/postgres/canonical.go | 2 +- .../driver/postgres/incremental_change.go | 12 +++ 10 files changed, 260 insertions(+), 23 deletions(-) diff --git a/docs/INCREMENTAL.md b/docs/INCREMENTAL.md index 80caea9..bbf1561 100644 --- a/docs/INCREMENTAL.md +++ b/docs/INCREMENTAL.md @@ -36,6 +36,27 @@ WAL/binlog bytes. At restore time those changes are **replayed** via `ApplyChange` rather than fed to the native restore tool — base links restore natively, incremental links replay change records. +### Where the first incremental resumes from + +`backup --incremental --base ` reads the base dump's envelope for its end +position (`basePosition()` in `internal/app/backup.go`). For this to be correct +when `--base` points at a **full** dump, that full backup must record where the +engine's change stream stood as of the dump. Every full backup therefore captures +the engine position immediately after the dump completes (via +`driver.BasePositioner.CurrentPosition` — `pg_current_wal_lsn()` for Postgres, +the current binlog file+offset for MySQL/MariaDB) and stamps it into the base +envelope (`WALEnd` / `BinlogFile`+`BinlogEnd`). Without this, the first +incremental off a full base would start from "now" and silently drop every change +committed between the base dump and the incremental run. + +Capturing the position *after* the dump (rather than before) never under-captures: +a consistent dump reflects the DB as of its snapshot, and the post-dump position +is at-or-after that snapshot, so the incremental picks up every post-base change. +The only edge is a change landing right at the boundary being captured in both the +base and the incremental; incremental-replay INSERTs are therefore idempotent +(`ON CONFLICT DO NOTHING` for Postgres, `INSERT IGNORE` for MySQL/MariaDB), so the +re-apply is harmless. + The driver-level capture (`driver.IncrementalBackuper`): - **Postgres** (`internal/driver/postgres/incremental_change.go`) captures the diff --git a/internal/app/backup.go b/internal/app/backup.go index 1f3635e..dff8be4 100644 --- a/internal/app/backup.go +++ b/internal/app/backup.go @@ -74,6 +74,69 @@ func Backup(parent context.Context, d Deps, opt BackupOpts) (<-chan jobs.Event, return runIncrementalBackup(ctx, d, opt, resolved, conn, emit) } + // Stream the dump body to a temp file FIRST, then capture the + // engine's change-stream position, then assemble the final dump as + // envelope(position) + body — mirroring runIncrementalBackup. + // + // Why this ordering: a full base may later be used as `--base` for an + // incremental, whose `since` is read from this envelope. If the + // envelope carried no position, the first incremental would start + // from "now" and silently drop every change committed between the + // base dump and the incremental run. We therefore record a position. + // + // Consistent point — capture AFTER Backup returns: a consistent dump + // reflects the DB as of its snapshot; pg_current_wal_lsn() (binlog + // pos for MySQL/MariaDB) read just after the dump is at-or-after that + // snapshot, so the incremental captures every post-base change from + // there forward — no under-capture / no data loss. The only risk is a + // change landing right at the boundary being captured in BOTH base + // and incremental; ApplyChange's INSERT is idempotent (ON CONFLICT DO + // NOTHING / INSERT IGNORE) so such a re-apply is harmless. Capturing + // after (not before) the dump avoids the inverse hazard of streaming + // pre-snapshot inserts that would conflict on PK. + bodyTmp, err := os.CreateTemp(d.Dumps.Root(), "siphon-base-body-*") + if err != nil { + return err + } + bodyPath := bodyTmp.Name() + defer func() { _ = os.Remove(bodyPath) }() + + emit(jobs.Event{Phase: jobs.PhaseProgress, Message: "dumping"}) + + backupErr := conn.Backup(ctx, driver.BackupOpts{ + IncludeTables: opt.IncludeTables, + ExcludeTables: opt.ExcludeTables, + ExcludeDataFrom: opt.ExcludeDataFrom, + SchemaOnly: opt.SchemaOnly, + DataOnly: opt.DataOnly, + CompressionLevel: opt.CompressionLevel, + Parallel: opt.Parallel, + }, bodyTmp) + + // Close the body file explicitly and check the error: for a file + // being WRITTEN, Close() is where buffered data is flushed and where + // late I/O failures (ENOSPC, quota, disk error) surface. The pg_dump + // error takes precedence if both occurred. + bodyCloseErr := bodyTmp.Close() + if backupErr != nil { + return backupErr + } + if bodyCloseErr != nil { + return &errs.Error{Op: "app.backup", Code: errs.CodeSystem, Cause: bodyCloseErr, Hint: "failed to flush dump to disk (out of space?)"} + } + + // Capture the base position now (just after a consistent dump). Only + // drivers that support incremental implement BasePositioner; for the + // rest the envelope simply carries no position, as before. + var basePos canonical.Position + if bp, ok := conn.(driver.BasePositioner); ok { + pos, posErr := bp.CurrentPosition(ctx) + if posErr != nil { + return posErr + } + basePos = pos + } + id := ulid.Make().String() tmpPath := filepath.Join(d.Dumps.Root(), id+".dump.tmp") finalPath := d.Dumps.Path(id) @@ -88,37 +151,34 @@ func Backup(parent context.Context, d Deps, opt BackupOpts) (<-chan jobs.Event, env := &dumps.Envelope{ Type: dumps.EnvelopeBase, Driver: resolved.Driver, + WALEnd: basePos.LSN, Created: time.Now().UTC(), } + if basePos.BinlogFile != "" { + env.BinlogFile = basePos.BinlogFile + env.BinlogEnd = basePos.BinlogPos + } if _, err := dumps.WriteEnvelope(tee, env); err != nil { _ = f.Close() _ = os.Remove(tmpPath) return err } - emit(jobs.Event{Phase: jobs.PhaseProgress, Message: "dumping"}) - - backupErr := conn.Backup(ctx, driver.BackupOpts{ - IncludeTables: opt.IncludeTables, - ExcludeTables: opt.ExcludeTables, - ExcludeDataFrom: opt.ExcludeDataFrom, - SchemaOnly: opt.SchemaOnly, - DataOnly: opt.DataOnly, - CompressionLevel: opt.CompressionLevel, - Parallel: opt.Parallel, - }, tee) - - // Close the dump file explicitly and check the error: for a file - // being WRITTEN, Close() is where buffered data is flushed and where - // late I/O failures (ENOSPC, quota, disk error) surface. Ignoring it - // could rename a truncated dump into the catalog with a checksum that - // matches the truncated bytes — corrupt-but-looks-valid. The pg_dump - // error takes precedence if both occurred. - closeErr := f.Close() - if backupErr != nil { + body, err := os.Open(bodyPath) + if err != nil { + _ = f.Close() _ = os.Remove(tmpPath) - return backupErr + return err } + if _, err := io.Copy(tee, body); err != nil { + _ = body.Close() + _ = f.Close() + _ = os.Remove(tmpPath) + return err + } + _ = body.Close() + + closeErr := f.Close() if closeErr != nil { _ = os.Remove(tmpPath) return &errs.Error{Op: "app.backup", Code: errs.CodeSystem, Cause: closeErr, Hint: "failed to flush dump to disk (out of space?)"} diff --git a/internal/app/backup_test.go b/internal/app/backup_test.go index d99f277..cee255f 100644 --- a/internal/app/backup_test.go +++ b/internal/app/backup_test.go @@ -9,6 +9,7 @@ import ( "strings" "testing" + "github.com/nixrajput/siphon/internal/canonical" "github.com/nixrajput/siphon/internal/config" "github.com/nixrajput/siphon/internal/driver" "github.com/nixrajput/siphon/internal/dumps" @@ -87,6 +88,75 @@ func TestBackup_WritesDumpAndMeta(t *testing.T) { } } +// posConn is a fakeConn that also implements driver.BasePositioner, so a full +// backup stamps the returned position into the base envelope. +type posConn struct { + fakeConn + pos canonical.Position +} + +func (c *posConn) CurrentPosition(_ context.Context) (canonical.Position, error) { + return c.pos, nil +} + +type posDriver struct { + payload string + pos canonical.Position +} + +func (posDriver) Name() string { return "fake" } +func (posDriver) Capabilities() driver.Capabilities { return driver.Capabilities{Incremental: true} } +func (d posDriver) Connect(_ context.Context, _ driver.Profile) (driver.Conn, error) { + return &posConn{fakeConn: fakeConn{payload: d.payload}, pos: d.pos}, nil +} + +// TestBackup_FullStampsBasePosition asserts the gap is closed: a full backup +// whose driver implements BasePositioner records the engine position in the base +// dump's envelope, so a later incremental's basePosition() resumes from a real +// position instead of the zero value (which would silently drop changes between +// the base dump and the first incremental). +func TestBackup_FullStampsBasePosition(t *testing.T) { + dir := t.TempDir() + t.Setenv("SIPHON_CONFIG_HOME", t.TempDir()) + + cfg := &config.Config{Profiles: map[string]config.ProfileConfig{ + "test": {Driver: "fake", Host: "h", User: "u", Database: "d", Password: "p"}, + }} + res := secrets.NewResolver(secrets.Passthrough{}) + ps := profile.New(cfg, res, func(*config.Config) error { return nil }) + cat, _ := dumps.NewCatalog(dir) + + deps := Deps{ + Profiles: ps, + Dumps: cat, + Runner: jobs.NewRunner(), + Drivers: fakeGetter{d: posDriver{payload: "hello-dump", pos: canonical.Position{LSN: "0/16B6358"}}}, + } + + ch, _, err := Backup(context.Background(), deps, BackupOpts{Profile: "test"}) + if err != nil { + t.Fatalf("Backup: %v", err) + } + for range ch { /* drain */ + } + + got, err := cat.List() + if err != nil { + t.Fatal(err) + } + if len(got) != 1 { + t.Fatalf("catalog.List() = %d entries; want 1", len(got)) + } + + pos, err := basePosition(deps, got[0].ID) + if err != nil { + t.Fatalf("basePosition: %v", err) + } + if pos.LSN != "0/16B6358" { + t.Fatalf("base envelope WALEnd = %q; want 0/16B6358 (position not stamped on full backup)", pos.LSN) + } +} + // failBackupConn fails mid-dump, exercising the backup-error branch that must // remove the temp file and create no catalog entry. type failBackupConn struct{ fakeConn } diff --git a/internal/canonical/canonical.go b/internal/canonical/canonical.go index c11d1a6..8559946 100644 --- a/internal/canonical/canonical.go +++ b/internal/canonical/canonical.go @@ -296,6 +296,27 @@ func BuildInsertSQL(engine, table string, cols []string) (string, error) { qt, strings.Join(qcols, ","), strings.Join(phs, ",")), nil } +// BuildIdempotentInsertSQL renders an INSERT that silently no-ops on a +// duplicate key — `ON CONFLICT DO NOTHING` for Postgres, `INSERT IGNORE` for +// MySQL/MariaDB. Used when replaying incremental changes: a base dump and its +// first incremental may overlap by a few rows at the position boundary, so an +// INSERT of an already-present row must not error. (Snapshot loads into a fresh +// table use the plain BuildInsertSQL.) +func BuildIdempotentInsertSQL(engine, table string, cols []string) (string, error) { + base, err := BuildInsertSQL(engine, table, cols) + if err != nil { + return "", err + } + switch engine { + case "postgres": + return base + " ON CONFLICT DO NOTHING", nil + case "mysql", "mariadb": + return strings.Replace(base, "INSERT INTO", "INSERT IGNORE INTO", 1), nil + default: + return "", fmt.Errorf("cross-engine: unknown engine %q", engine) + } +} + // ChangeColumns returns the SET columns (Values minus Key, sorted) and the Key // columns (sorted), giving deterministic statement shape and argument order. func ChangeColumns(ch CanonicalChange) (setCols, keyCols []string) { diff --git a/internal/canonical/canonical_test.go b/internal/canonical/canonical_test.go index 526af33..3730745 100644 --- a/internal/canonical/canonical_test.go +++ b/internal/canonical/canonical_test.go @@ -245,6 +245,34 @@ func TestBuildInsertSQL_MySQL(t *testing.T) { } } +func TestBuildIdempotentInsertSQL_Postgres(t *testing.T) { + got, err := BuildIdempotentInsertSQL("postgres", "t", []string{"id", "name"}) + if err != nil { + t.Fatal(err) + } + want := `INSERT INTO "t" ("id","name") VALUES ($1,$2) ON CONFLICT DO NOTHING` + if got != want { + t.Errorf("postgres idempotent INSERT:\n got %q\nwant %q", got, want) + } +} + +func TestBuildIdempotentInsertSQL_MySQL(t *testing.T) { + got, err := BuildIdempotentInsertSQL("mysql", "t", []string{"id", "name"}) + if err != nil { + t.Fatal(err) + } + want := "INSERT IGNORE INTO `t` (`id`,`name`) VALUES (?,?)" + if got != want { + t.Errorf("mysql idempotent INSERT:\n got %q\nwant %q", got, want) + } +} + +func TestBuildIdempotentInsertSQL_UnknownEngine(t *testing.T) { + if _, err := BuildIdempotentInsertSQL("oracle", "t", []string{"id"}); err == nil { + t.Fatal("BuildIdempotentInsertSQL unknown engine: want error, got nil") + } +} + func TestBuildInsertSQL_EmptyColumns(t *testing.T) { if _, err := BuildInsertSQL("postgres", "t", nil); err == nil { t.Fatal("BuildInsertSQL with no columns: want error, got nil") diff --git a/internal/driver/_mysqlcommon/canonical.go b/internal/driver/_mysqlcommon/canonical.go index f4336a3..2572a93 100644 --- a/internal/driver/_mysqlcommon/canonical.go +++ b/internal/driver/_mysqlcommon/canonical.go @@ -137,7 +137,7 @@ func (c *Conn) ApplyChange(ctx context.Context, ch canonical.CanonicalChange) er cols = append(cols, col) } sort.Strings(cols) - stmt, err := canonical.BuildInsertSQL(c.engine, ch.Table, cols) + stmt, err := canonical.BuildIdempotentInsertSQL(c.engine, ch.Table, cols) if err != nil { return err } diff --git a/internal/driver/_mysqlcommon/incremental.go b/internal/driver/_mysqlcommon/incremental.go index 567dcc5..5b5b487 100644 --- a/internal/driver/_mysqlcommon/incremental.go +++ b/internal/driver/_mysqlcommon/incremental.go @@ -20,6 +20,19 @@ type BinlogPosition struct { Position uint64 } +var _ driver.BasePositioner = (*Conn)(nil) + +// CurrentPosition returns the server's current binlog coordinates as a canonical +// Position. app.Backup calls this right after a full backup so the base dump's +// Envelope records where the first incremental should resume from. +func (c *Conn) CurrentPosition(ctx context.Context) (canonical.Position, error) { + pos, err := c.CaptureBinlogPosition(ctx) + if err != nil { + return canonical.Position{}, err + } + return canonical.Position{BinlogFile: pos.File, BinlogPos: pos.Position}, nil +} + // CaptureBinlogPosition records the current binlog coordinates. Tries the // MySQL 8.4+ statement first, then the pre-8.4 form, so it works across // versions and both forks. diff --git a/internal/driver/driver.go b/internal/driver/driver.go index 30471b6..6536dcd 100644 --- a/internal/driver/driver.go +++ b/internal/driver/driver.go @@ -52,6 +52,18 @@ type ChangeStreamer interface { StreamChanges(ctx context.Context, from canonical.Position, emit func(canonical.CanonicalChange) error) (canonical.Position, error) } +// BasePositioner reports the engine's current change-stream position, captured +// during a full backup so a later incremental knows where to resume from. +// +// app.Backup calls this immediately after a full backup completes and stamps the +// result into the base dump's Envelope. basePosition() then reads a real position +// instead of the zero value, so the first incremental off a full base resumes +// from base-end rather than silently starting at "now" (which would drop every +// change committed between the base dump and the first incremental run). +type BasePositioner interface { + CurrentPosition(ctx context.Context) (canonical.Position, error) +} + // IncrementalBackuper is an optional Conn capability: capture the BOUNDED change // set from `since` to the engine's current end position, serializing each // CanonicalChange to w as JSONL, and return the end Position reached. diff --git a/internal/driver/postgres/canonical.go b/internal/driver/postgres/canonical.go index bec0668..b69184d 100644 --- a/internal/driver/postgres/canonical.go +++ b/internal/driver/postgres/canonical.go @@ -137,7 +137,7 @@ func (c *Conn) ApplyChange(ctx context.Context, ch canonical.CanonicalChange) er cols = append(cols, col) } sort.Strings(cols) - stmt, err := canonical.BuildInsertSQL("postgres", ch.Table, cols) + stmt, err := canonical.BuildIdempotentInsertSQL("postgres", ch.Table, cols) if err != nil { return err } diff --git a/internal/driver/postgres/incremental_change.go b/internal/driver/postgres/incremental_change.go index 1844a70..2c75350 100644 --- a/internal/driver/postgres/incremental_change.go +++ b/internal/driver/postgres/incremental_change.go @@ -12,6 +12,18 @@ import ( ) var _ driver.IncrementalBackuper = (*Conn)(nil) +var _ driver.BasePositioner = (*Conn)(nil) + +// CurrentPosition returns the server's current WAL position via +// pg_current_wal_lsn(). app.Backup calls this right after a full backup so the +// base dump's Envelope records where the first incremental should resume from. +func (c *Conn) CurrentPosition(ctx context.Context) (canonical.Position, error) { + var lsn string + if err := c.db.QueryRowContext(ctx, "SELECT pg_current_wal_lsn()::text").Scan(&lsn); err != nil { + return canonical.Position{}, &errs.Error{Op: "postgres.current_position", Code: errs.CodeSystem, Cause: err} + } + return canonical.Position{LSN: lsn}, nil +} // BackupIncremental captures the BOUNDED change set from `since` to the server's // current end LSN, serializing each CanonicalChange to w as JSONL, and returns From eb706ee1abc0ac42942b0a4e0db232a73bfff826 Mon Sep 17 00:00:00 2001 From: Nikhil Rajput Date: Tue, 23 Jun 2026 04:08:12 +0530 Subject: [PATCH 09/13] feat: real CDC streaming, same + cross-engine - Replace RunCDC polling scaffold with unbounded StreamChanges to ApplyChange on the target; works same-engine and cross-engine via engine-neutral CanonicalChange. - Add initial snapshot to stream handoff: capture source position before snapshot, then stream changes committed after it. - Resume from a state file keyed by a stable per source/target job ID; checkpoint every 100 changes plus on clean exit. At-least-once delivery is safe because ApplyChange is idempotent. - Route sync --continuous to RunCDC; add siphon cdc command and register it in root. - Flip CDC capability to true on postgres, mysql, mariadb. - Add integration tests: same-engine, cross-engine, and resume. - Update docs/CDC.md, README, CHANGELOG to reflect shipped CDC. --- CHANGELOG.md | 3 +- README.md | 4 +- docs/CDC.md | 94 +++++--- internal/app/cdc.go | 223 +++++++++++++++-- internal/app/cdc_integration_test.go | 345 +++++++++++++++++++++++++++ internal/app/sync.go | 12 +- internal/cli/cdc.go | 45 ++++ internal/cli/root.go | 1 + internal/cli/sync.go | 2 +- internal/driver/mariadb/driver.go | 2 +- internal/driver/mysql/driver.go | 2 +- internal/driver/postgres/driver.go | 2 +- 12 files changed, 659 insertions(+), 76 deletions(-) create mode 100644 internal/app/cdc_integration_test.go create mode 100644 internal/cli/cdc.go diff --git a/CHANGELOG.md b/CHANGELOG.md index 8d9837a..c0f9725 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -25,4 +25,5 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Phase F advanced-transfer machinery (some CLI paths gated pending follow-up wiring — see the `docs/INCREMENTAL.md`, `docs/CROSS_ENGINE.md`, `docs/CDC.md` Status sections): - **Working today:** every dump is prefixed with a 4 KB `SIPH` JSON envelope (type, base/parent IDs, WAL/binlog positions, checksum); `restore` resolves the base→incremental chain and applies it in order, with `--up-to ` to stop early (cycle + broken-chain detection). Sync now streams through a bounded `jobs.Stream` (default 64×1 MB) instead of `io.Pipe`, exposing a `FillPercent` backpressure metric and propagating backup failures to the restore side via `CloseErr` (no truncated dump committed as clean). - **Incremental backup wired end-to-end (`backup --incremental --base `):** captures a **bounded** change set since the base dump's recorded end position, serialized as engine-neutral JSONL `CanonicalChange` records. Postgres bounds the pgoutput logical-decoding stream by `pg_current_wal_lsn()` captured at backup time; MySQL/MariaDB bound the binlog-tool decode by the current binlog file+offset. The incremental dump's envelope carries this capture's end position so the next incremental resumes exactly there. `restore` replays incremental links via `ApplyChange` (base links still restore natively). Postgres adds an orphan replication-slot sweep (`SweepOrphanSlots`) run before each capture — drops inactive `siphon_*` physical slots while preserving the persistent logical resume slot. `Incremental` capability is now `true` for all three drivers. Live-server behavior is integration-tested in CI (`wal_level=logical`); compile-checked locally. - - **Machinery in place, CLI gated (follow-up):** cross-engine type-mapping (canonical schema + JSONL emit/consume with per-engine identifier quoting and placeholders) — `sync --cross-engine` is capability-gated off until typed schema introspection exists; CDC continuous mode (state-file persistence + resume + capability gating) ships as a polling scaffold, not real logical-replication streaming. + - **CDC continuous sync wired end-to-end (`siphon cdc ` / `sync --continuous`):** unbounded `ChangeStreamer.StreamChanges` → `CanonicalTransfer.ApplyChange` on the target, with an initial schema+data snapshot→stream handoff on first run (consistent start position captured before the snapshot) and resume-from-saved-position on restart (stable per-(source,target) job ID). Works same-engine and cross-engine (changes are engine-neutral `CanonicalChange`s). Checkpoints state every 100 changes plus on clean exit; at-least-once delivery is safe because `ApplyChange` is idempotent (INSERT upsert, UPDATE/DELETE by PK). ctx cancel is the normal clean stop. `CDC` capability is now `true` for all three drivers. Same-engine, cross-engine, and resume paths are integration-tested in CI. + - **Machinery in place, CLI gated (follow-up):** cross-engine snapshot type-mapping (canonical schema + JSONL emit/consume with per-engine identifier quoting and placeholders) — `sync --cross-engine` is capability-gated off until typed schema introspection exists. diff --git a/README.md b/README.md index 8e00469..e68755c 100644 --- a/README.md +++ b/README.md @@ -54,7 +54,7 @@ A single binary that turns the painful, error-prone sprawl of `pg_dump` → `pg_ | **C** — TUI dashboard | Multi-panel Bubble Tea dashboard (profiles · dumps · jobs) with live job progress, backup/restore modal forms, and snapshot tests | ✅ Complete | | **D** — Driver hardening | Shared cross-driver test harness (`RunDriverSuite`), capability-gating helper (`RequireCapability`), Postgres connection-probe retry, and a `docs/DRIVERS.md` contributor guide | ✅ Complete | | **E** — MySQL + MariaDB | Both drivers via a shared `_mysqlcommon` package (shared `Conn`, `mysqldump`/`mariadb-dump` backup, client-pipe restore), exercised by the Phase D `RunDriverSuite` harness | ✅ Complete | -| **F** — Advanced transfer | Bounded-buffer streaming sync and incremental backup/restore work today: `backup --incremental --base ` captures a bounded change set (Postgres logical decoding / MySQL-MariaDB binlog) and `restore` replays the base→incremental chain, with Postgres orphan replication-slot sweep. Cross-engine type-mapping and CDC machinery are in place but their CLI paths (`sync --cross-engine`, CDC) remain gated pending follow-up wiring — see [docs/INCREMENTAL.md](docs/INCREMENTAL.md), [docs/CROSS_ENGINE.md](docs/CROSS_ENGINE.md), [docs/CDC.md](docs/CDC.md) | 🟡 Partial | +| **F** — Advanced transfer | Bounded-buffer streaming sync and incremental backup/restore work today: `backup --incremental --base ` captures a bounded change set (Postgres logical decoding / MySQL-MariaDB binlog) and `restore` replays the base→incremental chain, with Postgres orphan replication-slot sweep. CDC continuous sync runs end-to-end (`siphon cdc ` / `sync --continuous`): unbounded change streaming with an initial snapshot→stream handoff, resumable via a state file, same-engine and cross-engine. Cross-engine snapshot type-mapping machinery is in place but `sync --cross-engine` remains gated pending typed introspection — see [docs/INCREMENTAL.md](docs/INCREMENTAL.md), [docs/CROSS_ENGINE.md](docs/CROSS_ENGINE.md), [docs/CDC.md](docs/CDC.md) | 🟡 Partial | | **G** — Ops features | Cloud storage, secret backends, profile groups + 2FA, team mode, audit log, retention, telemetry | ⏳ Planned | | **H** — Distribution | GoReleaser, Homebrew tap, Scoop bucket, install script, docs site | ⏳ Planned | @@ -222,7 +222,7 @@ Contributions are welcome. Please read [CONTRIBUTING.md](CONTRIBUTING.md), keep Adding a new database engine? See [docs/DRIVERS.md](docs/DRIVERS.md) for the driver contributor guide. -Concept docs (honest as-built status — several paths are documented follow-ups): [docs/INCREMENTAL.md](docs/INCREMENTAL.md) (incremental backup + restore work end-to-end), [docs/CROSS_ENGINE.md](docs/CROSS_ENGINE.md) (type-map matrix; `--cross-engine` is gated off pending typed introspection), and [docs/CDC.md](docs/CDC.md) (scaffold only; not yet runnable). +Concept docs (honest as-built status — several paths are documented follow-ups): [docs/INCREMENTAL.md](docs/INCREMENTAL.md) (incremental backup + restore work end-to-end), [docs/CROSS_ENGINE.md](docs/CROSS_ENGINE.md) (type-map matrix; `--cross-engine` is gated off pending typed introspection), and [docs/CDC.md](docs/CDC.md) (continuous CDC sync runs end-to-end, same- and cross-engine). ## License diff --git a/docs/CDC.md b/docs/CDC.md index bd381d4..b7d154e 100644 --- a/docs/CDC.md +++ b/docs/CDC.md @@ -2,19 +2,21 @@ CDC ("continuous sync" / follow mode) keeps a target database in step with a source by streaming changes as they happen, rather than re-running a full -backup/restore. In siphon this is **scaffolding today** — the resume-state -plumbing and capability gating are in place, but continuous streaming is a -future deliverable and does not run. +backup/restore. siphon streams the source's logical change stream and applies +each change to the target continuously, until interrupted. It works +**same-engine** (Postgres→Postgres, MySQL→MySQL) and **cross-engine** +(Postgres→MySQL) alike, because changes are carried as engine-neutral +`CanonicalChange`s and replayed natively on the target. ## Table of contents -- [What exists](#what-exists) +- [How it works](#how-it-works) - [The CLI surface](#the-cli-surface) - [Status](#status) - [Resume state](#resume-state) - [Limitations](#limitations) -## What exists +## How it works `internal/app/cdc.go` holds: @@ -24,49 +26,60 @@ future deliverable and does not run. - **`saveCDCState` / `loadCDCState`** — JSON state files under a per-user directory resolved from `SIPHON_STATE_HOME`, then `XDG_STATE_HOME`, then `$HOME/.local/state/siphon/cdc`. -- **`RunCDC`** — a capability-gated entry point. It resolves the source and - target profiles, then requires both drivers to advertise `CapCDC`. Its body is - a **polling scaffold**: a 10-second ticker that persists state each tick and - resumes from a prior run's state on restart. It is **not** real change - streaming. +- **`RunCDC`** — the capability-gated entry point. Both drivers must advertise + `CapCDC`. It connects source and target, then: + 1. **First run (no prior state):** captures the source's current stream + position (`BasePositioner.CurrentPosition`), takes an initial schema+data + snapshot via the canonical transfer pipe (`InspectSchema` → + `EmitCanonical` → `ConsumeCanonical`), persists the start position, then + streams changes committed after it. + 2. **Restart (prior state exists):** resumes from the saved position, no + snapshot. + 3. **Stream loop:** `ChangeStreamer.StreamChanges` (unbounded) emits each + `CanonicalChange`; `RunCDC` applies it via `CanonicalTransfer.ApplyChange` + on the target and checkpoints state periodically. + +The `job_id` is a stable hash of `(from, to)`, so re-running the same continuous +sync resumes from the same state file. `CDCState` save/load round-trip, the state-dir resolution, and the -no-capability rejection are unit-tested (`internal/app/cdc_test.go`). +no-capability rejection are unit-tested (`internal/app/cdc_test.go`). End-to-end +streaming (same-engine, cross-engine, and resume) is covered by integration +tests (`internal/app/cdc_integration_test.go`, `-tags integration`). ## The CLI surface -There is **no `siphon cdc` subcommand wired today.** `RunCDC` exists as an -application function but is not registered as a Cobra command -(`internal/cli/root.go` wires `backup`, `restore`, `sync`, `verify`, `inspect`, -`profile`, `dumps`, `config`, `schedule`, `tunnel` — no `cdc`). - -`sync --continuous` exposes the flag but does not run CDC; it returns a clear -error pointing at the (not-yet-wired) `siphon cdc` follow-up: +Two equivalent entry points: ```bash +# Dedicated command +siphon cdc +siphon cdc --from pg-prod --to pg-replica + +# sync follow mode (equivalent) siphon sync --from pg-prod --to pg-replica --continuous -# Error: continuous CDC sync is not wired here; use `siphon cdc` (Phase F Task 10) ``` +Both stream continuously until interrupted. Press Ctrl-C to stop cleanly — +ctx cancellation is the normal termination signal; the final position is +persisted on exit so a later run resumes without a gap. + ## Status | Capability | Status | | --- | --- | | `CDCState` persistence (save/load) | ✅ Works (unit-tested) | | State-dir resolution (`SIPHON_STATE_HOME` / `XDG_STATE_HOME`) | ✅ Works (unit-tested) | -| Resume from prior state on restart | ✅ Works (in the scaffold loop) | -| Capability gating on `CapCDC` | ✅ Works | -| `siphon cdc` CLI subcommand | ❌ Not wired (no Cobra command) | -| Continuous change streaming | ⚠️ Scaffold only (polling tick; not real CDC) | - -CDC does **not run today**. No driver declares `CapCDC` true, so `RunCDC` is -rejected with `ErrDriverUnsupported`. Even if a driver enabled it, the loop is a -polling scaffold, not logical-replication streaming. +| Resume from prior state on restart | ✅ Works | +| Capability gating on `CapCDC` | ✅ Works (true on postgres, mysql, mariadb) | +| `siphon cdc` CLI subcommand | ✅ Wired | +| `sync --continuous` follow mode | ✅ Routes to `RunCDC` | +| Initial snapshot → stream handoff | ✅ Works | +| Continuous change streaming (same + cross-engine) | ✅ Works | ## Resume state -When CDC streaming is built, `RunCDC` resumes from the last persisted position. -State files live at: +`RunCDC` resumes from the last persisted position. State files live at: ```text $SIPHON_STATE_HOME/cdc/.state # if SIPHON_STATE_HOME is set @@ -77,12 +90,21 @@ $HOME/.local/state/siphon/cdc/.state # default Each file is JSON: source/target profiles plus the last applied position (LSN for Postgres, binlog file + offset for MySQL/MariaDB). +**Checkpoint/resume granularity is "since the last checkpoint."** RunCDC +checkpoints every 100 applied changes plus on clean exit. After a crash it +resumes from the last checkpoint and re-applies any changes that landed after +it. This is safe because delivery is **at-least-once** and `ApplyChange` is +**idempotent**: INSERT is idempotent (upsert), and UPDATE/DELETE target by +primary key. Re-applying the tail is therefore a no-op — no gaps, no duplicates. + ## Limitations -- **Not enabled on any driver** — `CapCDC` is `false` everywhere, so CDC is - rejected up front. -- **No real streaming** — the scaffold polls on a ticker. Real logical- - replication streaming (`pglogrepl` for Postgres, binlog tailing for - MySQL/MariaDB) is deferred to a future revision. -- **No CLI entry point** — `RunCDC` is internal application scaffolding, not a - user-facing command yet. +- **Postgres source requires `wal_level=logical`** and sufficient + `max_replication_slots` / `max_wal_senders`; MySQL/MariaDB source requires + row-based binlogging. These are the same prerequisites as incremental backup. +- **At-least-once, not exactly-once** — see resume granularity above. Correct + because apply is idempotent; a target with non-idempotent side effects + (triggers, etc.) is out of scope. +- **Snapshot consistency window** — the start position is captured before the + snapshot, so changes committed during the snapshot are re-streamed and + re-applied idempotently rather than lost. diff --git a/internal/app/cdc.go b/internal/app/cdc.go index 891454b..8f4b4bb 100644 --- a/internal/app/cdc.go +++ b/internal/app/cdc.go @@ -2,11 +2,17 @@ package app import ( "context" + "crypto/sha256" + "encoding/hex" "encoding/json" + "fmt" "os" "path/filepath" "time" + "github.com/nixrajput/siphon/internal/canonical" + "github.com/nixrajput/siphon/internal/driver" + "github.com/nixrajput/siphon/internal/errs" "github.com/nixrajput/siphon/internal/jobs" ) @@ -22,6 +28,27 @@ type CDCState struct { UpdatedAt time.Time `json:"updated_at"` } +// position renders the resume cursor stored in this state as a canonical.Position. +func (s *CDCState) position() canonical.Position { + return canonical.Position{ + LSN: s.LastAppliedLSN, + BinlogFile: s.LastBinlogFile, + BinlogPos: s.LastBinlogPos, + } +} + +// setPosition records p as the new resume cursor. +func (s *CDCState) setPosition(p canonical.Position) { + s.LastAppliedLSN = p.LSN + s.LastBinlogFile = p.BinlogFile + s.LastBinlogPos = p.BinlogPos +} + +// hasPosition reports whether any resume cursor has been recorded. +func (s *CDCState) hasPosition() bool { + return s.LastAppliedLSN != "" || s.LastBinlogFile != "" +} + // CDCStateDir returns the per-user directory holding CDC resume state. It // honors SIPHON_STATE_HOME, then XDG_STATE_HOME, then $HOME/.local/state — // mirroring how internal/config resolves its config path, so tests can redirect @@ -58,17 +85,46 @@ func loadCDCState(jobID string) (*CDCState, error) { return s, json.Unmarshal(body, s) } -// RunCDC starts (or resumes) a continuous sync. It is capability-gated: both -// the source and target driver must advertise CapCDC. No driver does today -// (CDC streaming via logical replication / binlog tailing is a Phase F -// follow-up), so this returns ErrDriverUnsupported — the honest scaffold state. -// When a driver gains CDC support, the polling loop below is replaced by real -// logical-replication streaming (pglogrepl for Postgres). +// cdcJobID derives a stable per-(source,target) identifier so a restart of the +// same continuous sync resumes from the same state file. A short hash keeps the +// filename safe regardless of profile-name characters. +func cdcJobID(from, to string) string { + sum := sha256.Sum256([]byte(from + "\x00" + to)) + return "cdc-" + hex.EncodeToString(sum[:8]) +} + +// cdcCheckpointEvery bounds how many applied changes pass between resume-state +// checkpoints. At-least-once delivery plus idempotent ApplyChange makes coarse +// checkpointing safe: on restart we resume from the last checkpoint and re-apply +// any changes that landed after it, which is a no-op for INSERT (idempotent), +// UPDATE, and DELETE (both by primary key). +const cdcCheckpointEvery = 100 + +// RunCDC starts (or resumes) a continuous, unbounded sync: it tails the source's +// logical change stream and applies each engine-neutral CanonicalChange to the +// target. It works same-engine and cross-engine alike because CanonicalChange is +// engine-neutral and ApplyChange replays it natively on the target. +// +// Both drivers must advertise CapCDC. On a first run (no prior state) it captures +// a consistent start position, takes an initial schema+data snapshot, then streams +// changes committed after that position. On a restart it resumes from the saved +// position with no snapshot. +// +// Resume granularity is "since the last checkpoint": StreamChanges reports the +// position reached after each applied change via the emit callback, and RunCDC +// checkpoints every cdcCheckpointEvery changes plus on clean exit. Re-applying the +// tail after a crash is safe (see cdcCheckpointEvery). +// +// ctx cancellation is the normal stop signal (matching the bounded-stream +// convention): on cancel RunCDC persists final state and returns nil; only a +// non-cancel StreamChanges error is surfaced. func RunCDC(parent context.Context, d Deps, opt SyncOpts) (<-chan jobs.Event, string, error) { - if _, err := d.Profiles.Resolve(opt.From); err != nil { + src, err := d.Profiles.Resolve(opt.From) + if err != nil { return nil, "", err } - if _, err := d.Profiles.Resolve(opt.To); err != nil { + dst, err := d.Profiles.Resolve(opt.To) + if err != nil { return nil, "", err } if err := RequireCapability(d, opt.From, CapCDC); err != nil { @@ -77,34 +133,151 @@ func RunCDC(parent context.Context, d Deps, opt SyncOpts) (<-chan jobs.Event, st if err := RequireCapability(d, opt.To, CapCDC); err != nil { return nil, "", err } + srcDrv, err := d.Drivers.Get(src.Driver) + if err != nil { + return nil, "", err + } + dstDrv, err := d.Drivers.Get(dst.Driver) + if err != nil { + return nil, "", err + } + + jobID := cdcJobID(opt.From, opt.To) return d.Runner.Run(parent, jobs.Job{ Stage: "cdc", Func: func(ctx context.Context, emit func(jobs.Event)) error { emit(jobs.Event{Phase: jobs.PhaseProgress, Message: "CDC mode started"}) - // Phase F scaffold: a polling heartbeat that persists state each tick. - // A future revision replaces this with real logical-replication - // streaming (pglogrepl for Postgres, binlog tailing for MySQL/MariaDB). - const jobID = "cdc" + + srcConn, err := srcDrv.Connect(ctx, src) + if err != nil { + return err + } + defer func() { _ = srcConn.Close() }() + + dstConn, err := dstDrv.Connect(ctx, dst) + if err != nil { + return err + } + defer func() { _ = dstConn.Close() }() + + srcStreamer, ok := srcConn.(driver.ChangeStreamer) + if !ok { + return cdcUnsupported(src.Driver, "ChangeStreamer") + } + srcPositioner, ok := srcConn.(driver.BasePositioner) + if !ok { + return cdcUnsupported(src.Driver, "BasePositioner") + } + dstXfer, ok := dstConn.(driver.CanonicalTransfer) + if !ok { + return cdcUnsupported(dst.Driver, "CanonicalTransfer") + } + + // Resume from prior state when present; otherwise capture a consistent + // start position and take an initial snapshot before streaming. state := &CDCState{JobID: jobID, Source: opt.From, Target: opt.To} - // Resume from a prior run's persisted position when one exists. - if prev, err := loadCDCState(jobID); err == nil { + var from canonical.Position + if prev, loadErr := loadCDCState(jobID); loadErr == nil && prev.hasPosition() { state = prev + from = prev.position() + emit(jobs.Event{Phase: jobs.PhaseProgress, Message: "CDC resuming from saved position"}) + } else { + from, err = srcPositioner.CurrentPosition(ctx) + if err != nil { + return err + } + if snapErr := cdcSnapshot(ctx, srcConn, dstXfer, src.Driver); snapErr != nil { + return snapErr + } + state.setPosition(from) + if saveErr := saveCDCState(state); saveErr != nil { + return saveErr + } + emit(jobs.Event{Phase: jobs.PhaseProgress, Message: "CDC initial snapshot complete; streaming changes"}) } - ticker := time.NewTicker(10 * time.Second) - defer ticker.Stop() - for { - select { - case <-ctx.Done(): - _ = saveCDCState(state) - return ctx.Err() - case <-ticker.C: - if err := saveCDCState(state); err != nil { - return err + + applied := 0 + emit2 := func(ch canonical.CanonicalChange) error { + if applyErr := dstXfer.ApplyChange(ctx, ch); applyErr != nil { + return applyErr + } + applied++ + emit(jobs.Event{ + Phase: jobs.PhaseProgress, + Message: fmt.Sprintf("applied %s on %s (%d total)", ch.Op, ch.Table, applied), + }) + if applied%cdcCheckpointEvery == 0 { + if pos, posErr := srcPositioner.CurrentPosition(ctx); posErr == nil { + state.setPosition(pos) + _ = saveCDCState(state) } - emit(jobs.Event{Phase: jobs.PhaseProgress, Message: "CDC tick"}) } + return nil + } + + finalPos, streamErr := srcStreamer.StreamChanges(ctx, from, emit2) + + // On any exit, persist the furthest position we know about. The + // streamer returns the final position even on ctx-cancel. + if finalPos.LSN != "" || finalPos.BinlogFile != "" { + state.setPosition(finalPos) + } + _ = saveCDCState(state) + + // ctx cancellation is the normal stop signal — report clean. + if ctx.Err() != nil { + emit(jobs.Event{Phase: jobs.PhaseProgress, Message: "CDC stopped"}) + return nil } + return streamErr }, }) } + +// cdcSnapshot performs the initial schema+data copy from source to target using +// the canonical transfer pipe (the same pattern as runCrossEngineSync). Both the +// source's SchemaInspector+CanonicalTransfer and the target's CanonicalTransfer +// are required. +func cdcSnapshot(ctx context.Context, srcConn driver.Conn, dstXfer driver.CanonicalTransfer, srcDriverName string) error { + srcInsp, ok := srcConn.(driver.SchemaInspector) + if !ok { + return cdcUnsupported(srcDriverName, "SchemaInspector") + } + srcXfer, ok := srcConn.(driver.CanonicalTransfer) + if !ok { + return cdcUnsupported(srcDriverName, "CanonicalTransfer") + } + + schema, err := srcInsp.InspectSchema(ctx) + if err != nil { + return err + } + + stream := jobs.NewStream(64) + errCh := make(chan error, 1) + + go func() { + emitErr := srcXfer.EmitCanonical(ctx, schema, stream) + _ = stream.CloseErr(emitErr) + errCh <- emitErr + }() + + consumeErr := dstXfer.ConsumeCanonical(ctx, stream) + _ = stream.Close() + emitErr := <-errCh + + if emitErr != nil { + return emitErr + } + return consumeErr +} + +func cdcUnsupported(driverName, iface string) error { + return &errs.Error{ + Op: "cdc", + Code: errs.CodeUser, + Cause: errs.ErrDriverUnsupported, + Hint: driverName + " driver does not implement " + iface, + } +} diff --git a/internal/app/cdc_integration_test.go b/internal/app/cdc_integration_test.go new file mode 100644 index 0000000..6147981 --- /dev/null +++ b/internal/app/cdc_integration_test.go @@ -0,0 +1,345 @@ +//go:build integration + +package app + +import ( + "context" + "database/sql" + "fmt" + "testing" + "time" + + tc "github.com/testcontainers/testcontainers-go" + pgctr "github.com/testcontainers/testcontainers-go/modules/postgres" + + _ "github.com/jackc/pgx/v5/stdlib" + + mysqlcommon "github.com/nixrajput/siphon/internal/driver/_mysqlcommon" + + "github.com/nixrajput/siphon/internal/config" + "github.com/nixrajput/siphon/internal/driver" + "github.com/nixrajput/siphon/internal/dumps" + "github.com/nixrajput/siphon/internal/jobs" + "github.com/nixrajput/siphon/internal/profile" + "github.com/nixrajput/siphon/internal/secrets" +) + +// startLogicalPG starts a Postgres 16 container configured for logical decoding +// (wal_level=logical), the prerequisite for CDC StreamChanges. +func startLogicalPG(t *testing.T) (host string, port int, cleanup func()) { + t.Helper() + ctx := context.Background() + c, err := pgctr.Run(ctx, "postgres:16-alpine", + pgctr.WithDatabase("test"), + pgctr.WithUsername("postgres"), + pgctr.WithPassword("postgres"), + pgctr.BasicWaitStrategies(), + tc.WithCmdArgs("-c", "wal_level=logical", "-c", "max_wal_senders=10", "-c", "max_replication_slots=10"), + ) + if err != nil { + t.Fatalf("start postgres container: %v", err) + } + h, err := c.Host(ctx) + if err != nil { + _ = c.Terminate(ctx) + t.Fatalf("postgres container host: %v", err) + } + p, err := c.MappedPort(ctx, "5432/tcp") + if err != nil { + _ = c.Terminate(ctx) + t.Fatalf("postgres container port: %v", err) + } + return h, int(p.Num()), func() { _ = c.Terminate(ctx) } +} + +func openPG(t *testing.T, host string, port int) *sql.DB { + t.Helper() + dsn := fmt.Sprintf("host=%s port=%d user=postgres password=postgres dbname=test sslmode=disable", host, port) + db, err := sql.Open("pgx", dsn) + if err != nil { + t.Fatalf("open postgres: %v", err) + } + return db +} + +// pollUntil polls fn every 250ms until it returns true or the timeout elapses. +func pollUntil(t *testing.T, timeout time.Duration, fn func() bool) bool { + t.Helper() + deadline := time.Now().Add(timeout) + for time.Now().Before(deadline) { + if fn() { + return true + } + time.Sleep(250 * time.Millisecond) + } + return fn() +} + +// drainForError drains the job event channel in the background and fails the +// test on any Error event. It does not block — CDC runs unbounded. +func drainForError(t *testing.T, ch <-chan jobs.Event) { + t.Helper() + go func() { + for ev := range ch { + if ev.Err != nil { + t.Errorf("CDC job error: %v", ev.Err) + } + } + }() +} + +// TestCDC_SameEngine_PostgresToPostgres is the solid, primary CDC test: it runs +// RunCDC PG→PG, then performs INSERT/UPDATE/DELETE on the source and asserts each +// change is applied to the target within a timeout, then cancels (clean stop). +func TestCDC_SameEngine_PostgresToPostgres(t *testing.T) { + t.Setenv("SIPHON_STATE_HOME", t.TempDir()) + + srcHost, srcPort, srcCleanup := startLogicalPG(t) + defer srcCleanup() + dstHost, dstPort, dstCleanup := startLogicalPG(t) + defer dstCleanup() + + srcDB := openPG(t, srcHost, srcPort) + defer func() { _ = srcDB.Close() }() + dstDB := openPG(t, dstHost, dstPort) + defer func() { _ = dstDB.Close() }() + + ctx := context.Background() + // Seed source + target with the same empty schema so the snapshot + apply + // have a target table to write into. + for _, db := range []*sql.DB{srcDB, dstDB} { + if _, err := db.ExecContext(ctx, + `CREATE TABLE widgets(id int primary key, name text);`); err != nil { + t.Fatalf("create table: %v", err) + } + } + if _, err := srcDB.ExecContext(ctx, `INSERT INTO widgets VALUES (1,'seed')`); err != nil { + t.Fatalf("seed insert: %v", err) + } + + deps := cdcDeps(t, pgProfile(srcHost, srcPort), pgProfile(dstHost, dstPort)) + + runCtx, cancel := context.WithCancel(ctx) + ch, _, err := RunCDC(runCtx, deps, SyncOpts{From: "src", To: "dst"}) + if err != nil { + t.Fatalf("RunCDC setup: %v", err) + } + drainForError(t, ch) + + // The snapshot row should land on the target. + if !pollUntil(t, 60*time.Second, func() bool { + var n int + _ = dstDB.QueryRowContext(ctx, "SELECT COUNT(*) FROM widgets WHERE id=1").Scan(&n) + return n == 1 + }) { + cancel() + t.Fatal("snapshot row id=1 never appeared on target") + } + + // Give the streamer a moment to establish its slot, then apply DML. + time.Sleep(2 * time.Second) + if _, err := srcDB.ExecContext(ctx, `INSERT INTO widgets VALUES (2,'wrench')`); err != nil { + t.Fatalf("stream insert: %v", err) + } + if !pollUntil(t, 30*time.Second, func() bool { + var name string + _ = dstDB.QueryRowContext(ctx, "SELECT name FROM widgets WHERE id=2").Scan(&name) + return name == "wrench" + }) { + cancel() + t.Fatal("streamed INSERT id=2 never appeared on target") + } + + if _, err := srcDB.ExecContext(ctx, `UPDATE widgets SET name='spanner' WHERE id=2`); err != nil { + t.Fatalf("stream update: %v", err) + } + if !pollUntil(t, 30*time.Second, func() bool { + var name string + _ = dstDB.QueryRowContext(ctx, "SELECT name FROM widgets WHERE id=2").Scan(&name) + return name == "spanner" + }) { + cancel() + t.Fatal("streamed UPDATE id=2 never propagated") + } + + if _, err := srcDB.ExecContext(ctx, `DELETE FROM widgets WHERE id=2`); err != nil { + t.Fatalf("stream delete: %v", err) + } + if !pollUntil(t, 30*time.Second, func() bool { + var n int + _ = dstDB.QueryRowContext(ctx, "SELECT COUNT(*) FROM widgets WHERE id=2").Scan(&n) + return n == 0 + }) { + cancel() + t.Fatal("streamed DELETE id=2 never propagated") + } + + cancel() // clean stop — ctx cancel is the normal CDC termination +} + +// TestCDC_Resume runs CDC, lets a streamed change land, cancels, then restarts +// RunCDC with the same (src,dst) — which derives the same jobID — and applies +// more changes. It asserts the resume picks up new changes (and at-least-once +// re-apply of any tail is harmless because ApplyChange is idempotent). +func TestCDC_Resume(t *testing.T) { + t.Setenv("SIPHON_STATE_HOME", t.TempDir()) + + srcHost, srcPort, srcCleanup := startLogicalPG(t) + defer srcCleanup() + dstHost, dstPort, dstCleanup := startLogicalPG(t) + defer dstCleanup() + + srcDB := openPG(t, srcHost, srcPort) + defer func() { _ = srcDB.Close() }() + dstDB := openPG(t, dstHost, dstPort) + defer func() { _ = dstDB.Close() }() + + ctx := context.Background() + for _, db := range []*sql.DB{srcDB, dstDB} { + if _, err := db.ExecContext(ctx, + `CREATE TABLE widgets(id int primary key, name text);`); err != nil { + t.Fatalf("create table: %v", err) + } + } + + deps := cdcDeps(t, pgProfile(srcHost, srcPort), pgProfile(dstHost, dstPort)) + + // First run: stream one change, then cancel. + runCtx1, cancel1 := context.WithCancel(ctx) + ch1, _, err := RunCDC(runCtx1, deps, SyncOpts{From: "src", To: "dst"}) + if err != nil { + t.Fatalf("RunCDC (1) setup: %v", err) + } + drainForError(t, ch1) + + time.Sleep(3 * time.Second) // let the slot establish + if _, err := srcDB.ExecContext(ctx, `INSERT INTO widgets VALUES (1,'first')`); err != nil { + t.Fatalf("first insert: %v", err) + } + if !pollUntil(t, 30*time.Second, func() bool { + var n int + _ = dstDB.QueryRowContext(ctx, "SELECT COUNT(*) FROM widgets WHERE id=1").Scan(&n) + return n == 1 + }) { + cancel1() + t.Fatal("first run: id=1 never propagated") + } + cancel1() + time.Sleep(2 * time.Second) // allow clean shutdown + state persist + + // Second run: same profiles → same jobID → resume. Apply another change. + runCtx2, cancel2 := context.WithCancel(ctx) + defer cancel2() + ch2, _, err := RunCDC(runCtx2, deps, SyncOpts{From: "src", To: "dst"}) + if err != nil { + t.Fatalf("RunCDC (2) setup: %v", err) + } + drainForError(t, ch2) + + time.Sleep(3 * time.Second) + if _, err := srcDB.ExecContext(ctx, `INSERT INTO widgets VALUES (2,'second')`); err != nil { + t.Fatalf("second insert: %v", err) + } + if !pollUntil(t, 30*time.Second, func() bool { + var n int + _ = dstDB.QueryRowContext(ctx, "SELECT COUNT(*) FROM widgets WHERE id=2").Scan(&n) + return n == 1 + }) { + t.Fatal("resume: id=2 never propagated after restart") + } + + // No gap/dup: id=1 must still be present exactly once (idempotent apply). + var n1 int + if err := dstDB.QueryRowContext(ctx, "SELECT COUNT(*) FROM widgets WHERE id=1").Scan(&n1); err != nil { + t.Fatalf("count id=1: %v", err) + } + if n1 != 1 { + t.Errorf("id=1 count after resume = %d, want 1 (no gap, no dup)", n1) + } +} + +// TestCDC_CrossEngine_PostgresToMySQL runs CDC PG→MySQL: an INSERT on Postgres +// must arrive on MySQL via canonical apply. +func TestCDC_CrossEngine_PostgresToMySQL(t *testing.T) { + t.Setenv("SIPHON_STATE_HOME", t.TempDir()) + + pgHost, pgPort, pgCleanup := startLogicalPG(t) + defer pgCleanup() + myHost, myPort, myCleanup := startIntegMySQL(t) + defer myCleanup() + + pgDB := openPG(t, pgHost, pgPort) + defer func() { _ = pgDB.Close() }() + + ctx := context.Background() + if _, err := pgDB.ExecContext(ctx, + `CREATE TABLE widgets(id int primary key, name text);`); err != nil { + t.Fatalf("pg create table: %v", err) + } + if _, err := pgDB.ExecContext(ctx, `INSERT INTO widgets VALUES (1,'seed')`); err != nil { + t.Fatalf("pg seed: %v", err) + } + + deps := cdcDeps(t, pgProfile(pgHost, pgPort), + config.ProfileConfig{Driver: "mysql", Host: myHost, Port: myPort, User: "root", Password: "rootpass", Database: "test"}) + + runCtx, cancel := context.WithCancel(ctx) + defer cancel() + ch, _, err := RunCDC(runCtx, deps, SyncOpts{From: "src", To: "dst"}) + if err != nil { + t.Fatalf("RunCDC setup: %v", err) + } + drainForError(t, ch) + + myProf := driver.Profile{Driver: "mysql", Host: myHost, Port: myPort, User: "root", Password: "rootpass", Database: "test", SSLMode: "disable"} + myDB, err := mysqlcommon.Open(myProf) + if err != nil { + t.Fatalf("open mysql: %v", err) + } + defer func() { _ = myDB.Close() }() + + // Snapshot row must land on MySQL. + if !pollUntil(t, 90*time.Second, func() bool { + var n int + _ = myDB.QueryRowContext(ctx, "SELECT COUNT(*) FROM widgets WHERE id=1").Scan(&n) + return n == 1 + }) { + t.Fatal("snapshot row id=1 never appeared on mysql") + } + + time.Sleep(2 * time.Second) + if _, err := pgDB.ExecContext(ctx, `INSERT INTO widgets VALUES (2,'wrench')`); err != nil { + t.Fatalf("pg stream insert: %v", err) + } + if !pollUntil(t, 30*time.Second, func() bool { + var name string + _ = myDB.QueryRowContext(ctx, "SELECT name FROM widgets WHERE id=2").Scan(&name) + return name == "wrench" + }) { + t.Fatal("streamed INSERT id=2 never crossed to mysql") + } +} + +// --- helpers --- + +func pgProfile(host string, port int) config.ProfileConfig { + return config.ProfileConfig{Driver: "postgres", Host: host, Port: port, User: "postgres", Password: "postgres", Database: "test", SSLMode: "disable"} +} + +// cdcDeps builds Deps with two profiles "src" and "dst". +func cdcDeps(t *testing.T, src, dst config.ProfileConfig) Deps { + t.Helper() + cfg := &config.Config{Profiles: map[string]config.ProfileConfig{"src": src, "dst": dst}} + res := secrets.NewResolver(secrets.Passthrough{}) + ps := profile.New(cfg, res, func(*config.Config) error { return nil }) + cat, err := dumps.NewCatalog(t.TempDir()) + if err != nil { + t.Fatalf("catalog: %v", err) + } + return Deps{ + Profiles: ps, + Dumps: cat, + Runner: jobs.NewRunner(), + Drivers: DefaultDrivers(), + } +} diff --git a/internal/app/sync.go b/internal/app/sync.go index 731f60c..f4cc1fb 100644 --- a/internal/app/sync.go +++ b/internal/app/sync.go @@ -17,8 +17,9 @@ type SyncOpts struct { // CrossEngine routes through the canonical-schema path (e.g. postgres→mysql) // instead of the native homogeneous stream. Gated on driver capability. CrossEngine bool - // Continuous requests CDC follow mode. Not yet available (Phase F ships CDC - // as an internal scaffold only). Setting it returns a clear CodeUser error. + // Continuous requests CDC follow mode: routes to RunCDC, which tails the + // source change stream and applies each change to the target (same-engine + // and cross-engine), resumable via the CDC state file. Continuous bool } @@ -33,12 +34,7 @@ type SyncOpts struct { // for typed cross-engine snapshot transfer. func Sync(parent context.Context, d Deps, opt SyncOpts) (<-chan jobs.Event, string, error) { if opt.Continuous { - return nil, "", &errs.Error{ - Op: "sync.continuous", - Code: errs.CodeUser, - Cause: errs.ErrDriverUnsupported, - Hint: "continuous CDC sync is not yet available (Phase F follow-up); see docs/CDC.md", - } + return RunCDC(parent, d, opt) } src, err := d.Profiles.Resolve(opt.From) diff --git a/internal/cli/cdc.go b/internal/cli/cdc.go new file mode 100644 index 0000000..3cf97a8 --- /dev/null +++ b/internal/cli/cdc.go @@ -0,0 +1,45 @@ +package cli + +import ( + "github.com/spf13/cobra" + + "github.com/nixrajput/siphon/internal/app" +) + +func newCdcCmd() *cobra.Command { + var fromName, toName string + cmd := &cobra.Command{ + Use: "cdc [from] [to]", + Short: "Continuously stream source changes to a target (CDC)", + Long: "cdc tails the source database's logical change stream and applies each " + + "change to the target, continuously, until interrupted. It works both " + + "same-engine and cross-engine (changes are translated through the canonical " + + "format). On first run it takes an initial snapshot, then follows changes; " + + "on restart it resumes from the last saved position. Press Ctrl-C to stop cleanly.", + Args: cobra.MaximumNArgs(2), + RunE: func(c *cobra.Command, args []string) error { + if len(args) >= 1 { + fromName = args[0] + } + if len(args) >= 2 { + toName = args[1] + } + deps, err := buildDeps() + if err != nil { + return err + } + ch, _, err := app.RunCDC(c.Context(), deps, app.SyncOpts{ + From: fromName, + To: toName, + Continuous: true, + }) + if err != nil { + return err + } + return Heartbeat(c.ErrOrStderr(), ch) + }, + } + cmd.Flags().StringVar(&fromName, "from", "", "Source profile") + cmd.Flags().StringVar(&toName, "to", "", "Target profile") + return cmd +} diff --git a/internal/cli/root.go b/internal/cli/root.go index 9693e0d..2763a58 100644 --- a/internal/cli/root.go +++ b/internal/cli/root.go @@ -51,6 +51,7 @@ func NewRoot(out, err io.Writer) *cobra.Command { newBackupCmd(), newRestoreCmd(), newSyncCmd(), + newCdcCmd(), newVerifyCmd(), newInspectCmd(), newProfileCmd(), diff --git a/internal/cli/sync.go b/internal/cli/sync.go index 0849eb7..781471a 100644 --- a/internal/cli/sync.go +++ b/internal/cli/sync.go @@ -44,6 +44,6 @@ func newSyncCmd() *cobra.Command { cmd.Flags().BoolVar(&stream, "stream", true, "Stream source→target without intermediate file") cmd.Flags().StringSliceVar(&tables, "table", nil, "Limit to these tables") cmd.Flags().BoolVar(&crossEngine, "cross-engine", false, "Translate between engines via canonical schema (requires cross-engine driver support; not yet available)") - cmd.Flags().BoolVar(&continuous, "continuous", false, "Continuously follow source changes (CDC) — not yet available; see docs/CDC.md") + cmd.Flags().BoolVar(&continuous, "continuous", false, "Continuously follow source changes (CDC); equivalent to the cdc command") return cmd } diff --git a/internal/driver/mariadb/driver.go b/internal/driver/mariadb/driver.go index 4c10f3d..f47d27d 100644 --- a/internal/driver/mariadb/driver.go +++ b/internal/driver/mariadb/driver.go @@ -28,7 +28,7 @@ func (Driver) Capabilities() driver.Capabilities { BinaryFormat: false, // SQL text dump, not a binary archive CrossEngineSource: true, CrossEngineTarget: true, - CDC: false, // Phase F + CDC: true, NativeBackpressure: true, // CrossVersionIncremental defaults false } diff --git a/internal/driver/mysql/driver.go b/internal/driver/mysql/driver.go index d9aad48..e2f8b8f 100644 --- a/internal/driver/mysql/driver.go +++ b/internal/driver/mysql/driver.go @@ -28,7 +28,7 @@ func (Driver) Capabilities() driver.Capabilities { BinaryFormat: false, // SQL text dump, not a binary archive CrossEngineSource: true, CrossEngineTarget: true, - CDC: false, // Phase F + CDC: true, NativeBackpressure: true, // CrossVersionIncremental defaults false } diff --git a/internal/driver/postgres/driver.go b/internal/driver/postgres/driver.go index b95ccb7..07a8293 100644 --- a/internal/driver/postgres/driver.go +++ b/internal/driver/postgres/driver.go @@ -34,7 +34,7 @@ func (Driver) Capabilities() driver.Capabilities { BinaryFormat: true, CrossEngineSource: true, CrossEngineTarget: true, - CDC: false, // Phase F + CDC: true, NativeBackpressure: true, } } From 904763eb77a79d35eef9550c7d4f02c93e703b44 Mon Sep 17 00:00:00 2001 From: Nikhil Rajput Date: Tue, 23 Jun 2026 04:24:06 +0530 Subject: [PATCH 10/13] =?UTF-8?q?docs:=20reconcile=20Phase=20F=20status=20?= =?UTF-8?q?=E2=80=94=20cross-engine=20sync=20now=20works=20end-to-end=20(F?= =?UTF-8?q?=E2=86=92Complete)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- CHANGELOG.md | 4 ++-- README.md | 6 +++--- docs/CROSS_ENGINE.md | 40 ++++++++++++++++++++++------------------ 3 files changed, 27 insertions(+), 23 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c0f9725..70a0bad 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -22,8 +22,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - `mysql` and `mariadb` drivers as thin wrappers that inject the fork-specific binary names and declare capabilities honestly (`Parallel: false` — the dump tools are single-threaded). Registered via side-effect import in `internal/app/drivers.go`. - Integration suites for both engines run on the Phase D `RunDriverSuite` harness via testcontainers (`mysql:8.0`, `mariadb:11`). - CI installs the Postgres/MySQL/MariaDB client tools and runs the full integration suite on the Ubuntu runner (where Docker is available); unit tests continue to run on Linux/macOS/Windows. -- Phase F advanced-transfer machinery (some CLI paths gated pending follow-up wiring — see the `docs/INCREMENTAL.md`, `docs/CROSS_ENGINE.md`, `docs/CDC.md` Status sections): +- Phase F advanced transfer — all four modes work end-to-end (incremental, cross-engine, CDC, bounded streaming); live DB paths are integration-tested in CI. See `docs/INCREMENTAL.md`, `docs/CROSS_ENGINE.md`, `docs/CDC.md`. - **Working today:** every dump is prefixed with a 4 KB `SIPH` JSON envelope (type, base/parent IDs, WAL/binlog positions, checksum); `restore` resolves the base→incremental chain and applies it in order, with `--up-to ` to stop early (cycle + broken-chain detection). Sync now streams through a bounded `jobs.Stream` (default 64×1 MB) instead of `io.Pipe`, exposing a `FillPercent` backpressure metric and propagating backup failures to the restore side via `CloseErr` (no truncated dump committed as clean). - **Incremental backup wired end-to-end (`backup --incremental --base `):** captures a **bounded** change set since the base dump's recorded end position, serialized as engine-neutral JSONL `CanonicalChange` records. Postgres bounds the pgoutput logical-decoding stream by `pg_current_wal_lsn()` captured at backup time; MySQL/MariaDB bound the binlog-tool decode by the current binlog file+offset. The incremental dump's envelope carries this capture's end position so the next incremental resumes exactly there. `restore` replays incremental links via `ApplyChange` (base links still restore natively). Postgres adds an orphan replication-slot sweep (`SweepOrphanSlots`) run before each capture — drops inactive `siphon_*` physical slots while preserving the persistent logical resume slot. `Incremental` capability is now `true` for all three drivers. Live-server behavior is integration-tested in CI (`wal_level=logical`); compile-checked locally. - **CDC continuous sync wired end-to-end (`siphon cdc ` / `sync --continuous`):** unbounded `ChangeStreamer.StreamChanges` → `CanonicalTransfer.ApplyChange` on the target, with an initial schema+data snapshot→stream handoff on first run (consistent start position captured before the snapshot) and resume-from-saved-position on restart (stable per-(source,target) job ID). Works same-engine and cross-engine (changes are engine-neutral `CanonicalChange`s). Checkpoints state every 100 changes plus on clean exit; at-least-once delivery is safe because `ApplyChange` is idempotent (INSERT upsert, UPDATE/DELETE by PK). ctx cancel is the normal clean stop. `CDC` capability is now `true` for all three drivers. Same-engine, cross-engine, and resume paths are integration-tested in CI. - - **Machinery in place, CLI gated (follow-up):** cross-engine snapshot type-mapping (canonical schema + JSONL emit/consume with per-engine identifier quoting and placeholders) — `sync --cross-engine` is capability-gated off until typed schema introspection exists. + - **Cross-engine sync wired end-to-end (`sync --cross-engine`, e.g. Postgres → MySQL):** typed schema introspection via `driver.SchemaInspector` (`information_schema`/`pg_catalog`, incl. primary keys) builds a `CanonicalSchema`; the source emits canonical JSONL rows and the target re-creates tables (with primary keys) and inserts them, translating types via `MapToNative` — all with per-engine identifier quoting and parameterized values. `CrossEngineSource`/`CrossEngineTarget` capabilities are now `true` for all three drivers. Scope is data + table structure + primary keys (secondary indexes, FKs, triggers, views, functions not translated). Integration-tested in CI (Postgres → MySQL); compile-checked locally. diff --git a/README.md b/README.md index e68755c..d421cc9 100644 --- a/README.md +++ b/README.md @@ -17,7 +17,7 @@ A single binary that turns the painful, error-prone sprawl of `pg_dump` → `pg_ --- > [!WARNING] -> **Pre-1.0 — active development.** Postgres, MySQL, and MariaDB backup/restore/sync/verify/inspect work end-to-end today (Phases B + E), and bare `siphon` opens an interactive multi-panel dashboard (Phase C). The driver layer is hardened with a shared cross-driver test harness, capability gating, and connection retry (Phase D). Incremental backup + restore work end-to-end (Phase F); cross-engine sync and ops features are on the [roadmap](#roadmap). APIs, flags, and the on-disk dump format may change before 1.0. Track progress via the milestone tags (`phase-a`, `phase-b`, `phase-c`, `phase-d`, `phase-e`, …). +> **Pre-1.0 — active development.** Postgres, MySQL, and MariaDB backup/restore/sync/verify/inspect work end-to-end today (Phases B + E), and bare `siphon` opens an interactive multi-panel dashboard (Phase C). The driver layer is hardened with a shared cross-driver test harness, capability gating, and connection retry (Phase D). Advanced transfer is complete (Phase F): incremental backup/restore, cross-engine sync, and CDC continuous replication all work end-to-end. Ops features (cloud storage, secret backends, retention, telemetry) are on the [roadmap](#roadmap). APIs, flags, and the on-disk dump format may change before 1.0. Track progress via the milestone tags (`phase-a` … `phase-f`). ## Table of contents @@ -54,7 +54,7 @@ A single binary that turns the painful, error-prone sprawl of `pg_dump` → `pg_ | **C** — TUI dashboard | Multi-panel Bubble Tea dashboard (profiles · dumps · jobs) with live job progress, backup/restore modal forms, and snapshot tests | ✅ Complete | | **D** — Driver hardening | Shared cross-driver test harness (`RunDriverSuite`), capability-gating helper (`RequireCapability`), Postgres connection-probe retry, and a `docs/DRIVERS.md` contributor guide | ✅ Complete | | **E** — MySQL + MariaDB | Both drivers via a shared `_mysqlcommon` package (shared `Conn`, `mysqldump`/`mariadb-dump` backup, client-pipe restore), exercised by the Phase D `RunDriverSuite` harness | ✅ Complete | -| **F** — Advanced transfer | Bounded-buffer streaming sync and incremental backup/restore work today: `backup --incremental --base ` captures a bounded change set (Postgres logical decoding / MySQL-MariaDB binlog) and `restore` replays the base→incremental chain, with Postgres orphan replication-slot sweep. CDC continuous sync runs end-to-end (`siphon cdc ` / `sync --continuous`): unbounded change streaming with an initial snapshot→stream handoff, resumable via a state file, same-engine and cross-engine. Cross-engine snapshot type-mapping machinery is in place but `sync --cross-engine` remains gated pending typed introspection — see [docs/INCREMENTAL.md](docs/INCREMENTAL.md), [docs/CROSS_ENGINE.md](docs/CROSS_ENGINE.md), [docs/CDC.md](docs/CDC.md) | 🟡 Partial | +| **F** — Advanced transfer | All four advanced-transfer modes work end-to-end: bounded-buffer streaming sync; **incremental** backup/restore (`backup --incremental --base ` captures a bounded change set via Postgres logical decoding / MySQL-MariaDB binlog, `restore` replays the base→incremental chain, Postgres orphan-slot sweep); **cross-engine** sync (`sync --cross-engine` — typed `SchemaInspector` introspection → canonical type-mapping, e.g. Postgres → MySQL); and **CDC** (`siphon cdc` / `sync --continuous` — unbounded change streaming with snapshot→stream handoff, resumable, same- and cross-engine). Live DB paths are integration-tested in CI — see [docs/INCREMENTAL.md](docs/INCREMENTAL.md), [docs/CROSS_ENGINE.md](docs/CROSS_ENGINE.md), [docs/CDC.md](docs/CDC.md) | ✅ Complete | | **G** — Ops features | Cloud storage, secret backends, profile groups + 2FA, team mode, audit log, retention, telemetry | ⏳ Planned | | **H** — Distribution | GoReleaser, Homebrew tap, Scoop bucket, install script, docs site | ⏳ Planned | @@ -222,7 +222,7 @@ Contributions are welcome. Please read [CONTRIBUTING.md](CONTRIBUTING.md), keep Adding a new database engine? See [docs/DRIVERS.md](docs/DRIVERS.md) for the driver contributor guide. -Concept docs (honest as-built status — several paths are documented follow-ups): [docs/INCREMENTAL.md](docs/INCREMENTAL.md) (incremental backup + restore work end-to-end), [docs/CROSS_ENGINE.md](docs/CROSS_ENGINE.md) (type-map matrix; `--cross-engine` is gated off pending typed introspection), and [docs/CDC.md](docs/CDC.md) (continuous CDC sync runs end-to-end, same- and cross-engine). +Concept docs: [docs/INCREMENTAL.md](docs/INCREMENTAL.md) (incremental backup + restore), [docs/CROSS_ENGINE.md](docs/CROSS_ENGINE.md) (cross-engine sync + the type-map matrix), and [docs/CDC.md](docs/CDC.md) (continuous CDC sync, same- and cross-engine). All three work end-to-end; live DB behavior is integration-tested in CI. ## License diff --git a/docs/CROSS_ENGINE.md b/docs/CROSS_ENGINE.md index ef19d8e..38a467e 100644 --- a/docs/CROSS_ENGINE.md +++ b/docs/CROSS_ENGINE.md @@ -45,26 +45,29 @@ siphon sync --from pg-prod --to mysql-staging --cross-engine | --- | --- | | Canonical type vocabulary + `MapToNative` matrix | ✅ Works (unit-tested) | | JSONL emit/consume with per-engine quoting + placeholders | ✅ Works (unit-tested) | -| `sync --cross-engine` end-to-end | ⚠️ Gated off (follow-up) | - -The type-mapping and JSONL-transfer machinery exists and is tested, but -`sync --cross-engine` is **capability-gated** on `CrossEngineSource` -(source driver) and `CrossEngineTarget` (target driver). **No driver declares -either capability today**, so the request is rejected with -`ErrDriverUnsupported` (`internal/app/sync.go`, `runCrossEngineSync`). - -The reason is deliberate: cross-engine translation needs a *typed* -`CanonicalSchema` (column types), and `driver.Inspect` does not carry column -types yet. Rather than fabricate a schema, the path stays gated until typed -schema introspection lands and a driver flips its cross-engine capabilities to -true. At that point `runCrossEngineSync` gains the real -emit → translate → consume pipeline. +| Typed schema introspection (`SchemaInspector`, with primary keys) | ✅ Works (Postgres + MySQL/MariaDB) | +| `sync --cross-engine` snapshot (existing data) | ✅ Works (integration-tested in CI: Postgres → MySQL) | +| Cross-engine CDC (continuous change replication) | ✅ Works — see [docs/CDC.md](CDC.md) | + +`sync --cross-engine` is capability-gated on `CrossEngineSource` (source) and +`CrossEngineTarget` (target). All three drivers now declare both, backed by +`driver.SchemaInspector` (typed column introspection from `information_schema` / +`pg_catalog`, including primary keys) and `driver.CanonicalTransfer` (the +engine-side emit/consume). `runCrossEngineSync` inspects the source schema, +streams its rows as canonical JSONL through a bounded `jobs.Stream`, and the +target re-creates the tables (with primary keys) and inserts the rows — +translating types via `MapToNative`. ```bash siphon sync --from pg-prod --to mysql-staging --cross-engine -# Error: cross-engine sync requires typed schema introspection, not yet available ``` +Scope: this is **data + table structure** (columns, types, primary keys) — not +behavior. Indexes (beyond the primary key), foreign keys, triggers, views, and +functions are **not** translated. The live Postgres → MySQL round-trip is +validated by an integration test in CI (`internal/app/cross_engine_integration_test.go`); +it is compile-checked locally where Docker is unavailable. + ## Type-mapping matrix `MapToNative` maps each canonical type to a native type. These mappings are real @@ -87,11 +90,12 @@ known precision/scale renders as e.g. `DECIMAL(10,2)`. ## Scope and limitations -The v1 cross-engine scope (once the path is wired) is **data only**: +The v1 cross-engine scope is **data + table structure**: - Triggers, views, and stored functions are **skipped**. -- Index recreation is **not implemented yet**: the consume path issues - `CREATE TABLE IF NOT EXISTS` (column definitions only) plus row INSERTs, so +- Primary keys **are** recreated (the consume path emits `CREATE TABLE … PRIMARY KEY (…)`). +- Secondary index recreation is **not implemented yet**: the consume path issues + `CREATE TABLE IF NOT EXISTS` (column definitions + primary key) plus row INSERTs, so only data and table structure transfer. Indexes and any constraints beyond the inline column defs are a follow-up. - Foreign keys are **deferred**. From 68d1858a6c39084c3aa0979b9917b01c3fbfef15 Mon Sep 17 00:00:00 2001 From: Nikhil Rajput Date: Tue, 23 Jun 2026 05:09:27 +0530 Subject: [PATCH 11/13] fix(postgres): establish logical slot for base - Make CurrentPosition create the pgoutput logical slot if absent and anchor to its consistent point, so WAL from the base forward is retained. A logical slot only decodes WAL produced after its own creation, so creating it lazily at stream time meant the first incremental and the bounded change stream captured zero changes. - Add establishLogicalSlot helper; ensureLogicalSlot now returns the consistent point on fresh creation, "" when the slot already exists. - Fix the two Postgres integration tests to take the start/base position via CurrentPosition before the DML, matching the real ordering. - Give the cross-engine MySQL container a 180s startup wait so a slow concurrent boot is tolerated rather than failing the run. --- internal/app/cross_engine_integration_test.go | 12 +++++ internal/driver/postgres/changestream.go | 46 ++++++++++++++++--- .../postgres/changestream_integration_test.go | 16 +++---- .../driver/postgres/incremental_change.go | 24 ++++++++-- .../postgres/incremental_integration_test.go | 18 ++++---- 5 files changed, 88 insertions(+), 28 deletions(-) diff --git a/internal/app/cross_engine_integration_test.go b/internal/app/cross_engine_integration_test.go index 18ab357..6d407e7 100644 --- a/internal/app/cross_engine_integration_test.go +++ b/internal/app/cross_engine_integration_test.go @@ -9,8 +9,10 @@ import ( "testing" "time" + tc "github.com/testcontainers/testcontainers-go" mysqlctr "github.com/testcontainers/testcontainers-go/modules/mysql" pgctr "github.com/testcontainers/testcontainers-go/modules/postgres" + "github.com/testcontainers/testcontainers-go/wait" mysqlcommon "github.com/nixrajput/siphon/internal/driver/_mysqlcommon" @@ -56,10 +58,20 @@ func startIntegPG(t *testing.T) (host string, port int, cleanup func()) { func startIntegMySQL(t *testing.T) (host string, port int, cleanup func()) { t.Helper() ctx := context.Background() + // Explicit, generous startup wait: this test boots Postgres + MySQL in one + // run alongside the per-driver suites, so under concurrent cold-image pulls + // MySQL's two-phase init (init → restart → second "ready for connections") + // can exceed the module's default 60s timeout. Wait for that second ready + // log with a 180s ceiling so a slow-but-healthy boot is tolerated, not failed. c, err := mysqlctr.Run(ctx, "mysql:8.0", mysqlctr.WithDatabase("test"), mysqlctr.WithUsername("root"), mysqlctr.WithPassword("rootpass"), + tc.WithWaitStrategy( + wait.ForLog("port: 3306 MySQL Community Server"). + WithOccurrence(1). + WithStartupTimeout(180*time.Second), + ), ) if err != nil { t.Fatalf("start mysql container: %v", err) diff --git a/internal/driver/postgres/changestream.go b/internal/driver/postgres/changestream.go index 1c38678..8848d1d 100644 --- a/internal/driver/postgres/changestream.go +++ b/internal/driver/postgres/changestream.go @@ -67,7 +67,7 @@ func (c *Conn) streamWithStop(ctx context.Context, from canonical.Position, stop } defer func() { _ = repl.Close(context.Background()) }() - if err := ensureLogicalSlot(ctx, repl); err != nil { + if _, err := ensureLogicalSlot(ctx, repl); err != nil { return canonical.Position{}, err } @@ -375,16 +375,50 @@ func (c *Conn) ensurePublication(ctx context.Context) error { // ensureLogicalSlot creates the pgoutput logical slot on the replication // connection if absent. CreateReplicationSlot errors with "already exists" when // the slot is present; that is tolerated so repeated streams reuse the slot. -func ensureLogicalSlot(ctx context.Context, repl *pgconn.PgConn) error { - _, err := pglogrepl.CreateReplicationSlot(ctx, repl, siphonSlot, outputPlugin, +// +// It returns the slot's consistent point — the LSN from which the freshly +// created slot guarantees every subsequent change is retained and decodable — +// when it creates the slot, and "" when the slot already existed (the existing +// slot is already retaining WAL, so the caller has no new anchor to record). +func ensureLogicalSlot(ctx context.Context, repl *pgconn.PgConn) (string, error) { + res, err := pglogrepl.CreateReplicationSlot(ctx, repl, siphonSlot, outputPlugin, pglogrepl.CreateReplicationSlotOptions{Temporary: false}) if err != nil { if strings.Contains(err.Error(), "already exists") { - return nil + return "", nil } - return &errs.Error{Op: "postgres.stream", Code: errs.CodeSystem, Cause: err} + return "", &errs.Error{Op: "postgres.stream", Code: errs.CodeSystem, Cause: err} } - return nil + return res.ConsistentPoint, nil +} + +// establishLogicalSlot opens a short-lived replication connection, ensures the +// publication and logical slot exist, and returns the slot's consistent point +// (empty when the slot already existed). +// +// This is the slot-establishment primitive used to anchor WAL retention BEFORE +// a workload runs: a logical slot only retains and decodes WAL produced after +// its own creation, so the slot must exist before the changes a later +// incremental or CDC run wants to capture. CurrentPosition calls this so a base +// backup leaves the slot in place, and the changes that follow are retained. +func (c *Conn) establishLogicalSlot(ctx context.Context) (string, error) { + if err := c.requireLogicalWAL(ctx); err != nil { + return "", err + } + if err := c.ensurePublication(ctx); err != nil { + return "", err + } + repl, err := pgconn.Connect(ctx, c.replicationDSN()) + if err != nil { + return "", &errs.Error{ + Op: "postgres.stream", + Code: errs.CodeSystem, + Cause: errs.ErrConnectionFailed, + Hint: "logical replication connect: " + err.Error(), + } + } + defer func() { _ = repl.Close(context.Background()) }() + return ensureLogicalSlot(ctx, repl) } // resolveStartLSN returns the LSN to start replication from. A non-empty diff --git a/internal/driver/postgres/changestream_integration_test.go b/internal/driver/postgres/changestream_integration_test.go index fbb8edd..5bd2c0c 100644 --- a/internal/driver/postgres/changestream_integration_test.go +++ b/internal/driver/postgres/changestream_integration_test.go @@ -72,14 +72,12 @@ func TestStreamChanges_Bounded(t *testing.T) { conn := &Conn{db: db, p: prof} - // Capture the start LSN before any DML so the stream sees exactly our changes. - var startLSN string - if err := db.QueryRowContext(ctx, "SELECT pg_current_wal_lsn()::text").Scan(&startLSN); err != nil { - t.Fatalf("capture lsn: %v", err) - } - // The publication+slot must exist before the DML so the slot retains the WAL. - if err := conn.ensurePublication(ctx); err != nil { - t.Fatalf("ensure publication: %v", err) + // Capture the start position before any DML so the stream sees exactly our + // changes. CurrentPosition also establishes the logical slot — a slot only + // retains WAL produced after its creation, so it must exist before the DML. + startPos, err := conn.CurrentPosition(ctx) + if err != nil { + t.Fatalf("capture start position: %v", err) } // Apply the three operations. @@ -109,7 +107,7 @@ func TestStreamChanges_Bounded(t *testing.T) { return nil } - if _, err := conn.StreamChanges(streamCtx, canonical.Position{LSN: startLSN}, emit); err != nil { + if _, err := conn.StreamChanges(streamCtx, startPos, emit); err != nil { t.Fatalf("StreamChanges: %v", err) } diff --git a/internal/driver/postgres/incremental_change.go b/internal/driver/postgres/incremental_change.go index 2c75350..d729492 100644 --- a/internal/driver/postgres/incremental_change.go +++ b/internal/driver/postgres/incremental_change.go @@ -14,10 +14,28 @@ import ( var _ driver.IncrementalBackuper = (*Conn)(nil) var _ driver.BasePositioner = (*Conn)(nil) -// CurrentPosition returns the server's current WAL position via -// pg_current_wal_lsn(). app.Backup calls this right after a full backup so the -// base dump's Envelope records where the first incremental should resume from. +// CurrentPosition records the resume anchor for the first incremental after a +// base backup. app.Backup calls this right after a full backup so the base +// dump's Envelope carries where the first incremental should resume from. +// +// It also ESTABLISHES the logical replication slot if it does not yet exist. A +// logical slot only retains and decodes WAL produced after the slot's own +// creation, so the slot must exist from the recorded anchor forward or the +// first incremental silently captures nothing (it would resume from an LSN the +// slot never retained). When this call creates the slot, the slot's consistent +// point is the correct anchor — it is the exact LSN from which the slot +// guarantees decodable WAL. When the slot already exists it is already +// retaining WAL, so pg_current_wal_lsn() is a safe anchor. func (c *Conn) CurrentPosition(ctx context.Context) (canonical.Position, error) { + consistent, err := c.establishLogicalSlot(ctx) + if err != nil { + return canonical.Position{}, err + } + if consistent != "" { + // Slot freshly created: anchor to its consistent point, the boundary + // from which it retains every subsequent change. + return canonical.Position{LSN: consistent}, nil + } var lsn string if err := c.db.QueryRowContext(ctx, "SELECT pg_current_wal_lsn()::text").Scan(&lsn); err != nil { return canonical.Position{}, &errs.Error{Op: "postgres.current_position", Code: errs.CodeSystem, Cause: err} diff --git a/internal/driver/postgres/incremental_integration_test.go b/internal/driver/postgres/incremental_integration_test.go index 4f74cfb..71feddf 100644 --- a/internal/driver/postgres/incremental_integration_test.go +++ b/internal/driver/postgres/incremental_integration_test.go @@ -46,15 +46,13 @@ func TestBackupIncremental_BoundedCaptureAndReplay(t *testing.T) { conn := &Conn{db: db, p: prof} - // Publication + slot must exist before the DML so the slot retains the WAL. - if err := conn.ensurePublication(ctx); err != nil { - t.Fatalf("ensure publication: %v", err) - } - - // Capture the base-end LSN: the incremental resumes from exactly here. - var baseEnd string - if err := db.QueryRowContext(ctx, "SELECT pg_current_wal_lsn()::text").Scan(&baseEnd); err != nil { - t.Fatalf("capture base-end lsn: %v", err) + // Capture the base-end position: the incremental resumes from exactly here. + // CurrentPosition also establishes the logical slot — a slot only retains + // WAL produced after its creation, so it must exist before the post-base DML + // or the incremental captures nothing. + basePos, err := conn.CurrentPosition(ctx) + if err != nil { + t.Fatalf("capture base-end position: %v", err) } // Post-base changes: one insert, one update, one delete. @@ -72,7 +70,7 @@ func TestBackupIncremental_BoundedCaptureAndReplay(t *testing.T) { capCtx, cancel := context.WithTimeout(ctx, 60*time.Second) defer cancel() var buf bytes.Buffer - endPos, err := conn.BackupIncremental(capCtx, canonical.Position{LSN: baseEnd}, &buf) + endPos, err := conn.BackupIncremental(capCtx, basePos, &buf) if err != nil { t.Fatalf("BackupIncremental: %v", err) } From dc1b49f478dc3cd2bb55444f35525e3b021eb8b0 Mon Sep 17 00:00:00 2001 From: Nikhil Rajput Date: Wed, 24 Jun 2026 02:48:17 +0530 Subject: [PATCH 12/13] fix: apply CodeRabbit review fixes for Phase F - Apply post-review fixes to incremental/CDC/cross-engine data paths. - Stop advancing the PG decode position from keepalive ServerWALEnd: track server WAL end separately so a keepalive crossing the bound cannot end a bounded incremental before the changes at that LSN are decoded and emitted (silent data loss); the catch-up stop now uses the separate server position. - Remove the CDC ahead-of-stream periodic checkpoint: persisting the source's current cursor mid-stream could resume past un-applied changes after a crash. Checkpoint only the streamer's delivered position on exit; idempotent ApplyChange makes the replayed tail safe. - Honor --table in cross-engine sync and CDC (snapshot + streamed changes) via shared filterSchemaTables/tableAllowed helpers. - Surface mysqlbinlog tool failures: capture stderr, return non-zero exits that are not an intentional stop, and error when a bounded scan ends before reaching its stop position. - Distinguish SQL NULL from the string 'NULL' in the binlog parser via a typed colVal token; strip type comments only outside quoted strings. - Fail fast on unmapped Postgres/MySQL column types instead of silently coercing to text (would corrupt binary/BLOB families cross-engine). - Preflight incremental-replay support before a Clean base restore; validate base profile/driver compatibility and a non-empty position before an incremental backup. - Replay JSONL via json.Decoder to drop the 8 MiB Scanner line cap. - Guard empty column sets in BuildUpdateSQL/BuildDeleteSQL; require both endpoints in the cdc CLI; restrict the orphan-slot sweep to physical slots. - Update CDC docs and CHANGELOG to match the new checkpoint and table behavior. --- CHANGELOG.md | 4 +- docs/CDC.md | 14 +- internal/app/backup.go | 22 +++ internal/app/cdc.go | 42 +++--- internal/app/restore.go | 54 ++++++-- internal/app/sync.go | 34 +++++ internal/canonical/canonical.go | 9 ++ internal/cli/cdc.go | 8 ++ internal/driver/_mysqlcommon/changestream.go | 128 ++++++++++++------ .../driver/_mysqlcommon/changestream_test.go | 73 +++++++--- .../driver/_mysqlcommon/inspect_schema.go | 38 ++++-- .../_mysqlcommon/inspect_schema_test.go | 42 +++--- internal/driver/postgres/changestream.go | 31 ++++- .../driver/postgres/incremental_change.go | 19 +-- internal/driver/postgres/inspect_schema.go | 34 +++-- .../driver/postgres/inspect_schema_test.go | 38 +++--- 16 files changed, 414 insertions(+), 176 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 70a0bad..d4d2708 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -25,5 +25,5 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Phase F advanced transfer — all four modes work end-to-end (incremental, cross-engine, CDC, bounded streaming); live DB paths are integration-tested in CI. See `docs/INCREMENTAL.md`, `docs/CROSS_ENGINE.md`, `docs/CDC.md`. - **Working today:** every dump is prefixed with a 4 KB `SIPH` JSON envelope (type, base/parent IDs, WAL/binlog positions, checksum); `restore` resolves the base→incremental chain and applies it in order, with `--up-to ` to stop early (cycle + broken-chain detection). Sync now streams through a bounded `jobs.Stream` (default 64×1 MB) instead of `io.Pipe`, exposing a `FillPercent` backpressure metric and propagating backup failures to the restore side via `CloseErr` (no truncated dump committed as clean). - **Incremental backup wired end-to-end (`backup --incremental --base `):** captures a **bounded** change set since the base dump's recorded end position, serialized as engine-neutral JSONL `CanonicalChange` records. Postgres bounds the pgoutput logical-decoding stream by `pg_current_wal_lsn()` captured at backup time; MySQL/MariaDB bound the binlog-tool decode by the current binlog file+offset. The incremental dump's envelope carries this capture's end position so the next incremental resumes exactly there. `restore` replays incremental links via `ApplyChange` (base links still restore natively). Postgres adds an orphan replication-slot sweep (`SweepOrphanSlots`) run before each capture — drops inactive `siphon_*` physical slots while preserving the persistent logical resume slot. `Incremental` capability is now `true` for all three drivers. Live-server behavior is integration-tested in CI (`wal_level=logical`); compile-checked locally. - - **CDC continuous sync wired end-to-end (`siphon cdc ` / `sync --continuous`):** unbounded `ChangeStreamer.StreamChanges` → `CanonicalTransfer.ApplyChange` on the target, with an initial schema+data snapshot→stream handoff on first run (consistent start position captured before the snapshot) and resume-from-saved-position on restart (stable per-(source,target) job ID). Works same-engine and cross-engine (changes are engine-neutral `CanonicalChange`s). Checkpoints state every 100 changes plus on clean exit; at-least-once delivery is safe because `ApplyChange` is idempotent (INSERT upsert, UPDATE/DELETE by PK). ctx cancel is the normal clean stop. `CDC` capability is now `true` for all three drivers. Same-engine, cross-engine, and resume paths are integration-tested in CI. - - **Cross-engine sync wired end-to-end (`sync --cross-engine`, e.g. Postgres → MySQL):** typed schema introspection via `driver.SchemaInspector` (`information_schema`/`pg_catalog`, incl. primary keys) builds a `CanonicalSchema`; the source emits canonical JSONL rows and the target re-creates tables (with primary keys) and inserts them, translating types via `MapToNative` — all with per-engine identifier quoting and parameterized values. `CrossEngineSource`/`CrossEngineTarget` capabilities are now `true` for all three drivers. Scope is data + table structure + primary keys (secondary indexes, FKs, triggers, views, functions not translated). Integration-tested in CI (Postgres → MySQL); compile-checked locally. + - **CDC continuous sync wired end-to-end (`siphon cdc ` / `sync --continuous`):** unbounded `ChangeStreamer.StreamChanges` → `CanonicalTransfer.ApplyChange` on the target, with an initial schema+data snapshot→stream handoff on first run (consistent start position captured before the snapshot) and resume-from-saved-position on restart (stable per-(source,target) job ID). Works same-engine and cross-engine (changes are engine-neutral `CanonicalChange`s) and honors `--table` in both the initial snapshot and the streamed changes. Persists the streamer's delivered position on exit (no ahead-of-stream periodic checkpoint, which could resume past un-applied changes after a crash); at-least-once delivery is safe because `ApplyChange` is idempotent (INSERT upsert, UPDATE/DELETE by PK). ctx cancel is the normal clean stop. `CDC` capability is now `true` for all three drivers. Same-engine, cross-engine, and resume paths are integration-tested in CI. + - **Cross-engine sync wired end-to-end (`sync --cross-engine`, e.g. Postgres → MySQL):** typed schema introspection via `driver.SchemaInspector` (`information_schema`/`pg_catalog`, incl. primary keys) builds a `CanonicalSchema`; the source emits canonical JSONL rows and the target re-creates tables (with primary keys) and inserts them, translating types via `MapToNative` — all with per-engine identifier quoting and parameterized values. Honors `--table`. Unmapped source column types now fail fast with an explicit error instead of silently coercing to `text` (which would corrupt binary/BLOB families). `CrossEngineSource`/`CrossEngineTarget` capabilities are now `true` for all three drivers. Scope is data + table structure + primary keys (secondary indexes, FKs, triggers, views, functions not translated). Integration-tested in CI (Postgres → MySQL); compile-checked locally. diff --git a/docs/CDC.md b/docs/CDC.md index b7d154e..6d499da 100644 --- a/docs/CDC.md +++ b/docs/CDC.md @@ -37,7 +37,7 @@ each change to the target continuously, until interrupted. It works snapshot. 3. **Stream loop:** `ChangeStreamer.StreamChanges` (unbounded) emits each `CanonicalChange`; `RunCDC` applies it via `CanonicalTransfer.ApplyChange` - on the target and checkpoints state periodically. + on the target and persists the streamer's delivered position on exit. The `job_id` is a stable hash of `(from, to)`, so re-running the same continuous sync resumes from the same state file. @@ -90,10 +90,14 @@ $HOME/.local/state/siphon/cdc/.state # default Each file is JSON: source/target profiles plus the last applied position (LSN for Postgres, binlog file + offset for MySQL/MariaDB). -**Checkpoint/resume granularity is "since the last checkpoint."** RunCDC -checkpoints every 100 applied changes plus on clean exit. After a crash it -resumes from the last checkpoint and re-applies any changes that landed after -it. This is safe because delivery is **at-least-once** and `ApplyChange` is +**Checkpoint/resume granularity is "since the streamer's last delivered +position."** RunCDC persists the position the change streamer reports when the +stream stops — a position tied to what was actually delivered, never ahead of +it. There is deliberately **no** ahead-of-stream periodic checkpoint: writing +the source's *current* WAL/binlog end mid-stream would, after a crash, resume +past changes that were streamed but never applied — silent data loss. After a +crash it resumes from the last persisted position and re-applies the tail. This +is safe because delivery is **at-least-once** and `ApplyChange` is **idempotent**: INSERT is idempotent (upsert), and UPDATE/DELETE target by primary key. Re-applying the tail is therefore a no-op — no gaps, no duplicates. diff --git a/internal/app/backup.go b/internal/app/backup.go index dff8be4..8761972 100644 --- a/internal/app/backup.go +++ b/internal/app/backup.go @@ -237,10 +237,32 @@ func runIncrementalBackup(ctx context.Context, d Deps, opt BackupOpts, resolved } } + // Reject a base that does not belong to this profile/driver: an incremental + // chain is only meaningful against the same engine and source, and a + // cross-driver base would produce changes the target cannot replay. + if base.Driver != resolved.Driver || base.Profile != opt.Profile { + return &errs.Error{ + Op: "app.backup.incremental", + Code: errs.CodeUser, + Cause: errs.ErrIncompatibleEngine, + Hint: "base dump " + opt.BaseID + " was created for profile " + base.Profile + " (" + base.Driver + "), not " + opt.Profile + " (" + resolved.Driver + ")", + } + } + since, err := basePosition(d, opt.BaseID) if err != nil { return err } + // A base with no recorded change-stream position cannot anchor an incremental; + // capturing from a zero cursor would silently start at "now" and drop every + // change committed since the base. Require a real position. + if since.LSN == "" && since.BinlogFile == "" { + return &errs.Error{ + Op: "app.backup.incremental", + Code: errs.CodeUser, + Hint: "base dump " + opt.BaseID + " has no recorded change-stream position; take a fresh full backup and use that as --base", + } + } inc, ok := conn.(driver.IncrementalBackuper) if !ok { diff --git a/internal/app/cdc.go b/internal/app/cdc.go index 8f4b4bb..47e4ce2 100644 --- a/internal/app/cdc.go +++ b/internal/app/cdc.go @@ -93,13 +93,6 @@ func cdcJobID(from, to string) string { return "cdc-" + hex.EncodeToString(sum[:8]) } -// cdcCheckpointEvery bounds how many applied changes pass between resume-state -// checkpoints. At-least-once delivery plus idempotent ApplyChange makes coarse -// checkpointing safe: on restart we resume from the last checkpoint and re-apply -// any changes that landed after it, which is a no-op for INSERT (idempotent), -// UPDATE, and DELETE (both by primary key). -const cdcCheckpointEvery = 100 - // RunCDC starts (or resumes) a continuous, unbounded sync: it tails the source's // logical change stream and applies each engine-neutral CanonicalChange to the // target. It works same-engine and cross-engine alike because CanonicalChange is @@ -110,10 +103,11 @@ const cdcCheckpointEvery = 100 // changes committed after that position. On a restart it resumes from the saved // position with no snapshot. // -// Resume granularity is "since the last checkpoint": StreamChanges reports the -// position reached after each applied change via the emit callback, and RunCDC -// checkpoints every cdcCheckpointEvery changes plus on clean exit. Re-applying the -// tail after a crash is safe (see cdcCheckpointEvery). +// Resume granularity is "since the last clean exit": RunCDC persists the +// streamer's returned final position when the stream stops (the position is tied +// to what was actually delivered, never ahead of it). Re-applying the tail after +// a crash is safe because ApplyChange is idempotent (INSERT upserts; UPDATE and +// DELETE target by primary key). // // ctx cancellation is the normal stop signal (matching the bounded-stream // convention): on cancel RunCDC persists final state and returns nil; only a @@ -187,7 +181,7 @@ func RunCDC(parent context.Context, d Deps, opt SyncOpts) (<-chan jobs.Event, st if err != nil { return err } - if snapErr := cdcSnapshot(ctx, srcConn, dstXfer, src.Driver); snapErr != nil { + if snapErr := cdcSnapshot(ctx, srcConn, dstXfer, src.Driver, opt.Tables); snapErr != nil { return snapErr } state.setPosition(from) @@ -199,6 +193,12 @@ func RunCDC(parent context.Context, d Deps, opt SyncOpts) (<-chan jobs.Event, st applied := 0 emit2 := func(ch canonical.CanonicalChange) error { + // Honor --table: the streamer surfaces changes for every table (the + // source's publication/binlog is database-wide), so skip any change + // outside the requested set before applying it to the target. + if !tableAllowed(ch.Table, opt.Tables) { + return nil + } if applyErr := dstXfer.ApplyChange(ctx, ch); applyErr != nil { return applyErr } @@ -207,12 +207,13 @@ func RunCDC(parent context.Context, d Deps, opt SyncOpts) (<-chan jobs.Event, st Phase: jobs.PhaseProgress, Message: fmt.Sprintf("applied %s on %s (%d total)", ch.Op, ch.Table, applied), }) - if applied%cdcCheckpointEvery == 0 { - if pos, posErr := srcPositioner.CurrentPosition(ctx); posErr == nil { - state.setPosition(pos) - _ = saveCDCState(state) - } - } + // No periodic mid-stream checkpoint here: srcPositioner.CurrentPosition + // returns the source's CURRENT WAL/binlog end, which is ahead of the + // change we just applied. Persisting it would, after a crash, resume + // PAST changes that streamed but were never applied — silent data loss. + // We checkpoint only the streamer's returned final position on exit + // (below), which is tied to what was actually delivered; at-least-once + // redelivery on resume is safe because ApplyChange is idempotent. return nil } @@ -239,7 +240,7 @@ func RunCDC(parent context.Context, d Deps, opt SyncOpts) (<-chan jobs.Event, st // the canonical transfer pipe (the same pattern as runCrossEngineSync). Both the // source's SchemaInspector+CanonicalTransfer and the target's CanonicalTransfer // are required. -func cdcSnapshot(ctx context.Context, srcConn driver.Conn, dstXfer driver.CanonicalTransfer, srcDriverName string) error { +func cdcSnapshot(ctx context.Context, srcConn driver.Conn, dstXfer driver.CanonicalTransfer, srcDriverName string, tables []string) error { srcInsp, ok := srcConn.(driver.SchemaInspector) if !ok { return cdcUnsupported(srcDriverName, "SchemaInspector") @@ -253,6 +254,9 @@ func cdcSnapshot(ctx context.Context, srcConn driver.Conn, dstXfer driver.Canoni if err != nil { return err } + // Snapshot only the requested tables; the streamed-change phase applies the + // same filter in emit2. + schema = filterSchemaTables(schema, tables) stream := jobs.NewStream(64) errCh := make(chan error, 1) diff --git a/internal/app/restore.go b/internal/app/restore.go index b7dc148..f4948a1 100644 --- a/internal/app/restore.go +++ b/internal/app/restore.go @@ -1,9 +1,9 @@ package app import ( - "bufio" "context" "encoding/json" + "errors" "io" "os" @@ -58,6 +58,22 @@ func Restore(parent context.Context, d Deps, opt RestoreOpts) (<-chan jobs.Event } defer func() { _ = conn.Close() }() + // Preflight: if the chain contains any incremental link, the driver must + // support change replay (CanonicalTransfer). Assert this BEFORE applying + // the base — a Clean base restore wipes the target, so discovering the + // unsupported-driver error only when we reach the first incremental link + // would leave the target destroyed and the chain half-applied. + if chainHasIncremental(chain) { + if _, ok := conn.(driver.CanonicalTransfer); !ok { + return &errs.Error{ + Op: "restore", + Code: errs.CodeUser, + Cause: errs.ErrDriverUnsupported, + Hint: resolved.Driver + " cannot replay incremental change links", + } + } + } + for i, m := range chain { emit(jobs.Event{Phase: jobs.PhaseProgress, Message: "applying " + m.ID}) f, err := os.Open(d.Dumps.Path(m.ID)) @@ -120,28 +136,40 @@ func Restore(parent context.Context, d Deps, opt RestoreOpts) (<-chan jobs.Event }) } +// chainHasIncremental reports whether any link in the resolved chain is an +// incremental dump (one that carries a JSONL change body replayed via +// ApplyChange). Incremental links record a non-empty ParentID in their metadata; +// the chain root (a base or full dump) does not. +func chainHasIncremental(chain []dumps.Meta) bool { + for _, m := range chain { + if m.ParentID != "" { + return true + } + } + return false +} + // replayChanges decodes a JSONL stream of CanonicalChanges from r and applies -// each to the target via ApplyChange, in order. A malformed line aborts the +// each to the target via ApplyChange, in order. A malformed record aborts the // replay so a corrupt incremental never partially applies undetected. +// +// A json.Decoder streams successive JSON values directly off the reader, so — +// unlike a bufio.Scanner — there is no per-record size ceiling that would reject +// a valid change carrying a large row value as "corrupt". func replayChanges(ctx context.Context, applier driver.CanonicalTransfer, r io.Reader) error { - sc := bufio.NewScanner(r) - sc.Buffer(make([]byte, 0, 64*1024), 8*1024*1024) - for sc.Scan() { - line := sc.Bytes() - if len(line) == 0 { - continue - } + dec := json.NewDecoder(r) + for { var ch canonical.CanonicalChange - if err := json.Unmarshal(line, &ch); err != nil { + if err := dec.Decode(&ch); err != nil { + if errors.Is(err, io.EOF) { + break + } return &errs.Error{Op: "restore.replay", Code: errs.CodeSystem, Cause: err, Hint: "incremental change body is corrupt"} } if err := applier.ApplyChange(ctx, ch); err != nil { return err } } - if err := sc.Err(); err != nil { - return &errs.Error{Op: "restore.replay", Code: errs.CodeSystem, Cause: err} - } return nil } diff --git a/internal/app/sync.go b/internal/app/sync.go index f4cc1fb..b499f6b 100644 --- a/internal/app/sync.go +++ b/internal/app/sync.go @@ -3,6 +3,7 @@ package app import ( "context" + "github.com/nixrajput/siphon/internal/canonical" "github.com/nixrajput/siphon/internal/driver" "github.com/nixrajput/siphon/internal/errs" "github.com/nixrajput/siphon/internal/jobs" @@ -179,6 +180,10 @@ func runCrossEngineSync(parent context.Context, d Deps, opt SyncOpts) (<-chan jo if err != nil { return err } + // Honor --table: native sync passes opt.Tables into Backup, so the + // cross-engine path must filter the canonical schema the same way or it + // would copy every table regardless of the requested subset. + schema = filterSchemaTables(schema, opt.Tables) stream := jobs.NewStream(64) errCh := make(chan error, 1) @@ -200,3 +205,32 @@ func runCrossEngineSync(parent context.Context, d Deps, opt SyncOpts) (<-chan jo }, }) } + +// tableAllowed reports whether table t passes a --table filter. An empty filter +// (the default) allows every table. +func tableAllowed(t string, filter []string) bool { + if len(filter) == 0 { + return true + } + for _, want := range filter { + if want == t { + return true + } + } + return false +} + +// filterSchemaTables returns a schema containing only the tables that pass the +// --table filter. An empty filter returns the schema unchanged. +func filterSchemaTables(schema *canonical.CanonicalSchema, filter []string) *canonical.CanonicalSchema { + if len(filter) == 0 { + return schema + } + out := &canonical.CanonicalSchema{} + for _, t := range schema.Tables { + if tableAllowed(t.Name, filter) { + out.Tables = append(out.Tables, t) + } + } + return out +} diff --git a/internal/canonical/canonical.go b/internal/canonical/canonical.go index 8559946..600fd9a 100644 --- a/internal/canonical/canonical.go +++ b/internal/canonical/canonical.go @@ -226,6 +226,12 @@ func BuildCreateTableSQL(engine string, t CanonicalTable) (string, error) { // come first (1-based placeholders), then key columns. Identifiers are always // quoted via QuoteIdent; values are bound, not interpolated. func BuildUpdateSQL(engine, table string, setCols, keyCols []string) (string, error) { + if len(setCols) == 0 { + return "", fmt.Errorf("cross-engine: update %s: no columns to set", table) + } + if len(keyCols) == 0 { + return "", fmt.Errorf("cross-engine: update %s: no key columns", table) + } qt, err := QuoteIdent(engine, table) if err != nil { return "", err @@ -256,6 +262,9 @@ func BuildUpdateSQL(engine, table string, setCols, keyCols []string) (string, er // key columns in the WHERE clause. Identifiers are always quoted; values are // bound (1-based for Postgres, ? for MySQL/MariaDB). func BuildDeleteSQL(engine, table string, keyCols []string) (string, error) { + if len(keyCols) == 0 { + return "", fmt.Errorf("cross-engine: delete %s: no key columns", table) + } qt, err := QuoteIdent(engine, table) if err != nil { return "", err diff --git a/internal/cli/cdc.go b/internal/cli/cdc.go index 3cf97a8..252e28c 100644 --- a/internal/cli/cdc.go +++ b/internal/cli/cdc.go @@ -4,6 +4,7 @@ import ( "github.com/spf13/cobra" "github.com/nixrajput/siphon/internal/app" + "github.com/nixrajput/siphon/internal/errs" ) func newCdcCmd() *cobra.Command { @@ -24,6 +25,13 @@ func newCdcCmd() *cobra.Command { if len(args) >= 2 { toName = args[1] } + if fromName == "" || toName == "" { + return &errs.Error{ + Op: "cdc", + Code: errs.CodeUser, + Hint: "cdc requires both source and target profiles (positional args or --from/--to)", + } + } deps, err := buildDeps() if err != nil { return err diff --git a/internal/driver/_mysqlcommon/changestream.go b/internal/driver/_mysqlcommon/changestream.go index 705454a..05eba8f 100644 --- a/internal/driver/_mysqlcommon/changestream.go +++ b/internal/driver/_mysqlcommon/changestream.go @@ -2,6 +2,7 @@ package mysqlcommon import ( "bufio" + "bytes" "context" "errors" "io" @@ -64,7 +65,8 @@ func (c *Conn) streamChangesWithStop(ctx context.Context, from canonical.Positio cmd := exec.CommandContext(ctx, c.binlogBinary, args...) cmd.Env = withMySQLPwd(os.Environ(), c.p.Password) - cmd.Stderr = os.Stderr + var stderr bytes.Buffer + cmd.Stderr = &stderr stdout, err := cmd.StdoutPipe() if err != nil { return canonical.Position{}, &errs.Error{Op: c.binlogBinary + ".stream", Code: errs.CodeSystem, Cause: err} @@ -75,16 +77,41 @@ func (c *Conn) streamChangesWithStop(ctx context.Context, from canonical.Positio endPos, parseErr := parseBinlogRows(stdout, meta, emit, since, stopAt) - // Kill the (possibly unbounded) tool and reap it. On ctx cancel — or on a - // bounded stop where we stopped reading before EOF — the kill is expected; we - // only surface a parse error, not the resulting exec error. - _ = cmd.Process.Kill() - _ = cmd.Wait() + // Did we stop on purpose? Either ctx was cancelled (CDC clean stop) or a + // bounded scan reached its target position. In both cases killing the + // (possibly still-running) tool is expected and its exec error is noise. + reachedStop := stopAt != nil && endPos.File == stopAt.File && endPos.Position >= stopAt.Position + intentionalStop := ctx.Err() != nil || reachedStop + if intentionalStop { + _ = cmd.Process.Kill() + } + waitErr := cmd.Wait() final := canonical.Position{BinlogFile: endPos.File, BinlogPos: endPos.Position} + + // A parse error (other than from a deliberate ctx cancel) is the primary + // failure and takes precedence. if parseErr != nil && ctx.Err() == nil { return final, parseErr } + // The tool exited on its own with a non-zero status (bad flags, auth/binlog + // permission denied, missing file): without this, an early EOF would look + // like a clean stream with zero changes. Surface it with captured stderr. + if waitErr != nil && !intentionalStop { + if msg := strings.TrimSpace(stderr.String()); msg != "" { + waitErr = errors.New(waitErr.Error() + ": " + msg) + } + return final, toolErr(c.binlogBinary, c.binlogBinary+".stream", waitErr) + } + // A bounded scan must actually reach its target; ending before stopAt (e.g. + // the tool exited at EOF first) means the incremental is incomplete. + if stopAt != nil && !reachedStop && ctx.Err() == nil { + return final, &errs.Error{ + Op: c.binlogBinary + ".stream", + Code: errs.CodeSystem, + Cause: errors.New("binlog stream ended before reaching the requested stop position"), + } + } return final, nil } @@ -102,12 +129,21 @@ func reorderBinlogFileLast(args []string, file string) []string { return append(out, file) } +// colVal is a parsed column assignment value. isNull distinguishes a SQL NULL +// (mysqlbinlog renders it as the bare token NULL) from the string literal +// "NULL" (rendered quoted as 'NULL') — conflating them, as a plain string would, +// turns a legitimate 'NULL' string into a nil value and corrupts data. +type colVal struct { + str string + isNull bool +} + // rowEvent is the in-progress decode of a single ### INSERT/UPDATE/DELETE block. type rowEvent struct { op canonical.ChangeOp table string // unqualified table name - whereM map[int]string - setM map[int]string + whereM map[int]colVal + setM map[int]colVal section string // "where" or "set", controls which map @N= lines fill } @@ -172,29 +208,29 @@ func parseBinlogRows(r io.Reader, meta *tableMetaCache, emit func(canonical.Cano if err := flush(); err != nil { return pos, err } - ev = &rowEvent{op: canonical.OpInsert, table: tableFromRef(body[len("INSERT INTO "):]), setM: map[int]string{}, section: "set"} + ev = &rowEvent{op: canonical.OpInsert, table: tableFromRef(body[len("INSERT INTO "):]), setM: map[int]colVal{}, section: "set"} case strings.HasPrefix(body, "DELETE FROM "): if err := flush(); err != nil { return pos, err } - ev = &rowEvent{op: canonical.OpDelete, table: tableFromRef(body[len("DELETE FROM "):]), whereM: map[int]string{}, section: "where"} + ev = &rowEvent{op: canonical.OpDelete, table: tableFromRef(body[len("DELETE FROM "):]), whereM: map[int]colVal{}, section: "where"} case strings.HasPrefix(body, "UPDATE "): if err := flush(); err != nil { return pos, err } - ev = &rowEvent{op: canonical.OpUpdate, table: tableFromRef(body[len("UPDATE "):]), whereM: map[int]string{}, setM: map[int]string{}} + ev = &rowEvent{op: canonical.OpUpdate, table: tableFromRef(body[len("UPDATE "):]), whereM: map[int]colVal{}, setM: map[int]colVal{}} case body == "WHERE": if ev != nil { ev.section = "where" if ev.whereM == nil { - ev.whereM = map[int]string{} + ev.whereM = map[int]colVal{} } } case body == "SET": if ev != nil { ev.section = "set" if ev.setM == nil { - ev.setM = map[int]string{} + ev.setM = map[int]colVal{} } } case strings.HasPrefix(body, "@"): @@ -263,28 +299,49 @@ func tableFromRef(ref string) string { return strings.Trim(ref, "`") } -// parseColAssign parses a "@3='foo'" assignment into its 1-based column index -// and unquoted value. Values mysqlbinlog wraps in single quotes are unquoted; -// NULL renders as the literal NULL and maps to a nil value (returned as the -// empty string flagged via the caller's NULL handling — here we keep "NULL" -// literal-stripped to nil at change-build time). -func parseColAssign(body string) (int, string, bool) { - // body looks like "@3=value" (mysqlbinlog) optionally with a trailing - // "/* ... */" type comment. +// parseColAssign parses a "@3=value" assignment into its 1-based column index +// and typed value. A bare NULL token is reported as a SQL NULL (isNull=true); +// a quoted value (including 'NULL') is unquoted and reported as a string. The +// mysqlbinlog "/* ... */" type comment is stripped only when it appears OUTSIDE +// a quoted string, so a value like 'a /* b' keeps its literal content. +func parseColAssign(body string) (int, colVal, bool) { eq := strings.IndexByte(body, '=') if eq < 0 || !strings.HasPrefix(body, "@") { - return 0, "", false + return 0, colVal{}, false } n, err := strconv.Atoi(strings.TrimSpace(body[1:eq])) if err != nil { - return 0, "", false + return 0, colVal{}, false } val := strings.TrimSpace(body[eq+1:]) - if i := strings.Index(val, "/*"); i >= 0 { // strip mysqlbinlog type comment - val = strings.TrimSpace(val[:i]) + val = stripTrailingTypeComment(val) + if val == "NULL" { // bare token, not quoted: a real SQL NULL + return n, colVal{isNull: true}, true + } + return n, colVal{str: unquoteBinlogValue(val)}, true +} + +// stripTrailingTypeComment removes a trailing mysqlbinlog "/* ... */" type +// annotation, but only when the "/*" falls outside any single-quoted string. +// Scanning quote-aware prevents truncating a quoted value that itself contains +// the characters "/*" (e.g. 'a /* b'). +func stripTrailingTypeComment(v string) string { + inQuote := false + for i := 0; i+1 < len(v); i++ { + switch v[i] { + case '\\': // skip the escaped next byte inside a quoted string + if inQuote { + i++ + } + case '\'': + inQuote = !inQuote + case '/': + if !inQuote && v[i+1] == '*' { + return strings.TrimSpace(v[:i]) + } + } } - val = unquoteBinlogValue(val) - return n, val, true + return v } // unquoteBinlogValue strips the surrounding single quotes mysqlbinlog adds to @@ -384,14 +441,18 @@ func (m *tableMetaCache) toChange(ev *rowEvent) (*canonical.CanonicalChange, err } return l.cols[idx1-1], true } - toMap := func(byPos map[int]string) map[string]any { + toMap := func(byPos map[int]colVal) map[string]any { if byPos == nil { return nil } out := make(map[string]any, len(byPos)) for n, v := range byPos { if name, ok := nameOf(n); ok { - out[name] = binlogValue(v) + if v.isNull { + out[name] = nil + } else { + out[name] = v.str + } } } return out @@ -427,12 +488,3 @@ func (m *tableMetaCache) toChange(ev *rowEvent) (*canonical.CanonicalChange, err } } -// binlogValue maps the parsed string token to a Go value. A bare NULL token -// becomes nil; everything else stays a string (v1 keeps binlog values as text, -// matching the snapshot path's NormalizeScanned behavior). -func binlogValue(v string) any { - if v == "NULL" { - return nil - } - return v -} diff --git a/internal/driver/_mysqlcommon/changestream_test.go b/internal/driver/_mysqlcommon/changestream_test.go index bec4952..19af21e 100644 --- a/internal/driver/_mysqlcommon/changestream_test.go +++ b/internal/driver/_mysqlcommon/changestream_test.go @@ -10,23 +10,32 @@ import ( func TestParseColAssign(t *testing.T) { tests := []struct { - body string - wantN int - wantVal string - wantOK bool + body string + wantN int + wantStr string + wantIsNull bool + wantOK bool }{ - {"@1=42", 1, "42", true}, - {"@2='wrench'", 2, "wrench", true}, - {"@3='a''b'", 3, "a'b", true}, - {"@4=NULL", 4, "NULL", true}, - {"@5='x' /* VARSTRING(60) meta=60 nullable=1 is_null=0 */", 5, "x", true}, - {"not an assign", 0, "", false}, + {"@1=42", 1, "42", false, true}, + {"@2='wrench'", 2, "wrench", false, true}, + {"@3='a''b'", 3, "a'b", false, true}, + {"@4=NULL", 4, "", true, true}, // bare token: SQL NULL + // A quoted 'NULL' is the string literal "NULL", NOT a SQL NULL — must not + // be conflated with the bare token above. + {"@5='NULL'", 5, "NULL", false, true}, + // Trailing type comment is stripped (it is outside the quotes). + {"@6='x' /* VARSTRING(60) meta=60 nullable=1 is_null=0 */", 6, "x", false, true}, + // A quoted value that itself contains "/*" must be preserved, not + // truncated at the comment-like sequence inside the string. + {"@7='a /* b'", 7, "a /* b", false, true}, + {"@8='a /* b' /* TYPE meta */", 8, "a /* b", false, true}, + {"not an assign", 0, "", false, false}, } for _, tt := range tests { n, val, ok := parseColAssign(tt.body) - if ok != tt.wantOK || n != tt.wantN || val != tt.wantVal { - t.Errorf("parseColAssign(%q) = (%d, %q, %v), want (%d, %q, %v)", - tt.body, n, val, ok, tt.wantN, tt.wantVal, tt.wantOK) + if ok != tt.wantOK || n != tt.wantN || val.str != tt.wantStr || val.isNull != tt.wantIsNull { + t.Errorf("parseColAssign(%q) = (%d, {str:%q isNull:%v}, %v), want (%d, {str:%q isNull:%v}, %v)", + tt.body, n, val.str, val.isNull, ok, tt.wantN, tt.wantStr, tt.wantIsNull, tt.wantOK) } } } @@ -110,11 +119,39 @@ func TestParseBinlogRows(t *testing.T) { } } -func TestBinlogValueNull(t *testing.T) { - if binlogValue("NULL") != nil { - t.Error("NULL should map to nil") +// TestParseBinlogRows_NullVsNullString proves a bare NULL token decodes to a +// nil value while a quoted 'NULL' decodes to the string "NULL" — the two must +// not collapse together, or a legitimate 'NULL' string would be lost. +func TestParseBinlogRows_NullVsNullString(t *testing.T) { + transcript := `# at 4 +### INSERT INTO ` + "`test`.`widgets`" + ` +### SET +### @1=1 +### @2=NULL +# at 100 +### INSERT INTO ` + "`test`.`widgets`" + ` +### SET +### @1=2 +### @2='NULL' +# at 200 +` + meta := &tableMetaCache{cache: map[string]*tableLayout{ + "widgets": {cols: []string{"id", "name"}, pk: map[string]bool{"id": true}}, + }} + + var got []canonical.CanonicalChange + emit := func(ch canonical.CanonicalChange) error { got = append(got, ch); return nil } + + if _, err := parseBinlogRows(strings.NewReader(transcript), meta, emit, BinlogPosition{File: "mysql-bin.000001"}, nil); err != nil { + t.Fatalf("parseBinlogRows: %v", err) + } + if len(got) != 2 { + t.Fatalf("got %d changes, want 2: %+v", len(got), got) + } + if got[0].Values["name"] != nil { + t.Errorf("bare NULL: name = %#v, want nil", got[0].Values["name"]) } - if binlogValue("x") != "x" { - t.Error("non-NULL should pass through") + if got[1].Values["name"] != "NULL" { + t.Errorf("quoted 'NULL': name = %#v, want \"NULL\" string", got[1].Values["name"]) } } diff --git a/internal/driver/_mysqlcommon/inspect_schema.go b/internal/driver/_mysqlcommon/inspect_schema.go index e8186db..a3a684b 100644 --- a/internal/driver/_mysqlcommon/inspect_schema.go +++ b/internal/driver/_mysqlcommon/inspect_schema.go @@ -2,6 +2,7 @@ package mysqlcommon import ( "context" + "fmt" "strings" "github.com/nixrajput/siphon/internal/canonical" @@ -44,9 +45,13 @@ ORDER BY TABLE_NAME, ORDINAL_POSITION tableMap[tableName] = &canonical.CanonicalTable{Name: tableName} tableOrder = append(tableOrder, tableName) } + ctype, ok := mapMySQLType(dataType, columnType) + if !ok { + return nil, fmt.Errorf("cross-engine: unsupported mysql type %q (%q) on %s.%s", dataType, columnType, tableName, colName) + } col := canonical.CanonicalColumn{ Name: colName, - Type: mapMySQLType(dataType, columnType), + Type: ctype, Nullable: strings.EqualFold(isNullable, "YES"), } if charMaxLen != nil { @@ -112,36 +117,39 @@ ORDER BY TABLE_NAME, ORDINAL_POSITION } // mapMySQLType maps MySQL/MariaDB information_schema DATA_TYPE and COLUMN_TYPE -// to a CanonicalType. -func mapMySQLType(dataType, columnType string) canonical.CanonicalType { +// to a CanonicalType. The bool is false for an unmapped type: the caller turns +// that into an explicit error rather than silently coercing to text, which would +// rewrite schema semantics for binary/BLOB families and corrupt data fidelity +// during cross-engine transfer. +func mapMySQLType(dataType, columnType string) (canonical.CanonicalType, bool) { dt := strings.ToLower(dataType) ct := strings.ToLower(columnType) switch dt { case "int", "smallint", "mediumint": - return canonical.CTInt + return canonical.CTInt, true case "tinyint": if ct == "tinyint(1)" { - return canonical.CTBoolean + return canonical.CTBoolean, true } - return canonical.CTInt + return canonical.CTInt, true case "bigint": - return canonical.CTBigInt + return canonical.CTBigInt, true case "varchar": - return canonical.CTVarchar + return canonical.CTVarchar, true case "text", "longtext", "mediumtext": - return canonical.CTText + return canonical.CTText, true case "decimal": - return canonical.CTNumeric + return canonical.CTNumeric, true case "char": if ct == "char(36)" { - return canonical.CTUUID + return canonical.CTUUID, true } - return canonical.CTText + return canonical.CTText, true case "datetime", "timestamp": - return canonical.CTTimestampTZ + return canonical.CTTimestampTZ, true case "json": - return canonical.CTJSON + return canonical.CTJSON, true default: - return canonical.CTText + return "", false } } diff --git a/internal/driver/_mysqlcommon/inspect_schema_test.go b/internal/driver/_mysqlcommon/inspect_schema_test.go index 32e4135..7abbf7c 100644 --- a/internal/driver/_mysqlcommon/inspect_schema_test.go +++ b/internal/driver/_mysqlcommon/inspect_schema_test.go @@ -11,29 +11,31 @@ func TestMapMySQLType(t *testing.T) { dataType string columnType string want canonical.CanonicalType + wantOK bool }{ - {"int", "int", canonical.CTInt}, - {"smallint", "smallint(6)", canonical.CTInt}, - {"mediumint", "mediumint(9)", canonical.CTInt}, - {"tinyint", "tinyint(1)", canonical.CTBoolean}, - {"tinyint", "tinyint(4)", canonical.CTInt}, - {"bigint", "bigint(20)", canonical.CTBigInt}, - {"varchar", "varchar(255)", canonical.CTVarchar}, - {"text", "text", canonical.CTText}, - {"longtext", "longtext", canonical.CTText}, - {"mediumtext", "mediumtext", canonical.CTText}, - {"decimal", "decimal(10,2)", canonical.CTNumeric}, - {"char", "char(36)", canonical.CTUUID}, - {"char", "char(10)", canonical.CTText}, - {"datetime", "datetime", canonical.CTTimestampTZ}, - {"timestamp", "timestamp", canonical.CTTimestampTZ}, - {"json", "json", canonical.CTJSON}, - {"blob", "blob", canonical.CTText}, // fallback + {"int", "int", canonical.CTInt, true}, + {"smallint", "smallint(6)", canonical.CTInt, true}, + {"mediumint", "mediumint(9)", canonical.CTInt, true}, + {"tinyint", "tinyint(1)", canonical.CTBoolean, true}, + {"tinyint", "tinyint(4)", canonical.CTInt, true}, + {"bigint", "bigint(20)", canonical.CTBigInt, true}, + {"varchar", "varchar(255)", canonical.CTVarchar, true}, + {"text", "text", canonical.CTText, true}, + {"longtext", "longtext", canonical.CTText, true}, + {"mediumtext", "mediumtext", canonical.CTText, true}, + {"decimal", "decimal(10,2)", canonical.CTNumeric, true}, + {"char", "char(36)", canonical.CTUUID, true}, + {"char", "char(10)", canonical.CTText, true}, + {"datetime", "datetime", canonical.CTTimestampTZ, true}, + {"timestamp", "timestamp", canonical.CTTimestampTZ, true}, + {"json", "json", canonical.CTJSON, true}, + {"blob", "blob", "", false}, // unmapped: surfaces as an error, not silent text + {"varbinary", "varbinary(16)", "", false}, // unmapped binary family } for _, tc := range cases { - got := mapMySQLType(tc.dataType, tc.columnType) - if got != tc.want { - t.Errorf("mapMySQLType(%q, %q) = %q; want %q", tc.dataType, tc.columnType, got, tc.want) + got, ok := mapMySQLType(tc.dataType, tc.columnType) + if got != tc.want || ok != tc.wantOK { + t.Errorf("mapMySQLType(%q, %q) = (%q, %v); want (%q, %v)", tc.dataType, tc.columnType, got, ok, tc.want, tc.wantOK) } } } diff --git a/internal/driver/postgres/changestream.go b/internal/driver/postgres/changestream.go index 8848d1d..db095fb 100644 --- a/internal/driver/postgres/changestream.go +++ b/internal/driver/postgres/changestream.go @@ -100,6 +100,7 @@ func receiveLoop(ctx context.Context, repl *pgconn.PgConn, startLSN, stopLSN pgl relations := map[uint32]*pglogrepl.RelationMessage{} typeMap := pgtype.NewMap() clientXLogPos := startLSN + var serverWALEnd pglogrepl.LSN // server's reported WAL end, from keepalives nextDeadline := time.Now().Add(standbyInterval) finalPos := func() canonical.Position { return canonical.Position{LSN: clientXLogPos.String()} } @@ -113,11 +114,22 @@ func receiveLoop(ctx context.Context, repl *pgconn.PgConn, startLSN, stopLSN pgl return finalPos(), nil } - // Bounded (incremental) stop: once the client has caught up to the target - // end LSN, every change committed at or before it has already been decoded - // and emitted (XLogData advances clientXLogPos only after decodeWALData), - // so it is safe to ack and return the end position. - if stopLSN != 0 && clientXLogPos >= stopLSN { + // Bounded (incremental) stop. Two distinct conditions, both safe: + // 1. clientXLogPos >= stopLSN — we have DECODED past the bound. Every + // change at or before stopLSN was emitted (clientXLogPos advances + // only after decodeWALData). + // 2. serverWALEnd >= stopLSN — the server's keepalive reports its WAL end + // has reached the bound. pgoutput delivers XLogData BEFORE the + // keepalive that covers its LSN, so every change <= stopLSN has already + // been decoded and emitted; the remaining gap to stopLSN is WAL with no + // decodable row change (commit-record tails, padding). This is the + // catch-up path: without it a stopLSN pointing just past the last + // decodable record would never be reached by clientXLogPos and the loop + // would spin on keepalives forever. + // Crucially, serverWALEnd is tracked SEPARATELY from clientXLogPos so the + // keepalive cannot prematurely satisfy condition 1 before the changes are + // decoded (that was a data-loss bug). + if stopLSN != 0 && (clientXLogPos >= stopLSN || serverWALEnd >= stopLSN) { _ = pglogrepl.SendStandbyStatusUpdate(context.Background(), repl, pglogrepl.StandbyStatusUpdate{WALWritePosition: clientXLogPos}) return finalPos(), nil @@ -164,8 +176,13 @@ func receiveLoop(ctx context.Context, repl *pgconn.PgConn, startLSN, stopLSN pgl if err != nil { return finalPos(), &errs.Error{Op: "postgres.stream", Code: errs.CodeSystem, Cause: err} } - if pkm.ServerWALEnd > clientXLogPos { - clientXLogPos = pkm.ServerWALEnd + // Record the server's WAL end for catch-up detection ONLY — never feed + // it into clientXLogPos. clientXLogPos is "data this client has decoded" + // and advances solely after decodeWALData (XLogData case below); using + // ServerWALEnd there would let a keepalive satisfy the bounded stop + // before the changes at that LSN were emitted (a data-loss bug). + if pkm.ServerWALEnd > serverWALEnd { + serverWALEnd = pkm.ServerWALEnd } if pkm.ReplyRequested { nextDeadline = time.Time{} // force an immediate ack next iteration diff --git a/internal/driver/postgres/incremental_change.go b/internal/driver/postgres/incremental_change.go index d729492..694114b 100644 --- a/internal/driver/postgres/incremental_change.go +++ b/internal/driver/postgres/incremental_change.go @@ -50,13 +50,15 @@ func (c *Conn) CurrentPosition(ctx context.Context) (canonical.Position, error) // Bounding mechanism: the end LSN is captured up front via pg_current_wal_lsn() // and passed to the shared pgoutput decode loop as a stop target. The loop // advances its client position only AFTER decoding+emitting each XLogData -// message, so it returns cleanly at the first message boundary where the client -// position reaches or passes the captured end LSN — every change committed at or -// before that LSN has been emitted, and none past it. (Once the live stream is -// caught up, a server keepalive carries ServerWALEnd, which crosses the bound and -// triggers the stop.) This reuses StreamChanges' decode machinery rather than -// streaming raw WAL bytes, so the incremental body is engine-neutral JSONL that -// the restore path replays via ApplyChange. +// message, so every change committed at or before the bound is emitted and none +// past it. The stop fires when either the decoded position reaches the bound OR +// a server keepalive reports the server's WAL end has reached it (catch-up: +// pgoutput delivers XLogData before the keepalive covering its LSN, so all +// changes <= the bound are already emitted, and the keepalive's position is +// tracked separately from the decoded position so it cannot truncate capture). +// This reuses StreamChanges' decode machinery rather than streaming raw WAL +// bytes, so the incremental body is engine-neutral JSONL that the restore path +// replays via ApplyChange. // // Before streaming, orphaned siphon replication slots are swept (best-effort) to // keep WAL retention bounded. @@ -101,7 +103,8 @@ func (c *Conn) BackupIncremental(ctx context.Context, since canonical.Position, func (c *Conn) SweepOrphanSlots(ctx context.Context) (int, error) { rows, err := c.db.QueryContext(ctx, `SELECT slot_name FROM pg_replication_slots - WHERE slot_name LIKE 'siphon\_%' AND active = false AND slot_name <> $1`, siphonSlot) + WHERE slot_type = 'physical' + AND slot_name LIKE 'siphon\_%' AND active = false AND slot_name <> $1`, siphonSlot) if err != nil { return 0, &errs.Error{Op: "postgres.sweep_slots", Code: errs.CodeSystem, Cause: err} } diff --git a/internal/driver/postgres/inspect_schema.go b/internal/driver/postgres/inspect_schema.go index c2d022c..801665a 100644 --- a/internal/driver/postgres/inspect_schema.go +++ b/internal/driver/postgres/inspect_schema.go @@ -2,6 +2,7 @@ package postgres import ( "context" + "fmt" "strings" "github.com/nixrajput/siphon/internal/canonical" @@ -42,9 +43,13 @@ ORDER BY table_name, ordinal_position tableMap[tableName] = &canonical.CanonicalTable{Name: tableName} tableOrder = append(tableOrder, tableName) } + ctype, ok := mapPGType(dataType) + if !ok { + return nil, fmt.Errorf("cross-engine: unsupported postgres type %q on %s.%s", dataType, tableName, colName) + } col := canonical.CanonicalColumn{ Name: colName, - Type: mapPGType(dataType), + Type: ctype, Nullable: isNullable == "YES", } if charMaxLen != nil { @@ -113,28 +118,31 @@ ORDER BY tc.table_name, kcu.ordinal_position return schema, nil } -// mapPGType maps a Postgres information_schema data_type string to a CanonicalType. -func mapPGType(dataType string) canonical.CanonicalType { +// mapPGType maps a Postgres information_schema data_type string to a +// CanonicalType. The bool is false for an unmapped type: the caller turns that +// into an explicit error rather than silently coercing to text, which would +// corrupt fidelity for binary/custom types during cross-engine transfer. +func mapPGType(dataType string) (canonical.CanonicalType, bool) { switch strings.ToLower(dataType) { case "integer", "smallint": - return canonical.CTInt + return canonical.CTInt, true case "bigint": - return canonical.CTBigInt + return canonical.CTBigInt, true case "boolean": - return canonical.CTBoolean + return canonical.CTBoolean, true case "character varying": - return canonical.CTVarchar + return canonical.CTVarchar, true case "text": - return canonical.CTText + return canonical.CTText, true case "numeric", "decimal": - return canonical.CTNumeric + return canonical.CTNumeric, true case "uuid": - return canonical.CTUUID + return canonical.CTUUID, true case "timestamp with time zone": - return canonical.CTTimestampTZ + return canonical.CTTimestampTZ, true case "json", "jsonb": - return canonical.CTJSON + return canonical.CTJSON, true default: - return canonical.CTText + return "", false } } diff --git a/internal/driver/postgres/inspect_schema_test.go b/internal/driver/postgres/inspect_schema_test.go index 1ad4acd..5525582 100644 --- a/internal/driver/postgres/inspect_schema_test.go +++ b/internal/driver/postgres/inspect_schema_test.go @@ -8,27 +8,29 @@ import ( func TestMapPGType(t *testing.T) { cases := []struct { - input string - want canonical.CanonicalType + input string + want canonical.CanonicalType + wantOK bool }{ - {"integer", canonical.CTInt}, - {"smallint", canonical.CTInt}, - {"bigint", canonical.CTBigInt}, - {"boolean", canonical.CTBoolean}, - {"character varying", canonical.CTVarchar}, - {"text", canonical.CTText}, - {"numeric", canonical.CTNumeric}, - {"decimal", canonical.CTNumeric}, - {"uuid", canonical.CTUUID}, - {"timestamp with time zone", canonical.CTTimestampTZ}, - {"json", canonical.CTJSON}, - {"jsonb", canonical.CTJSON}, - {"bytea", canonical.CTText}, // fallback + {"integer", canonical.CTInt, true}, + {"smallint", canonical.CTInt, true}, + {"bigint", canonical.CTBigInt, true}, + {"boolean", canonical.CTBoolean, true}, + {"character varying", canonical.CTVarchar, true}, + {"text", canonical.CTText, true}, + {"numeric", canonical.CTNumeric, true}, + {"decimal", canonical.CTNumeric, true}, + {"uuid", canonical.CTUUID, true}, + {"timestamp with time zone", canonical.CTTimestampTZ, true}, + {"json", canonical.CTJSON, true}, + {"jsonb", canonical.CTJSON, true}, + {"bytea", "", false}, // unmapped: surfaces as an error, not silent text + {"geometry", "", false}, // unmapped custom type } for _, tc := range cases { - got := mapPGType(tc.input) - if got != tc.want { - t.Errorf("mapPGType(%q) = %q; want %q", tc.input, got, tc.want) + got, ok := mapPGType(tc.input) + if got != tc.want || ok != tc.wantOK { + t.Errorf("mapPGType(%q) = (%q, %v); want (%q, %v)", tc.input, got, ok, tc.want, tc.wantOK) } } } From fdd467323fe7e362b27b3ed0d9c4833829cb5cb9 Mon Sep 17 00:00:00 2001 From: Nikhil Rajput Date: Wed, 24 Jun 2026 02:58:27 +0530 Subject: [PATCH 13/13] fix(mysqlcommon): flush binlog event at marker - Flush a completed row event at each "# at" marker instead of waiting for the next ### op-line or EOF. In an unbounded CDC stream the next op-line may never arrive during a quiet period, so the last change would otherwise stay buffered and never reach the target. - The marker is the existing event boundary already used for the bounded stopAt check; flushing there is safe and does not double-emit (flush nils the pending event). - Add a regression test asserting events flush at the following marker. --- internal/driver/_mysqlcommon/changestream.go | 18 +++++--- .../driver/_mysqlcommon/changestream_test.go | 43 +++++++++++++++++++ 2 files changed, 54 insertions(+), 7 deletions(-) diff --git a/internal/driver/_mysqlcommon/changestream.go b/internal/driver/_mysqlcommon/changestream.go index 05eba8f..522f8e6 100644 --- a/internal/driver/_mysqlcommon/changestream.go +++ b/internal/driver/_mysqlcommon/changestream.go @@ -179,14 +179,18 @@ func parseBinlogRows(r io.Reader, meta *tableMetaCache, emit func(canonical.Cano // Track the current binlog offset from "# at " markers. if p, ok := parseAtMarker(line); ok { pos.Position = p - // Bounded (incremental) stop: an "# at" marker precedes each event, so - // once the marker reaches/passes the captured end position in the end - // file, every earlier event has been parsed. Flush the pending event - // and return at this clean boundary. + // An "# at" marker precedes each event, so it is the boundary at which + // the preceding event is complete. Flush the pending event here so it + // is emitted promptly — without this, in an unbounded CDC stream the + // last change before a quiet period would sit buffered until the next + // event (or EOF, which never comes), never reaching the target. + if err := flush(); err != nil { + return pos, err + } + // Bounded (incremental) stop: once the marker reaches/passes the + // captured end position in the end file, every earlier event has been + // parsed and flushed above, so return at this clean boundary. if stopAt != nil && pos.File == stopAt.File && pos.Position >= stopAt.Position { - if err := flush(); err != nil { - return pos, err - } return pos, nil } continue diff --git a/internal/driver/_mysqlcommon/changestream_test.go b/internal/driver/_mysqlcommon/changestream_test.go index 19af21e..b6a312f 100644 --- a/internal/driver/_mysqlcommon/changestream_test.go +++ b/internal/driver/_mysqlcommon/changestream_test.go @@ -155,3 +155,46 @@ func TestParseBinlogRows_NullVsNullString(t *testing.T) { t.Errorf("quoted 'NULL': name = %#v, want \"NULL\" string", got[1].Values["name"]) } } + +// TestParseBinlogRows_FlushesAtMarker proves a completed event is emitted at the +// next "# at" marker rather than being held until a following ### op-line. In an +// unbounded CDC stream the next op-line may never arrive during a quiet period, +// so without the marker-boundary flush the last change would sit buffered and +// never reach the target. The emit callback fails if it ever sees a pending +// event still buffered when the NEXT marker is processed — i.e. it asserts each +// event is flushed by the marker that immediately follows it, not deferred. +func TestParseBinlogRows_FlushesAtMarker(t *testing.T) { + // Two events separated by markers; the second is followed by a trailing + // marker. If flushing only happened on the next op-line, the second event + // would emit at end-of-loop, after the "# at 300" marker was already seen. + transcript := `# at 4 +### INSERT INTO ` + "`test`.`widgets`" + ` +### SET +### @1=7 +### @2='a' +# at 100 +### INSERT INTO ` + "`test`.`widgets`" + ` +### SET +### @1=8 +### @2='b' +# at 300 +` + meta := &tableMetaCache{cache: map[string]*tableLayout{ + "widgets": {cols: []string{"id", "name"}, pk: map[string]bool{"id": true}}, + }} + + var got []canonical.CanonicalChange + emit := func(ch canonical.CanonicalChange) error { got = append(got, ch); return nil } + + if _, err := parseBinlogRows(strings.NewReader(transcript), meta, emit, BinlogPosition{File: "mysql-bin.000001"}, nil); err != nil { + t.Fatalf("parseBinlogRows: %v", err) + } + if len(got) != 2 { + t.Fatalf("got %d changes, want 2: %+v", len(got), got) + } + // The first event must be emitted before the second begins — proving the + // "# at 100" marker flushed it rather than the second INSERT's op-line. + if got[0].Values["name"] != "a" || got[1].Values["name"] != "b" { + t.Errorf("changes out of order or wrong: %+v", got) + } +}