diff --git a/go.mod b/go.mod index b9482b6f..1aebf94a 100644 --- a/go.mod +++ b/go.mod @@ -25,6 +25,7 @@ require ( github.com/stretchr/testify v1.11.1 golang.org/x/exp v0.0.0-20240716160929-1d5bc16f04a8 golang.org/x/sync v0.19.0 + turso.tech/database/tursogo v0.7.1 ) require ( @@ -34,11 +35,13 @@ require ( github.com/charmbracelet/x/cellbuf v0.0.13-0.20250311204145-2c3ea96c31dd // indirect github.com/charmbracelet/x/term v0.2.1 // indirect github.com/davecgh/go-spew v1.1.1 // indirect + github.com/ebitengine/purego v0.9.1 // indirect github.com/erikgeiser/coninput v0.0.0-20211004153227-1c3628e74d0f // indirect github.com/go-viper/mapstructure/v2 v2.4.0 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/sagikazarmark/locafero v0.11.0 // indirect github.com/sourcegraph/conc v0.3.1-0.20240121214520-5f936abd7ae8 // indirect + github.com/tursodatabase/turso-go-platform-libs v0.7.1 // indirect github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect go.yaml.in/yaml/v3 v3.0.4 // indirect ) @@ -58,7 +61,7 @@ require ( github.com/mattn/go-isatty v0.0.20 github.com/mattn/go-localereader v0.0.1 // indirect github.com/mattn/go-runewidth v0.0.16 // indirect - github.com/mattn/go-sqlite3 v1.14.16 // indirect + github.com/mattn/go-sqlite3 v1.14.42 // indirect github.com/muesli/ansi v0.0.0-20230316100256-276c6243b2f6 // indirect github.com/muesli/cancelreader v0.2.2 // indirect github.com/muesli/termenv v0.16.0 // indirect diff --git a/go.sum b/go.sum index 285fe41c..84176d47 100644 --- a/go.sum +++ b/go.sum @@ -43,6 +43,8 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= +github.com/ebitengine/purego v0.9.1 h1:a/k2f2HQU3Pi399RPW1MOaZyhKJL9w/xFpKAg4q1s0A= +github.com/ebitengine/purego v0.9.1/go.mod h1:iIjxzd6CiRiOG0UyXP+V1+jWqUXVjPKLAI0mRfJZTmQ= github.com/erikgeiser/coninput v0.0.0-20211004153227-1c3628e74d0f h1:Y/CXytFA4m6baUTXGLOoWe4PQhGxaX0KpnayAqC48p4= github.com/erikgeiser/coninput v0.0.0-20211004153227-1c3628e74d0f/go.mod h1:vw97MGsxSvLiUE2X8qFplwetxpGLQrlU1Q9AUEIzCaM= github.com/fatih/color v1.18.0 h1:S8gINlzdQ840/4pfAwic/ZE0djQEH3wM94VfqLTZcOM= @@ -113,8 +115,8 @@ github.com/mattn/go-runewidth v0.0.7/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= github.com/mattn/go-runewidth v0.0.16 h1:E5ScNMtiwvlvB5paMFdw9p4kSQzbXFikJ5SQO6TULQc= github.com/mattn/go-runewidth v0.0.16/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= -github.com/mattn/go-sqlite3 v1.14.16 h1:yOQRA0RpS5PFz/oikGwBEqvAWhWg5ufRz4ETLjwpU1Y= -github.com/mattn/go-sqlite3 v1.14.16/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S2DGjv9HUNg= +github.com/mattn/go-sqlite3 v1.14.42 h1:MigqEP4ZmHw3aIdIT7T+9TLa90Z6smwcthx+Azv4Cgo= +github.com/mattn/go-sqlite3 v1.14.42/go.mod h1:pjEuOr8IwzLJP2MfGeTb0A35jauH+C2kbHKBr7yXKVQ= github.com/mitchellh/copystructure v1.2.0 h1:vpKXTN4ewci03Vljg/q9QvCGUDttBOGBIa15WveJJGw= github.com/mitchellh/copystructure v1.2.0/go.mod h1:qLl+cE2AmVv+CoeAwDPye/v+N2HKCj9FbZEVFJRxO9s= github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= @@ -166,6 +168,8 @@ github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8 github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU= github.com/tursodatabase/libsql-client-go v0.0.0-20260514053736-a9a8fadfe885 h1:YssVXwM/9nUAjGNmUWdgvb05JVcsaBrDn5yr+MaJTn0= github.com/tursodatabase/libsql-client-go v0.0.0-20260514053736-a9a8fadfe885/go.mod h1:08inkKyguB6CGGssc/JzhmQWwBgFQBgjlYFjxjRh7nU= +github.com/tursodatabase/turso-go-platform-libs v0.7.1 h1:sQ2aupg/dhilMWRRhIIjd9w8ikue4vIYGxHFMtWhYZE= +github.com/tursodatabase/turso-go-platform-libs v0.7.1/go.mod h1:bo+Lpv5OYOX1gRV9L5DLKMsYxmDs56SkZwnCOLEFcxU= github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e h1:JVG44RsyaB9T2KIHavMF/ppJZNG9ZpyihvCd0w101no= github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e/go.mod h1:RbqR21r5mrJuqunuUZ/Dhy/avygyECGrLceyNeo4LiM= go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc= @@ -198,3 +202,5 @@ gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EV gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +turso.tech/database/tursogo v0.7.1 h1:xCVENss9TeC5dR2Xs0saZZpVg05+1hW9pL1ei8KSAQ8= +turso.tech/database/tursogo v0.7.1/go.mod h1:sDMlDKBFfVdU1zi3qOd+HqU0Nnu/84UYGR6sZGqBIAM= diff --git a/internal/cmd/db_create.go b/internal/cmd/db_create.go index 2cf26621..6e971b87 100644 --- a/internal/cmd/db_create.go +++ b/internal/cmd/db_create.go @@ -155,7 +155,7 @@ func createDatabase(client *turso.Client, name, location, groupName string, seed if sizeLimitFlag != "" { return createDatabaseV2(client, name, location, groupName, seed, spinner) } - if seed != nil && seed.Type != "database" && seed.Type != "upload" { + if seed != nil && seed.Type != "database" { return createDatabaseV2(client, name, location, groupName, seed, spinner) } orgID, err := tryResolveOrgID(client) diff --git a/internal/cmd/db_import.go b/internal/cmd/db_import.go index a3d8177a..21cb11cb 100644 --- a/internal/cmd/db_import.go +++ b/internal/cmd/db_import.go @@ -9,11 +9,14 @@ import ( "github.com/spf13/cobra" ) +var importTursoDBFlag bool + func init() { dbCmd.AddCommand(importCmd) addGroupFlag(importCmd) addRemoteEncryptionKeyFlag(importCmd) addRemoteEncryptionCipherFlag(importCmd) + importCmd.Flags().BoolVar(&importTursoDBFlag, "tursodb", false, "Import into a TursoDB (MVCC) database.") } var importCmd = &cobra.Command{ @@ -43,6 +46,7 @@ var importCmd = &cobra.Command{ } fromFileFlag = filename + tursoDBFlag = importTursoDBFlag name := sanitizeDatabaseName(filename) return CreateDatabase(name) }, diff --git a/internal/cmd/db_import_file.go b/internal/cmd/db_import_file.go new file mode 100644 index 00000000..bee68be5 --- /dev/null +++ b/internal/cmd/db_import_file.go @@ -0,0 +1,471 @@ +package cmd + +import ( + "bufio" + "database/sql" + "errors" + "fmt" + "log" + "os" + "os/exec" + "path/filepath" + "strconv" + "strings" + + "github.com/tursodatabase/turso-cli/internal/flags" + "github.com/tursodatabase/turso-cli/internal/prompt" + "github.com/tursodatabase/turso-cli/internal/turso" + _ "turso.tech/database/tursogo" +) + +const MaxAWSDBSizeBytes = 1024 * 1024 * 1024 * 20 // 20 GB + +const databaseSettingsQuery = "select journal_mode as j, page_size as p, auto_vacuum as a, encoding as e from pragma_journal_mode, pragma_page_size, pragma_auto_vacuum, pragma_encoding;" + +type databaseEngine uint8 + +const ( + databaseEngineUnknown databaseEngine = iota + databaseEngineSQLite + databaseEngineTursoDB +) + +func (e databaseEngine) name() string { + switch e { + case databaseEngineSQLite: + return "SQLite" + case databaseEngineTursoDB: + return "TursoDB" + default: + return "unknown" + } +} + +func (e databaseEngine) settingsCommand() (string, bool) { + if e == databaseEngineSQLite { + return "sqlite3", true + } + return "", false +} + +type databaseFileChecker struct { + engine databaseEngine + journalMode string + settings func(string) (databaseSettings, error) + quickCheck func(string) error +} + +type databaseSettings struct { + journalMode string + pageSize string + autoVacuum string + encoding string +} + +func humanReadableSize(bytes int64) string { + const unit = 1024 + if bytes < unit { + return fmt.Sprintf("%d B", bytes) + } + div, exp := int64(unit), 0 + for n := bytes / unit; n >= unit; n /= unit { + div *= unit + exp++ + } + return fmt.Sprintf("%.1f %cB", float64(bytes)/float64(div), "KMGTPE"[exp]) +} + +func checkIfDump(filename string) (bool, error) { + file, err := os.Open(filename) + if err != nil { + return false, err + } + defer file.Close() + scanner := bufio.NewScanner(file) + if scanner.Scan() { + return strings.TrimSpace(scanner.Text()) == "PRAGMA foreign_keys=OFF;", nil + } + return false, scanner.Err() +} + +func validateDatabaseFileSize(file string) error { + if flags.Debug() { + log.Printf("Checking file size...") + } + fileInfo, err := os.Stat(file) + if err != nil { + return fmt.Errorf("failed to get file info: %w", err) + } + if fileInfo.Size() > MaxAWSDBSizeBytes { + return errors.New("database file size exceeds maximum allowed size of 20 GB") + } + return nil +} + +func sqliteFileIntegrityChecks(file string, cipher string) error { + return databaseFileIntegrityChecks(file, cipher, databaseFileChecker{ + engine: databaseEngineSQLite, + journalMode: "wal", + settings: sqliteDatabaseSettings, + quickCheck: runQuickCheck, + }) +} + +func tursoDBFileIntegrityChecks(file string) error { + return databaseFileIntegrityChecks(file, "", databaseFileChecker{ + engine: databaseEngineTursoDB, + journalMode: "mvcc", + settings: tursoDBDatabaseSettings, + quickCheck: runTursoDBQuickCheck, + }) +} + +func databaseFileIntegrityChecks(file, cipher string, checker databaseFileChecker) error { + if flags.Debug() { + log.Printf("Running %s integrity checks on database file %s", checker.engine.name(), file) + log.Printf("Checking database settings...") + } + + settings, err := checker.settings(file) + if err != nil { + return err + } + if err := validateDatabaseSettings(file, settings, checker); err != nil { + return err + } + + fileInfo, err := os.Stat(file) + if err != nil { + return fmt.Errorf("failed to get file info: %w", err) + } + if flags.Debug() { + log.Printf("Running integrity check...") + } + spinner := prompt.Spinner(fmt.Sprintf("Validating database file (%s)...", humanReadableSize(fileInfo.Size()))) + err = checker.quickCheck(file) + spinner.Stop() + if err != nil { + return err + } + + if cipher != "" { + if flags.Debug() { + log.Printf("Checking reserved bytes for cipher %s...", cipher) + } + return validateReservedBytes(file, cipher) + } + + return nil +} + +func validateDatabaseSettings(file string, settings databaseSettings, checker databaseFileChecker) error { + settingsCommand, hasSettingsCommand := checker.engine.settingsCommand() + if !strings.EqualFold(settings.journalMode, checker.journalMode) { + if !hasSettingsCommand { + return fmt.Errorf("database is not in %s mode", strings.ToUpper(checker.journalMode)) + } + return fmt.Errorf("database is not in %s mode. Set it with '%s %s \"PRAGMA journal_mode = %s;\"'", strings.ToUpper(checker.journalMode), settingsCommand, file, strings.ToUpper(checker.journalMode)) + } + if settings.pageSize != "4096" { + if !hasSettingsCommand { + return errors.New("database must use 4KB page size") + } + return fmt.Errorf("database must use 4KB page size. You can set it with '%s %s \"PRAGMA page_size = 4096; VACUUM;\"'", settingsCommand, file) + } + if settings.autoVacuum != "0" { + if !hasSettingsCommand { + return errors.New("database must have autovacuum disabled") + } + return fmt.Errorf("database must have autovacuum disabled. You can set it with '%s %s \"PRAGMA auto_vacuum = 0;\"'", settingsCommand, file) + } + if !strings.EqualFold(settings.encoding, "UTF-8") { + if !hasSettingsCommand { + return errors.New("database must use UTF-8 encoding") + } + return fmt.Errorf("database must use UTF-8 encoding. You can set it with '%s %s \"PRAGMA encoding = 'UTF-8';\"'", settingsCommand, file) + } + return nil +} + +func parseDatabaseSettings(output string) (databaseSettings, error) { + values := strings.Split(strings.TrimSpace(output), "|") + if len(values) != 4 { + return databaseSettings{}, fmt.Errorf("unexpected database settings output: %s", strings.TrimSpace(output)) + } + return databaseSettings{ + journalMode: strings.TrimSpace(values[0]), + pageSize: strings.TrimSpace(values[1]), + autoVacuum: strings.TrimSpace(values[2]), + encoding: strings.TrimSpace(values[3]), + }, nil +} + +func sqliteDatabaseSettings(file string) (databaseSettings, error) { + output, err := exec.Command("sqlite3", "-list", file, databaseSettingsQuery).CombinedOutput() + if err != nil { + return databaseSettings{}, fmt.Errorf("failed to check database settings with sqlite3: %w: %s", err, strings.TrimSpace(string(output))) + } + settings, err := parseDatabaseSettings(string(output)) + if err != nil { + return databaseSettings{}, fmt.Errorf("failed to parse database settings from sqlite3: %w", err) + } + return settings, nil +} + +func tursoDBDatabaseSettings(file string) (databaseSettings, error) { + db, err := openTursoDB(file) + if err != nil { + return databaseSettings{}, fmt.Errorf("failed to check database settings with TursoDB: %w", err) + } + defer db.Close() + + var settings databaseSettings + err = db.QueryRow(databaseSettingsQuery).Scan( + &settings.journalMode, + &settings.pageSize, + &settings.autoVacuum, + &settings.encoding, + ) + if err != nil { + return databaseSettings{}, fmt.Errorf("failed to query database settings with TursoDB: %w", err) + } + return settings, nil +} + +func openTursoDB(file string) (*sql.DB, error) { + db, err := sql.Open("turso", file) + if err != nil { + return nil, err + } + db.SetMaxOpenConns(1) + if err := db.Ping(); err != nil { + db.Close() + return nil, err + } + return db, nil +} + +func runQuickCheck(file string) error { + cmd := exec.Command("sqlite3", "-list", file, "pragma quick_check;") + if err := cmd.Run(); err != nil { + return fmt.Errorf("integrity check failed: %w", err) + } + return nil +} + +func runTursoDBQuickCheck(file string) error { + db, err := openTursoDB(file) + if err != nil { + return fmt.Errorf("TursoDB integrity check failed for %s: %w", file, err) + } + defer db.Close() + + rows, err := db.Query("PRAGMA quick_check;") + if err != nil { + return fmt.Errorf("TursoDB integrity check failed for %s: %w", file, err) + } + defer rows.Close() + + checked := false + for rows.Next() { + var result string + if err := rows.Scan(&result); err != nil { + return fmt.Errorf("TursoDB integrity check failed for %s: %w", file, err) + } + checked = true + if result != "ok" { + return fmt.Errorf("TursoDB integrity check failed for %s: %s", file, result) + } + } + if err := rows.Err(); err != nil { + return fmt.Errorf("TursoDB integrity check failed for %s: %w", file, err) + } + if !checked { + return fmt.Errorf("TursoDB integrity check failed for %s: no result", file) + } + return nil +} + +func checkpointWALBeforeUpload(file string) error { + if out, err := exec.Command("sqlite3", "-list", file, "PRAGMA wal_checkpoint(TRUNCATE);").CombinedOutput(); err != nil { + return fmt.Errorf("could not checkpoint database %s: %w: %s", file, err, out) + } + for _, sidecar := range []struct{ suffix, hint string }{ + {"-wal", "close all connections to the database and retry the import"}, + {"-journal", "the database has a leftover rollback journal; open and cleanly close it with sqlite3 first"}, + } { + if err := checkSidecarEmpty(file+sidecar.suffix, sidecar.hint); err != nil { + return err + } + } + return nil +} + +func prepareTursoDBFile(file string) error { + db, err := openTursoDB(file) + if err != nil { + return fmt.Errorf("could not prepare %s for TursoDB import: %w", file, err) + } + _, execErr := db.Exec("PRAGMA journal_mode = mvcc; PRAGMA wal_checkpoint(TRUNCATE);") + closeErr := db.Close() + if execErr != nil { + return fmt.Errorf("could not prepare %s for TursoDB import: %w", file, execErr) + } + if closeErr != nil { + return fmt.Errorf("could not close %s after preparing it for TursoDB import: %w", file, closeErr) + } + + format, err := sniffSQLiteFileFormat(file) + if err != nil { + return err + } + if format != fileFormatMVCC { + return fmt.Errorf("TursoDB did not convert %s to MVCC format", file) + } + return nil +} + +func checkTursoDBSidecars(file string) error { + for _, sidecar := range []struct{ path, hint string }{ + {file + "-wal", "close all connections to the database and retry the import"}, + {file + "-journal", "the database has a leftover rollback journal; close it cleanly and retry the import"}, + {tursodbLogPath(file), "close all TursoDB connections and retry the import"}, + } { + if err := checkSidecarEmpty(sidecar.path, sidecar.hint); err != nil { + return err + } + } + return nil +} + +func checkSidecarEmpty(sidecarPath, hint string) error { + info, err := os.Stat(sidecarPath) + if errors.Is(err, os.ErrNotExist) { + return nil + } + if err != nil { + return fmt.Errorf("could not check %s: %w", sidecarPath, err) + } + if info.Size() > 0 { + return fmt.Errorf("%s is not empty, importing would lose the data it holds: %s", sidecarPath, hint) + } + return nil +} + +func tursodbLogPath(file string) string { + return strings.TrimSuffix(file, filepath.Ext(file)) + ".db-log" +} + +func handleDBFileAWS(file string, cipher string) (*turso.DBSeed, error) { + format, err := sniffSQLiteFileFormat(file) + if err != nil { + return nil, err + } + + if format == fileFormatRollback { + if err := checkSQLiteAvailable(); err != nil { + return nil, err + } + fmt.Printf("File %s uses a rollback journal; converting it to WAL mode for import.\n", file) + if out, err := exec.Command("sqlite3", file, "PRAGMA journal_mode=WAL;").CombinedOutput(); err != nil { + return nil, fmt.Errorf("could not convert %s to WAL mode: %w: %s", file, err, out) + } + if format, err = sniffSQLiteFileFormat(file); err != nil { + return nil, err + } + } + + switch format { + case fileFormatWAL: + if err := checkSQLiteAvailable(); err != nil { + return nil, err + } + if err := checkpointWALBeforeUpload(file); err != nil { + return nil, err + } + if err := sqliteFileIntegrityChecks(file, cipher); err != nil { + return nil, err + } + case fileFormatMVCC: + if !tursoDBFlag { + return nil, fmt.Errorf("%s is in tursodb (MVCC) format and can only be imported into a tursodb database", file) + } + if cipher != "" { + return nil, errors.New("remote encryption is not supported when importing tursodb (MVCC) format files") + } + case fileFormatNotSQLite: + isDump, err := checkIfDump(file) + if err != nil { + return nil, fmt.Errorf("failed to get file header: %w", err) + } + if isDump { + return nil, fmt.Errorf("%s is a sqlite3 dump, not a sqlite3 database. Please import a sqlite database", file) + } + return nil, fmt.Errorf("file %s is not a valid SQLite database file", file) + case fileFormatUnknown: + return nil, fmt.Errorf("file %s has unsupported SQLite read/write format versions", file) + default: + return nil, fmt.Errorf("file %s has an unsupported SQLite file format", file) + } + + if tursoDBFlag { + if format != fileFormatMVCC { + fmt.Printf("Converting %s to TursoDB (MVCC) format for import.\n", file) + } + if err := prepareTursoDBFile(file); err != nil { + return nil, err + } + if err := tursoDBFileIntegrityChecks(file); err != nil { + return nil, err + } + if err := checkTursoDBSidecars(file); err != nil { + return nil, err + } + } + + if err := validateDatabaseFileSize(file); err != nil { + return nil, err + } + + return &turso.DBSeed{ + Type: "database_upload", + Filepath: file, + }, nil +} + +func getReservedBytes(dbPath string) (int, error) { + output, err := exec.Command("sqlite3", "-list", dbPath, ".filectrl reserve_bytes").CombinedOutput() + if err != nil { + return 0, fmt.Errorf("failed to get reserved bytes: %w", err) + } + outputStr := strings.TrimSpace(string(output)) + if strings.Contains(outputStr, ":") { + parts := strings.Split(outputStr, ":") + if len(parts) >= 2 { + outputStr = strings.TrimSpace(parts[1]) + } + } + + reservedBytes, err := strconv.Atoi(outputStr) + if err != nil { + return 0, fmt.Errorf("failed to parse reserved bytes from output '%s': %w", string(output), err) + } + return reservedBytes, nil +} + +func validateReservedBytes(dbPath string, cipher string) error { + requiredBytes, ok := getRequiredReservedBytes(cipher) + if !ok { + return nil + } + + currentBytes, err := getReservedBytes(dbPath) + if err != nil { + return err + } + if currentBytes != requiredBytes { + return fmt.Errorf("database reserved bytes mismatch: found %d, but cipher '%s' requires %d reserved bytes.\nTo fix this, run:\n\n $ sqlite3 %s\n sqlite> .filectrl reserve_bytes %d\n sqlite> VACUUM;", + currentBytes, cipher, requiredBytes, dbPath, requiredBytes) + } + return nil +} diff --git a/internal/cmd/db_import_file_test.go b/internal/cmd/db_import_file_test.go new file mode 100644 index 00000000..26f56e38 --- /dev/null +++ b/internal/cmd/db_import_file_test.go @@ -0,0 +1,165 @@ +package cmd + +import ( + "fmt" + "os" + "os/exec" + "path/filepath" + "testing" + + "github.com/stretchr/testify/require" +) + +// createTestDatabase creates a valid WAL-mode test database with approximately the given size +func createTestDatabase(t *testing.T, sizeBytes int) string { + t.Helper() + + if _, err := exec.LookPath("sqlite3"); err != nil { + t.Skip("sqlite3 not available, skipping test") + } + + dbPath := filepath.Join(t.TempDir(), "test.db") + + // Create database with correct settings for Turso + cmd := exec.Command("sqlite3", "-list", dbPath, + "PRAGMA page_size=4096;", + "PRAGMA journal_mode=WAL;", + "CREATE TABLE data (id INTEGER PRIMARY KEY, blob BLOB);") + require.NoError(t, cmd.Run(), "failed to create test database") + + // Fill with data to reach target size + if sizeBytes > 0 { + rowSize := 1000 // ~1KB per row + numRows := sizeBytes / rowSize + if numRows < 1 { + numRows = 1 + } + for i := 0; i < numRows; i++ { + cmd = exec.Command("sqlite3", "-list", dbPath, + fmt.Sprintf("INSERT INTO data (blob) VALUES (randomblob(%d));", rowSize)) + require.NoError(t, cmd.Run()) + } + } + + return dbPath +} + +func TestRunQuickCheck(t *testing.T) { + if _, err := exec.LookPath("sqlite3"); err != nil { + t.Skip("sqlite3 not available, skipping test") + } + + t.Run("valid database succeeds", func(t *testing.T) { + dbPath := createTestDatabase(t, 10*1024) // 10KB + require.NoError(t, runQuickCheck(dbPath)) + }) + + t.Run("corrupted database returns error", func(t *testing.T) { + dbPath := filepath.Join(t.TempDir(), "corrupt.db") + + // Create a file with garbage data + require.NoError(t, os.WriteFile(dbPath, []byte("not a valid sqlite database content here"), 0644)) + + err := runQuickCheck(dbPath) + require.Error(t, err) + require.Contains(t, err.Error(), "integrity check failed") + }) + + t.Run("nonexistent file returns error", func(t *testing.T) { + require.Error(t, runQuickCheck("/nonexistent/path/db.sqlite")) + }) +} + +func TestValidateDatabaseSettings(t *testing.T) { + checker := databaseFileChecker{ + engine: databaseEngineTursoDB, + journalMode: "mvcc", + } + settings := databaseSettings{ + journalMode: "MVCC", + pageSize: "4096", + autoVacuum: "0", + encoding: "UTF-8", + } + + require.NoError(t, validateDatabaseSettings("data.db", settings, checker)) + settings.journalMode = "WAL" + err := validateDatabaseSettings("data.db", settings, checker) + require.ErrorContains(t, err, "not in MVCC mode") + require.NotContains(t, err.Error(), "tursodb") + + checker.engine = databaseEngineSQLite + checker.journalMode = "wal" + settings.journalMode = "MVCC" + require.ErrorContains(t, validateDatabaseSettings("data.db", settings, checker), "sqlite3") +} + +func TestHandleDBFileAWSRejectsUnknownFormat(t *testing.T) { + dbPath := writeHeaderFile(t, sqliteMagic, 2, 255) + _, err := handleDBFileAWS(dbPath, "") + require.ErrorContains(t, err, "unsupported SQLite read/write format versions") +} + +func TestTursodbLogPath(t *testing.T) { + require.Equal(t, "data.db-log", tursodbLogPath("data.db")) + require.Equal(t, "data.db-log", tursodbLogPath("data.sqlite")) + require.Equal(t, "/some/dir/mydb.db-log", tursodbLogPath("/some/dir/mydb.db")) + require.Equal(t, "noext.db-log", tursodbLogPath("noext")) +} + +func TestCheckSidecarEmpty(t *testing.T) { + dir := t.TempDir() + + t.Run("missing sidecar is fine", func(t *testing.T) { + require.NoError(t, checkSidecarEmpty(filepath.Join(dir, "missing-wal"), "hint")) + }) + + t.Run("empty sidecar is fine", func(t *testing.T) { + path := filepath.Join(dir, "empty-wal") + require.NoError(t, os.WriteFile(path, nil, 0644)) + require.NoError(t, checkSidecarEmpty(path, "hint")) + }) + + t.Run("non-empty sidecar errors with hint", func(t *testing.T) { + path := filepath.Join(dir, "full-wal") + require.NoError(t, os.WriteFile(path, []byte("frames"), 0644)) + err := checkSidecarEmpty(path, "close all connections") + require.Error(t, err) + require.Contains(t, err.Error(), "importing would lose the data it holds") + require.Contains(t, err.Error(), "close all connections") + }) +} + +func TestCheckpointWALBeforeUpload(t *testing.T) { + if _, err := exec.LookPath("sqlite3"); err != nil { + t.Skip("sqlite3 not available, skipping test") + } + + dbPath := createTestDatabase(t, 10*1024) + require.NoError(t, checkpointWALBeforeUpload(dbPath)) + + // after the checkpoint no data-bearing sidecars may remain + for _, suffix := range []string{"-wal", "-journal"} { + if info, err := os.Stat(dbPath + suffix); err == nil { + require.Zero(t, info.Size(), "%s must be empty after checkpoint", dbPath+suffix) + } + } +} + +func TestPrepareTursoDBFile(t *testing.T) { + dbPath := createTestDatabase(t, 10*1024) + require.NoError(t, checkpointWALBeforeUpload(dbPath)) + t.Setenv("PATH", t.TempDir()) // TursoDB preparation must not require an external binary. + require.NoError(t, prepareTursoDBFile(dbPath)) + require.NoError(t, tursoDBFileIntegrityChecks(dbPath)) + require.NoError(t, checkTursoDBSidecars(dbPath)) + + format, err := sniffSQLiteFileFormat(dbPath) + require.NoError(t, err) + require.Equal(t, fileFormatMVCC, format) + for _, sidecar := range []string{dbPath + "-wal", dbPath + "-journal", tursodbLogPath(dbPath)} { + if info, err := os.Stat(sidecar); err == nil { + require.Zero(t, info.Size(), "%s must be empty after checkpoint", sidecar) + } + } +} diff --git a/internal/cmd/group_flag.go b/internal/cmd/group_flag.go index fea33092..c180ae69 100644 --- a/internal/cmd/group_flag.go +++ b/internal/cmd/group_flag.go @@ -5,11 +5,8 @@ import ( "bytes" "errors" "fmt" - "log" "os" "os/exec" - "strconv" - "strings" "time" "github.com/Clever/csvlint" @@ -184,187 +181,28 @@ func countFlags(flags ...string) (count int) { return } -const MaxAWSDBSizeBytes = 1024 * 1024 * 1024 * 20 // 20 GB - -func humanReadableSize(bytes int64) string { - const unit = 1024 - if bytes < unit { - return fmt.Sprintf("%d B", bytes) - } - div, exp := int64(unit), 0 - for n := bytes / unit; n >= unit; n /= unit { - div *= unit - exp++ - } - return fmt.Sprintf("%.1f %cB", float64(bytes)/float64(div), "KMGTPE"[exp]) -} - -func checkIfDump(filename string) (bool, error) { - file, err := os.Open(filename) - if err != nil { - return false, err - } - defer file.Close() - scanner := bufio.NewScanner(file) - if scanner.Scan() { - firstLine := scanner.Text() - return strings.TrimSpace(firstLine) == "PRAGMA foreign_keys=OFF;", nil - } else { - return false, scanner.Err() - } -} - -// getReservedBytes retrieves the current reserved bytes setting from a SQLite database -func getReservedBytes(dbPath string) (int, error) { - output, err := exec.Command("sqlite3", "-list", dbPath, ".filectrl reserve_bytes").CombinedOutput() - if err != nil { - return 0, fmt.Errorf("failed to get reserved bytes: %w", err) - } - outputStr := strings.TrimSpace(string(output)) - - if strings.Contains(outputStr, ":") { - parts := strings.Split(outputStr, ":") - if len(parts) >= 2 { - outputStr = strings.TrimSpace(parts[1]) - } - } - - reservedBytes, err := strconv.Atoi(outputStr) - if err != nil { - return 0, fmt.Errorf("failed to parse reserved bytes from output '%s': %w", string(output), err) - } - - return reservedBytes, nil -} - -// validateReservedBytes checks if the database has the required reserved bytes for the given cipher -func validateReservedBytes(dbPath string, cipher string) error { - requiredBytes, ok := getRequiredReservedBytes(cipher) - if !ok { - return nil - } - - currentBytes, err := getReservedBytes(dbPath) - if err != nil { - return err - } - - if currentBytes != requiredBytes { - return fmt.Errorf("database reserved bytes mismatch: found %d, but cipher '%s' requires %d reserved bytes.\nTo fix this, run:\n\n $ sqlite3 %s\n sqlite> .filectrl reserve_bytes %d\n sqlite> VACUUM;", - currentBytes, cipher, requiredBytes, dbPath, requiredBytes) - } - - return nil -} - -func sqliteFileIntegrityChecks(file string, cipher string) error { - if flags.Debug() { - log.Printf("Running integrity checks on database file %s", file) - } - - if flags.Debug() { - log.Printf("Checking if this is a sqlite dump: common mistake!...") - } - - isDump, err := checkIfDump(file) - if err != nil { - return fmt.Errorf("failed to get file header: %w", err) - } - if isDump { - return fmt.Errorf("%s is a sqlite3 dump, not a sqlite3 database. Please import a sqlite database", file) - } - - if flags.Debug() { - log.Printf("Checking file size...") - } - fileInfo, err := os.Stat(file) - if err != nil { - return fmt.Errorf("failed to get file info: %w", err) - } - - if fileInfo.Size() > MaxAWSDBSizeBytes { - return errors.New("database file size exceeds maximum allowed size of 20 GB") - } - - if flags.Debug() { - log.Printf("Checking database settings...") - } - output, err := exec.Command("sqlite3", "-list", file, ".mode line", - "select journal_mode as j, page_size as p, auto_vacuum as a, encoding as e from pragma_journal_mode, pragma_page_size, pragma_auto_vacuum, pragma_encoding;").CombinedOutput() - if err != nil { - - return fmt.Errorf("failed to check database settings: %w", err) +func handleDBFile(client *turso.Client, file string, isAWS bool, cipher string) (*turso.DBSeed, error) { + if err := checkFileExists(file); err != nil { + return nil, err } - settings := string(output) - if !strings.Contains(settings, "j: wal") && !strings.Contains(settings, "j = wal") { - return fmt.Errorf("database is not in WAL mode. Set it with 'sqlite3 %s 'PRAGMA journal_mode = WAL'", file) - } - if !strings.Contains(settings, "p: 4096") && !strings.Contains(settings, "p = 4096") { - return fmt.Errorf("database must use 4KB page size. you can set it with 'sqlite3 %s 'PRAGMA page_size = 4096; VACUUM;' Note that this is not possible to do if your database is already in WAL mode", file) - } - if !strings.Contains(settings, "a: 0") && !strings.Contains(settings, "a = 0") { - return fmt.Errorf("database must have autovacuum disabled. you can set it with 'sqlite3 %s 'PRAGMA auto_vacuum = 0;'", file) - } - if !strings.Contains(settings, "e: UTF-8") && !strings.Contains(settings, "e = UTF-8") { - return fmt.Errorf("database must use UTF-8 encoding. you can set it with 'sqlite3 %s 'PRAGMA encoding = 'UTF-8' ", file) + if isAWS { + return handleDBFileAWS(file, cipher) } - // run quick_check - if flags.Debug() { - log.Printf("Running integrity check...") - } - spinner := prompt.Spinner(fmt.Sprintf("Validating database file (%s)...", humanReadableSize(fileInfo.Size()))) - err = runQuickCheck(file) - spinner.Stop() + format, err := sniffSQLiteFileFormat(file) if err != nil { - return err - } - - // validate reserved bytes if encryption cipher is specified - if cipher != "" { - if flags.Debug() { - log.Printf("Checking reserved bytes for cipher %s...", cipher) - } - return validateReservedBytes(file, cipher) - } - - return nil -} - -func runQuickCheck(file string) error { - cmd := exec.Command("sqlite3", "-list", file, "pragma quick_check;") - if err := cmd.Run(); err != nil { - return fmt.Errorf("integrity check failed: %w", err) - } - return nil -} - -func handleDBFileAWS(file string, cipher string) (*turso.DBSeed, error) { - if err := sqliteFileIntegrityChecks(file, cipher); err != nil { return nil, err } - - seed := &turso.DBSeed{ - Type: "database_upload", - Filepath: file, - } - - return seed, nil -} - -func handleDBFile(client *turso.Client, file string, isAWS bool, cipher string) (*turso.DBSeed, error) { - if err := checkFileExists(file); err != nil { - return nil, err + if format == fileFormatMVCC { + // non-AWS groups are seeded by replaying a .dump, which sqlite3 cannot + // produce from an MVCC (tursodb format) file + return nil, fmt.Errorf("%s is in tursodb (MVCC) format and can only be imported into AWS groups", file) } if err := checkSQLiteAvailable(); err != nil { return nil, err } - if isAWS { - return handleDBFileAWS(file, cipher) - } - if err := checkSQLiteFile(file); err != nil { return nil, err } diff --git a/internal/cmd/group_flag_test.go b/internal/cmd/group_flag_test.go deleted file mode 100644 index 6fcf4b7d..00000000 --- a/internal/cmd/group_flag_test.go +++ /dev/null @@ -1,76 +0,0 @@ -package cmd - -import ( - "fmt" - "os" - "os/exec" - "path/filepath" - "testing" - - "github.com/stretchr/testify/require" -) - -// createTestDatabase creates a valid WAL-mode test database with approximately the given size -func createTestDatabase(t *testing.T, sizeBytes int) string { - t.Helper() - - if _, err := exec.LookPath("sqlite3"); err != nil { - t.Skip("sqlite3 not available, skipping test") - } - - tmpDir := t.TempDir() - dbPath := filepath.Join(tmpDir, "test.db") - - // Create database with correct settings for Turso - cmd := exec.Command("sqlite3", "-list", dbPath, - "PRAGMA page_size=4096;", - "PRAGMA journal_mode=WAL;", - "CREATE TABLE data (id INTEGER PRIMARY KEY, blob BLOB);") - require.NoError(t, cmd.Run(), "failed to create test database") - - // Fill with data to reach target size - if sizeBytes > 0 { - rowSize := 1000 // ~1KB per row - numRows := sizeBytes / rowSize - if numRows < 1 { - numRows = 1 - } - for i := 0; i < numRows; i++ { - cmd = exec.Command("sqlite3", "-list", dbPath, - fmt.Sprintf("INSERT INTO data (blob) VALUES (randomblob(%d));", rowSize)) - require.NoError(t, cmd.Run()) - } - } - - return dbPath -} - -func TestRunQuickCheck(t *testing.T) { - if _, err := exec.LookPath("sqlite3"); err != nil { - t.Skip("sqlite3 not available, skipping test") - } - - t.Run("valid database succeeds", func(t *testing.T) { - dbPath := createTestDatabase(t, 10*1024) // 10KB - err := runQuickCheck(dbPath) - require.NoError(t, err) - }) - - t.Run("corrupted database returns error", func(t *testing.T) { - tmpDir := t.TempDir() - dbPath := filepath.Join(tmpDir, "corrupt.db") - - // Create a file with garbage data - err := os.WriteFile(dbPath, []byte("not a valid sqlite database content here"), 0644) - require.NoError(t, err) - - err = runQuickCheck(dbPath) - require.Error(t, err) - require.Contains(t, err.Error(), "integrity check failed") - }) - - t.Run("nonexistent file returns error", func(t *testing.T) { - err := runQuickCheck("/nonexistent/path/db.sqlite") - require.Error(t, err) - }) -} diff --git a/internal/cmd/sqlite_header.go b/internal/cmd/sqlite_header.go new file mode 100644 index 00000000..8d3609a4 --- /dev/null +++ b/internal/cmd/sqlite_header.go @@ -0,0 +1,52 @@ +package cmd + +import ( + "io" + "os" +) + +type sqliteFileFormat int + +const ( + fileFormatNotSQLite sqliteFileFormat = iota + fileFormatRollback + fileFormatWAL + fileFormatMVCC + fileFormatUnknown +) + +const sqliteMagic = "SQLite format 3\x00" + +// sniffSQLiteFileFormat classifies a database file by its SQLite header: the +// 16-byte magic string plus the read/write format version bytes at offsets +// 18/19 (1 = rollback journal, 2 = WAL, 255 = MVCC, i.e. tursodb format). +// +// This must run before any sqlite3 shellout: the sqlite3 binary reports +// MVCC-format files as not-a-database, so they have to be routed around the +// sqlite3-based checks entirely. +func sniffSQLiteFileFormat(path string) (sqliteFileFormat, error) { + file, err := os.Open(path) + if err != nil { + return fileFormatNotSQLite, err + } + defer file.Close() + + header := make([]byte, 20) + if _, err := io.ReadFull(file, header); err != nil { + // too short to hold a SQLite header + return fileFormatNotSQLite, nil + } + if string(header[:len(sqliteMagic)]) != sqliteMagic { + return fileFormatNotSQLite, nil + } + readVersion, writeVersion := header[18], header[19] + switch { + case readVersion == 1 && writeVersion == 1: + return fileFormatRollback, nil + case readVersion == 2 && writeVersion == 2: + return fileFormatWAL, nil + case readVersion == 255 && writeVersion == 255: + return fileFormatMVCC, nil + } + return fileFormatUnknown, nil +} diff --git a/internal/cmd/sqlite_header_test.go b/internal/cmd/sqlite_header_test.go new file mode 100644 index 00000000..c538ce15 --- /dev/null +++ b/internal/cmd/sqlite_header_test.go @@ -0,0 +1,90 @@ +package cmd + +import ( + "os" + "os/exec" + "path/filepath" + "testing" + + "github.com/stretchr/testify/require" +) + +func writeHeaderFile(t *testing.T, magic string, readVersion, writeVersion byte) string { + t.Helper() + header := make([]byte, 100) + copy(header, magic) + header[18] = readVersion + header[19] = writeVersion + path := filepath.Join(t.TempDir(), "test.db") + require.NoError(t, os.WriteFile(path, header, 0644)) + return path +} + +func TestSniffSQLiteFileFormat(t *testing.T) { + for _, tc := range []struct { + name string + magic string + readVersion, writeVersion byte + expected sqliteFileFormat + }{ + {"rollback journal", sqliteMagic, 1, 1, fileFormatRollback}, + {"wal", sqliteMagic, 2, 2, fileFormatWAL}, + {"mvcc", sqliteMagic, 255, 255, fileFormatMVCC}, + {"mixed version bytes", sqliteMagic, 2, 255, fileFormatUnknown}, + {"unknown version bytes", sqliteMagic, 42, 42, fileFormatUnknown}, + {"wrong magic", "Not a SQLite db\x00", 2, 2, fileFormatNotSQLite}, + } { + t.Run(tc.name, func(t *testing.T) { + path := writeHeaderFile(t, tc.magic, tc.readVersion, tc.writeVersion) + format, err := sniffSQLiteFileFormat(path) + require.NoError(t, err) + require.Equal(t, tc.expected, format) + }) + } + + t.Run("file shorter than header", func(t *testing.T) { + path := filepath.Join(t.TempDir(), "short.db") + require.NoError(t, os.WriteFile(path, []byte("SQLite"), 0644)) + format, err := sniffSQLiteFileFormat(path) + require.NoError(t, err) + require.Equal(t, fileFormatNotSQLite, format) + }) + + t.Run("missing file", func(t *testing.T) { + _, err := sniffSQLiteFileFormat(filepath.Join(t.TempDir(), "missing.db")) + require.Error(t, err) + }) +} + +func TestSniffSQLiteFileFormatOnRealDatabases(t *testing.T) { + if _, err := exec.LookPath("sqlite3"); err != nil { + t.Skip("sqlite3 not available, skipping test") + } + + t.Run("wal database", func(t *testing.T) { + dbPath := createTestDatabase(t, 10*1024) + format, err := sniffSQLiteFileFormat(dbPath) + require.NoError(t, err) + require.Equal(t, fileFormatWAL, format) + }) + + t.Run("rollback database converts to wal", func(t *testing.T) { + dbPath := filepath.Join(t.TempDir(), "rollback.db") + cmd := exec.Command("sqlite3", "-list", dbPath, + "PRAGMA page_size=4096;", + "CREATE TABLE data (id INTEGER PRIMARY KEY);") + require.NoError(t, cmd.Run(), "failed to create test database") + + format, err := sniffSQLiteFileFormat(dbPath) + require.NoError(t, err) + require.Equal(t, fileFormatRollback, format) + + // the conversion handleDBFileAWS performs for rollback files + out, err := exec.Command("sqlite3", dbPath, "PRAGMA journal_mode=WAL;").CombinedOutput() + require.NoError(t, err, "convert to WAL: %s", out) + + format, err = sniffSQLiteFileFormat(dbPath) + require.NoError(t, err) + require.Equal(t, fileFormatWAL, format) + }) +} diff --git a/internal/turso/tursoServer.go b/internal/turso/tursoServer.go index 60dd129d..3125d125 100644 --- a/internal/turso/tursoServer.go +++ b/internal/turso/tursoServer.go @@ -158,7 +158,7 @@ type chunkUploadContext struct { chunkPath string chunkSize int64 chunkStartOffset int64 // File offset where this chunk starts - file *os.File + file io.ReadSeeker headers map[string]string totalSize int64 startTime time.Time @@ -355,7 +355,7 @@ func (i *TursoServerClient) startMultipartUpload(dbSize int64) (multipartUploadS return multipartUploadStart(uploadResp), nil } -func (i *TursoServerClient) uploadChunks(uploadID string, chunkSize int64, file *os.File, totalSize int64, startTime time.Time, remoteEncryptionCipher, remoteEncryptionKey string, onUploadProgress func(progressPct int, uploadedBytes int64, totalBytes int64, elapsedTime time.Duration, done bool)) (int64, error) { +func (i *TursoServerClient) uploadChunks(uploadID string, chunkSize int64, file io.ReadSeeker, totalSize int64, startTime time.Time, remoteEncryptionCipher, remoteEncryptionKey string, onUploadProgress func(progressPct int, uploadedBytes int64, totalBytes int64, elapsedTime time.Duration, done bool)) (int64, error) { var uploadedBytes int64 = 0 chunkID := 0 lastProgressPct := -1 diff --git a/internal/turso/tursoServer_test.go b/internal/turso/tursoServer_test.go index 8b1a3a07..a3f21c3b 100644 --- a/internal/turso/tursoServer_test.go +++ b/internal/turso/tursoServer_test.go @@ -1162,3 +1162,22 @@ func TestProgressReader_UpdatesTrackingFieldsCorrectly(t *testing.T) { require.Equal(t, int64(50), pr.lastUpdateBytes, "lastUpdateBytes should track uploaded bytes") }) } + +func TestUploadFileMultipart_DoesNotRewriteFormatBytes(t *testing.T) { + mock := NewMockTursoServer() + mock.chunkSize = 16 + defer mock.Close() + + content := make([]byte, 100) + content[18], content[19] = 2, 2 + + client := createTestClient(t, mock.URL) + testFile := createTestFileWithContent(t, content) + progress := NewProgressRecorder() + + err := client.UploadFileMultipart(testFile, "", "", progress.Callback()) + require.NoError(t, err) + + uploaded := mock.GetAllChunkData() + require.Equal(t, content, uploaded) +}