From 231f7a6669f292b3c1b4c6e78b6632db3fab0e62 Mon Sep 17 00:00:00 2001 From: pedrocarlo Date: Mon, 27 Jul 2026 13:47:00 -0300 Subject: [PATCH 1/6] support TursoDB database imports --- internal/cmd/db_create.go | 7 +- internal/cmd/db_import.go | 9 +++ internal/cmd/group_flag.go | 112 ++++++++++++++++++++++++++++- internal/cmd/group_flag_test.go | 46 ++++++++++++ internal/cmd/sqlite_header.go | 52 ++++++++++++++ internal/cmd/sqlite_header_test.go | 90 +++++++++++++++++++++++ internal/turso/databases.go | 12 +++- internal/turso/tursoServer.go | 49 ++++++++++++- internal/turso/tursoServer_test.go | 51 +++++++++++++ 9 files changed, 420 insertions(+), 8 deletions(-) create mode 100644 internal/cmd/sqlite_header.go create mode 100644 internal/cmd/sqlite_header_test.go diff --git a/internal/cmd/db_create.go b/internal/cmd/db_create.go index 2cf26621..8ddb7504 100644 --- a/internal/cmd/db_create.go +++ b/internal/cmd/db_create.go @@ -155,7 +155,12 @@ func createDatabase(client *turso.Client, name, location, groupName string, seed if sizeLimitFlag != "" { return createDatabaseV2(client, name, location, groupName, seed, spinner) } - if seed != nil && seed.Type != "database" && seed.Type != "upload" { + // Only fork seeds ("database") can go through the v3 API. File seeds + // ("database_upload") must use the v2 flow: the v3 branch never uploads + // the file after creating the database. (This used to compare against + // "upload", a seed type that doesn't exist, which routed uploads to v2 + // by accident.) + if seed != nil && seed.Type != "database" { return createDatabaseV2(client, name, location, groupName, seed, spinner) } orgID, err := tryResolveOrgID(client) diff --git a/internal/cmd/db_import.go b/internal/cmd/db_import.go index a3d8177a..84ec54a3 100644 --- a/internal/cmd/db_import.go +++ b/internal/cmd/db_import.go @@ -9,11 +9,19 @@ import ( "github.com/spf13/cobra" ) +// Import defaults to TursoDB (MVCC) databases — the only mode the cloud +// supports for new databases. The flag is separate from createCmd's +// --tursodb (which shares the tursoDBFlag global with a false default) so +// the two commands can have different defaults; RunE copies it over before +// delegating to CreateDatabase. +var importTursoDBFlag bool + func init() { dbCmd.AddCommand(importCmd) addGroupFlag(importCmd) addRemoteEncryptionKeyFlag(importCmd) addRemoteEncryptionCipherFlag(importCmd) + importCmd.Flags().BoolVar(&importTursoDBFlag, "tursodb", true, "Import into a TursoDB (MVCC) database.") } var importCmd = &cobra.Command{ @@ -43,6 +51,7 @@ var importCmd = &cobra.Command{ } fromFileFlag = filename + tursoDBFlag = importTursoDBFlag name := sanitizeDatabaseName(filename) return CreateDatabase(name) }, diff --git a/internal/cmd/group_flag.go b/internal/cmd/group_flag.go index fea33092..0e4e876b 100644 --- a/internal/cmd/group_flag.go +++ b/internal/cmd/group_flag.go @@ -8,6 +8,7 @@ import ( "log" "os" "os/exec" + "path/filepath" "strconv" "strings" "time" @@ -340,11 +341,110 @@ func runQuickCheck(file string) error { return nil } +// checkpointWALBeforeUpload folds any pending WAL frames into the main +// database file and verifies no data is left behind in sidecar files. The +// upload ships only the main database file, so frames still sitting in +// -wal (or a hot rollback journal) would silently be missing from the +// imported database. +func checkpointWALBeforeUpload(file string) error { + if out, err := exec.Command("sqlite3", "-list", file, "PRAGMA wal_checkpoint(TRUNCATE);").CombinedOutput(); err != nil { + return fmt.Errorf("could not checkpoint database %s: %w: %s", file, err, out) + } + for _, sidecar := range []struct{ suffix, hint string }{ + {"-wal", "close all connections to the database and retry the import"}, + {"-journal", "the database has a leftover rollback journal; open and cleanly close it with sqlite3 first"}, + } { + if err := checkSidecarEmpty(file+sidecar.suffix, sidecar.hint); err != nil { + return err + } + } + return nil +} + +func checkSidecarEmpty(sidecarPath, hint string) error { + info, err := os.Stat(sidecarPath) + if errors.Is(err, os.ErrNotExist) { + return nil + } + if err != nil { + return fmt.Errorf("could not check %s: %w", sidecarPath, err) + } + if info.Size() > 0 { + return fmt.Errorf("%s is not empty, importing would lose the data it holds: %s", sidecarPath, hint) + } + return nil +} + +// tursodbLogPath returns the logical-log sidecar path tursodb uses for a +// database file, mirroring turso_core's `with_extension("db-log")`: the +// file's extension (if any) is replaced with "db-log". +func tursodbLogPath(file string) string { + return strings.TrimSuffix(file, filepath.Ext(file)) + ".db-log" +} + func handleDBFileAWS(file string, cipher string) (*turso.DBSeed, error) { - if err := sqliteFileIntegrityChecks(file, cipher); err != nil { + format, err := sniffSQLiteFileFormat(file) + if err != nil { return nil, err } + if format == fileFormatRollback { + // The server only accepts WAL or MVCC format files. Converting to WAL + // is exactly the remediation the error message used to instruct users + // to run themselves, and sqlite3 is already a hard requirement here. + fmt.Printf("File %s uses a rollback journal; converting it to WAL mode for import.\n", file) + if out, err := exec.Command("sqlite3", file, "PRAGMA journal_mode=WAL;").CombinedOutput(); err != nil { + return nil, fmt.Errorf("could not convert %s to WAL mode: %w: %s", file, err, out) + } + if format, err = sniffSQLiteFileFormat(file); err != nil { + return nil, err + } + } + + switch format { + case fileFormatWAL: + if err := checkpointWALBeforeUpload(file); err != nil { + return nil, err + } + if err := sqliteFileIntegrityChecks(file, cipher); err != nil { + return nil, err + } + case fileFormatMVCC: + // sqlite3-based checks can't run on MVCC (tursodb format) files: the + // sqlite3 binary reports them as not-a-database. The server verifies + // the file with the tursodb engine after upload. + if !tursoDBFlag { + return nil, fmt.Errorf("%s is in tursodb (MVCC) format and can only be imported into a tursodb database", file) + } + if cipher != "" { + return nil, errors.New("remote encryption is not supported when importing tursodb (MVCC) format files") + } + // The upload ships only the main database file: a non-empty tursodb + // logical log next to it means the file is not a fully checkpointed + // snapshot and importing it would lose the log's data. + if err := checkSidecarEmpty(tursodbLogPath(file), "checkpoint the database with tursodb before importing"); err != nil { + return nil, err + } + fileInfo, err := os.Stat(file) + if err != nil { + return nil, fmt.Errorf("failed to get file info: %w", err) + } + if fileInfo.Size() > MaxAWSDBSizeBytes { + return nil, errors.New("database file size exceeds maximum allowed size of 20 GB") + } + case fileFormatNotSQLite: + isDump, err := checkIfDump(file) + if err != nil { + return nil, fmt.Errorf("failed to get file header: %w", err) + } + if isDump { + return nil, fmt.Errorf("%s is a sqlite3 dump, not a sqlite3 database. Please import a sqlite database", file) + } + return nil, fmt.Errorf("file %s is not a valid SQLite database file", file) + default: + return nil, fmt.Errorf("file %s has an unsupported SQLite file format", file) + } + seed := &turso.DBSeed{ Type: "database_upload", Filepath: file, @@ -365,6 +465,16 @@ func handleDBFile(client *turso.Client, file string, isAWS bool, cipher string) return handleDBFileAWS(file, cipher) } + format, err := sniffSQLiteFileFormat(file) + if err != nil { + return nil, err + } + if format == fileFormatMVCC { + // non-AWS groups are seeded by replaying a .dump, which sqlite3 cannot + // produce from an MVCC (tursodb format) file + return nil, fmt.Errorf("%s is in tursodb (MVCC) format and can only be imported into AWS groups", file) + } + if err := checkSQLiteFile(file); err != nil { return nil, err } diff --git a/internal/cmd/group_flag_test.go b/internal/cmd/group_flag_test.go index 6fcf4b7d..275d5d48 100644 --- a/internal/cmd/group_flag_test.go +++ b/internal/cmd/group_flag_test.go @@ -74,3 +74,49 @@ func TestRunQuickCheck(t *testing.T) { require.Error(t, err) }) } + +func TestTursodbLogPath(t *testing.T) { + require.Equal(t, "data.db-log", tursodbLogPath("data.db")) + require.Equal(t, "data.db-log", tursodbLogPath("data.sqlite")) + require.Equal(t, "/some/dir/mydb.db-log", tursodbLogPath("/some/dir/mydb.db")) + require.Equal(t, "noext.db-log", tursodbLogPath("noext")) +} + +func TestCheckSidecarEmpty(t *testing.T) { + dir := t.TempDir() + + t.Run("missing sidecar is fine", func(t *testing.T) { + require.NoError(t, checkSidecarEmpty(filepath.Join(dir, "missing-wal"), "hint")) + }) + + t.Run("empty sidecar is fine", func(t *testing.T) { + path := filepath.Join(dir, "empty-wal") + require.NoError(t, os.WriteFile(path, nil, 0644)) + require.NoError(t, checkSidecarEmpty(path, "hint")) + }) + + t.Run("non-empty sidecar errors with hint", func(t *testing.T) { + path := filepath.Join(dir, "full-wal") + require.NoError(t, os.WriteFile(path, []byte("frames"), 0644)) + err := checkSidecarEmpty(path, "close all connections") + require.Error(t, err) + require.Contains(t, err.Error(), "importing would lose the data it holds") + require.Contains(t, err.Error(), "close all connections") + }) +} + +func TestCheckpointWALBeforeUpload(t *testing.T) { + if _, err := exec.LookPath("sqlite3"); err != nil { + t.Skip("sqlite3 not available, skipping test") + } + + dbPath := createTestDatabase(t, 10*1024) + require.NoError(t, checkpointWALBeforeUpload(dbPath)) + + // after the checkpoint no data-bearing sidecars may remain + for _, suffix := range []string{"-wal", "-journal"} { + if info, err := os.Stat(dbPath + suffix); err == nil { + require.Zero(t, info.Size(), "%s must be empty after checkpoint", dbPath+suffix) + } + } +} diff --git a/internal/cmd/sqlite_header.go b/internal/cmd/sqlite_header.go new file mode 100644 index 00000000..8d3609a4 --- /dev/null +++ b/internal/cmd/sqlite_header.go @@ -0,0 +1,52 @@ +package cmd + +import ( + "io" + "os" +) + +type sqliteFileFormat int + +const ( + fileFormatNotSQLite sqliteFileFormat = iota + fileFormatRollback + fileFormatWAL + fileFormatMVCC + fileFormatUnknown +) + +const sqliteMagic = "SQLite format 3\x00" + +// sniffSQLiteFileFormat classifies a database file by its SQLite header: the +// 16-byte magic string plus the read/write format version bytes at offsets +// 18/19 (1 = rollback journal, 2 = WAL, 255 = MVCC, i.e. tursodb format). +// +// This must run before any sqlite3 shellout: the sqlite3 binary reports +// MVCC-format files as not-a-database, so they have to be routed around the +// sqlite3-based checks entirely. +func sniffSQLiteFileFormat(path string) (sqliteFileFormat, error) { + file, err := os.Open(path) + if err != nil { + return fileFormatNotSQLite, err + } + defer file.Close() + + header := make([]byte, 20) + if _, err := io.ReadFull(file, header); err != nil { + // too short to hold a SQLite header + return fileFormatNotSQLite, nil + } + if string(header[:len(sqliteMagic)]) != sqliteMagic { + return fileFormatNotSQLite, nil + } + readVersion, writeVersion := header[18], header[19] + switch { + case readVersion == 1 && writeVersion == 1: + return fileFormatRollback, nil + case readVersion == 2 && writeVersion == 2: + return fileFormatWAL, nil + case readVersion == 255 && writeVersion == 255: + return fileFormatMVCC, nil + } + return fileFormatUnknown, nil +} diff --git a/internal/cmd/sqlite_header_test.go b/internal/cmd/sqlite_header_test.go new file mode 100644 index 00000000..c538ce15 --- /dev/null +++ b/internal/cmd/sqlite_header_test.go @@ -0,0 +1,90 @@ +package cmd + +import ( + "os" + "os/exec" + "path/filepath" + "testing" + + "github.com/stretchr/testify/require" +) + +func writeHeaderFile(t *testing.T, magic string, readVersion, writeVersion byte) string { + t.Helper() + header := make([]byte, 100) + copy(header, magic) + header[18] = readVersion + header[19] = writeVersion + path := filepath.Join(t.TempDir(), "test.db") + require.NoError(t, os.WriteFile(path, header, 0644)) + return path +} + +func TestSniffSQLiteFileFormat(t *testing.T) { + for _, tc := range []struct { + name string + magic string + readVersion, writeVersion byte + expected sqliteFileFormat + }{ + {"rollback journal", sqliteMagic, 1, 1, fileFormatRollback}, + {"wal", sqliteMagic, 2, 2, fileFormatWAL}, + {"mvcc", sqliteMagic, 255, 255, fileFormatMVCC}, + {"mixed version bytes", sqliteMagic, 2, 255, fileFormatUnknown}, + {"unknown version bytes", sqliteMagic, 42, 42, fileFormatUnknown}, + {"wrong magic", "Not a SQLite db\x00", 2, 2, fileFormatNotSQLite}, + } { + t.Run(tc.name, func(t *testing.T) { + path := writeHeaderFile(t, tc.magic, tc.readVersion, tc.writeVersion) + format, err := sniffSQLiteFileFormat(path) + require.NoError(t, err) + require.Equal(t, tc.expected, format) + }) + } + + t.Run("file shorter than header", func(t *testing.T) { + path := filepath.Join(t.TempDir(), "short.db") + require.NoError(t, os.WriteFile(path, []byte("SQLite"), 0644)) + format, err := sniffSQLiteFileFormat(path) + require.NoError(t, err) + require.Equal(t, fileFormatNotSQLite, format) + }) + + t.Run("missing file", func(t *testing.T) { + _, err := sniffSQLiteFileFormat(filepath.Join(t.TempDir(), "missing.db")) + require.Error(t, err) + }) +} + +func TestSniffSQLiteFileFormatOnRealDatabases(t *testing.T) { + if _, err := exec.LookPath("sqlite3"); err != nil { + t.Skip("sqlite3 not available, skipping test") + } + + t.Run("wal database", func(t *testing.T) { + dbPath := createTestDatabase(t, 10*1024) + format, err := sniffSQLiteFileFormat(dbPath) + require.NoError(t, err) + require.Equal(t, fileFormatWAL, format) + }) + + t.Run("rollback database converts to wal", func(t *testing.T) { + dbPath := filepath.Join(t.TempDir(), "rollback.db") + cmd := exec.Command("sqlite3", "-list", dbPath, + "PRAGMA page_size=4096;", + "CREATE TABLE data (id INTEGER PRIMARY KEY);") + require.NoError(t, cmd.Run(), "failed to create test database") + + format, err := sniffSQLiteFileFormat(dbPath) + require.NoError(t, err) + require.Equal(t, fileFormatRollback, format) + + // the conversion handleDBFileAWS performs for rollback files + out, err := exec.Command("sqlite3", dbPath, "PRAGMA journal_mode=WAL;").CombinedOutput() + require.NoError(t, err, "convert to WAL: %s", out) + + format, err = sniffSQLiteFileFormat(dbPath) + require.NoError(t, err) + require.Equal(t, fileFormatWAL, format) + }) +} diff --git a/internal/turso/databases.go b/internal/turso/databases.go index 6726a275..9c0ebb69 100644 --- a/internal/turso/databases.go +++ b/internal/turso/databases.go @@ -242,7 +242,7 @@ func (d *DatabasesClient) Create(name, location, image, extensions, group string } if isTursoServerUpload { - if _, err = d.UploadDatabaseAWS(data, group, uploadFilepath, remoteEncryptionCipher, remoteEncryptionKey, spinner); err != nil { + if _, err = d.UploadDatabaseAWS(data, group, uploadFilepath, remoteEncryptionCipher, remoteEncryptionKey, useTursoDB, spinner); err != nil { // Clean up the database if the upload fails if deleteErr := d.Delete(data.Database.Name); deleteErr != nil { fmt.Printf("%v", deleteErr) @@ -263,7 +263,7 @@ func (d *DatabasesClient) Create(name, location, image, extensions, group string // This call happens in DatabasesClient.Create() above, after which it calls this function. // 2. This function creates a DB token for the newly-created DB, and then calls turso-server to upload the database file. // turso-server will perform validations on the file and 'activate' the db if everything is ok. -func (d *DatabasesClient) UploadDatabaseAWS(resp *CreateDatabaseResponse, group, uploadFilepath, remoteEncryptionCipher, remoteEncryptionKey string, spinner *prompt.SpinnerT) (*CreateDatabaseResponse, error) { +func (d *DatabasesClient) UploadDatabaseAWS(resp *CreateDatabaseResponse, group, uploadFilepath, remoteEncryptionCipher, remoteEncryptionKey string, useTursoDB bool, spinner *prompt.SpinnerT) (*CreateDatabaseResponse, error) { dbName := resp.Database.Name tokenTTL := 5 * time.Minute tokenProvider := func() (string, error) { @@ -282,7 +282,13 @@ func (d *DatabasesClient) UploadDatabaseAWS(resp *CreateDatabaseResponse, group, // Upload the database file spinner.Text(fmt.Sprintf("Uploading database %s in group %s, this may take a while...", internal.Emph(resp.Database.Name), internal.Emph(group))) - err = tursoServerClient.UploadFileMultipart(uploadFilepath, remoteEncryptionCipher, remoteEncryptionKey, func(progressPct int, uploadedBytes int64, totalBytes int64, elapsedTime time.Duration, done bool) { + // TursoDB databases only accept MVCC-format uploads: the WAL file's + // format version bytes are rewritten to MVCC in the upload stream. + upload := tursoServerClient.UploadFileMultipart + if useTursoDB { + upload = tursoServerClient.UploadFileMultipartMVCC + } + err = upload(uploadFilepath, remoteEncryptionCipher, remoteEncryptionKey, func(progressPct int, uploadedBytes int64, totalBytes int64, elapsedTime time.Duration, done bool) { totalSeconds := int(elapsedTime.Seconds()) minutes := totalSeconds / 60 seconds := totalSeconds % 60 diff --git a/internal/turso/tursoServer.go b/internal/turso/tursoServer.go index 60dd129d..8b53b0aa 100644 --- a/internal/turso/tursoServer.go +++ b/internal/turso/tursoServer.go @@ -158,7 +158,7 @@ type chunkUploadContext struct { chunkPath string chunkSize int64 chunkStartOffset int64 // File offset where this chunk starts - file *os.File + file io.ReadSeeker headers map[string]string totalSize int64 startTime time.Time @@ -271,6 +271,19 @@ func (i *TursoServerClient) uploadChunkWithRetry(ctx *chunkUploadContext, maxRet // UploadFileMultipart uploads a database file using the multipart upload flow. func (i *TursoServerClient) UploadFileMultipart(filepath string, remoteEncryptionCipher, remoteEncryptionKey string, onUploadProgress func(progressPct int, uploadedBytes int64, totalBytes int64, elapsedTime time.Duration, done bool)) error { + return i.uploadFileMultipart(filepath, false, remoteEncryptionCipher, remoteEncryptionKey, onUploadProgress) +} + +// UploadFileMultipartMVCC uploads a database file like UploadFileMultipart, +// but rewrites the SQLite read/write format version bytes (header offsets +// 18/19) to 255 (MVCC) in the upload stream. TursoDB databases only accept +// MVCC-format uploads, and a checkpointed WAL file differs from tursodb +// format in exactly those two bytes; the file on disk is left untouched. +func (i *TursoServerClient) UploadFileMultipartMVCC(filepath string, remoteEncryptionCipher, remoteEncryptionKey string, onUploadProgress func(progressPct int, uploadedBytes int64, totalBytes int64, elapsedTime time.Duration, done bool)) error { + return i.uploadFileMultipart(filepath, true, remoteEncryptionCipher, remoteEncryptionKey, onUploadProgress) +} + +func (i *TursoServerClient) uploadFileMultipart(filepath string, convertToMVCC bool, remoteEncryptionCipher, remoteEncryptionKey string, onUploadProgress func(progressPct int, uploadedBytes int64, totalBytes int64, elapsedTime time.Duration, done bool)) error { file, err := os.Open(filepath) if err != nil { return fmt.Errorf("failed to open file %s: %w", filepath, err) @@ -295,7 +308,12 @@ func (i *TursoServerClient) UploadFileMultipart(filepath string, remoteEncryptio return err } - uploadedBytes, err := i.uploadChunks(uploadStart.UploadID, uploadStart.ChunkSize, file, totalSize, startTime, remoteEncryptionCipher, remoteEncryptionKey, onUploadProgress) + var reader io.ReadSeeker = file + if convertToMVCC { + reader = &mvccFormatReader{file: file} + } + + uploadedBytes, err := i.uploadChunks(uploadStart.UploadID, uploadStart.ChunkSize, reader, totalSize, startTime, remoteEncryptionCipher, remoteEncryptionKey, onUploadProgress) if err != nil { return err } @@ -355,7 +373,32 @@ func (i *TursoServerClient) startMultipartUpload(dbSize int64) (multipartUploadS return multipartUploadStart(uploadResp), nil } -func (i *TursoServerClient) uploadChunks(uploadID string, chunkSize int64, file *os.File, totalSize int64, startTime time.Time, remoteEncryptionCipher, remoteEncryptionKey string, onUploadProgress func(progressPct int, uploadedBytes int64, totalBytes int64, elapsedTime time.Duration, done bool)) (int64, error) { +// mvccFormatReader wraps a database file and rewrites the SQLite read/write +// format version bytes (offsets 18/19) to 255 (MVCC) as the data streams +// through. Seeks pass through, so chunk retries re-read patched data. +type mvccFormatReader struct { + file *os.File + pos int64 +} + +func (r *mvccFormatReader) Seek(offset int64, whence int) (int64, error) { + pos, err := r.file.Seek(offset, whence) + r.pos = pos + return pos, err +} + +func (r *mvccFormatReader) Read(p []byte) (int, error) { + n, err := r.file.Read(p) + for _, formatByteOffset := range []int64{18, 19} { + if formatByteOffset >= r.pos && formatByteOffset < r.pos+int64(n) { + p[formatByteOffset-r.pos] = 255 + } + } + r.pos += int64(n) + return n, err +} + +func (i *TursoServerClient) uploadChunks(uploadID string, chunkSize int64, file io.ReadSeeker, totalSize int64, startTime time.Time, remoteEncryptionCipher, remoteEncryptionKey string, onUploadProgress func(progressPct int, uploadedBytes int64, totalBytes int64, elapsedTime time.Duration, done bool)) (int64, error) { var uploadedBytes int64 = 0 chunkID := 0 lastProgressPct := -1 diff --git a/internal/turso/tursoServer_test.go b/internal/turso/tursoServer_test.go index 8b1a3a07..21ee6c35 100644 --- a/internal/turso/tursoServer_test.go +++ b/internal/turso/tursoServer_test.go @@ -1162,3 +1162,54 @@ func TestProgressReader_UpdatesTrackingFieldsCorrectly(t *testing.T) { require.Equal(t, int64(50), pr.lastUpdateBytes, "lastUpdateBytes should track uploaded bytes") }) } + +func TestUploadFileMultipartMVCC_RewritesFormatBytes(t *testing.T) { + mock := NewMockTursoServer() + mock.chunkSize = 16 // tiny chunks so bytes 18/19 land mid-stream + defer mock.Close() + + content := make([]byte, 100) + for i := range content { + content[i] = byte(i) + } + content[18], content[19] = 2, 2 // WAL format version bytes + + client := createTestClient(t, mock.URL) + testFile := createTestFileWithContent(t, content) + progress := NewProgressRecorder() + + err := client.UploadFileMultipartMVCC(testFile, "", "", progress.Callback()) + require.NoError(t, err) + + uploaded := mock.GetAllChunkData() + require.Len(t, uploaded, len(content)) + require.Equal(t, byte(255), uploaded[18]) + require.Equal(t, byte(255), uploaded[19]) + // everything else must be untouched + require.Equal(t, content[:18], uploaded[:18]) + require.Equal(t, content[20:], uploaded[20:]) + + // the file on disk keeps its original bytes + onDisk, err := os.ReadFile(testFile) + require.NoError(t, err) + require.Equal(t, content, onDisk) +} + +func TestUploadFileMultipart_DoesNotRewriteFormatBytes(t *testing.T) { + mock := NewMockTursoServer() + mock.chunkSize = 16 + defer mock.Close() + + content := make([]byte, 100) + content[18], content[19] = 2, 2 + + client := createTestClient(t, mock.URL) + testFile := createTestFileWithContent(t, content) + progress := NewProgressRecorder() + + err := client.UploadFileMultipart(testFile, "", "", progress.Callback()) + require.NoError(t, err) + + uploaded := mock.GetAllChunkData() + require.Equal(t, content, uploaded) +} From b538e422cf012043c49468d55b73aec09ea598be Mon Sep 17 00:00:00 2001 From: pedrocarlo Date: Mon, 27 Jul 2026 16:09:05 -0300 Subject: [PATCH 2/6] use tursodb to prepare database imports --- internal/cmd/group_flag.go | 92 ++++++++++++++++++++++++------ internal/cmd/group_flag_test.go | 19 ++++++ internal/turso/databases.go | 12 +--- internal/turso/tursoServer.go | 45 +-------------- internal/turso/tursoServer_test.go | 32 ----------- 5 files changed, 97 insertions(+), 103 deletions(-) diff --git a/internal/cmd/group_flag.go b/internal/cmd/group_flag.go index 0e4e876b..36a59d41 100644 --- a/internal/cmd/group_flag.go +++ b/internal/cmd/group_flag.go @@ -361,6 +361,42 @@ func checkpointWALBeforeUpload(file string) error { return nil } +// prepareTursoDBFile asks the TursoDB engine to convert the database to MVCC, +// checkpoint any logical-log entries into the main file, and validate the +// resulting database. Uploading only the main file is safe once all data-bearing +// sidecars are empty. +func prepareTursoDBFile(file string) error { + output, err := exec.Command("tursodb", "-q", "-m", "list", file, + "PRAGMA journal_mode = mvcc; PRAGMA wal_checkpoint(TRUNCATE); PRAGMA quick_check;").CombinedOutput() + if err != nil { + return fmt.Errorf("could not prepare %s for TursoDB import: %w: %s", file, err, strings.TrimSpace(string(output))) + } + + lines := strings.Fields(string(output)) + if len(lines) == 0 || lines[len(lines)-1] != "ok" { + return fmt.Errorf("TursoDB integrity check failed for %s: %s", file, strings.TrimSpace(string(output))) + } + + format, err := sniffSQLiteFileFormat(file) + if err != nil { + return err + } + if format != fileFormatMVCC { + return fmt.Errorf("TursoDB did not convert %s to MVCC format", file) + } + + for _, sidecar := range []struct{ path, hint string }{ + {file + "-wal", "close all connections to the database and retry the import"}, + {file + "-journal", "the database has a leftover rollback journal; close it cleanly and retry the import"}, + {tursodbLogPath(file), "close all TursoDB connections and retry the import"}, + } { + if err := checkSidecarEmpty(sidecar.path, sidecar.hint); err != nil { + return err + } + } + return nil +} + func checkSidecarEmpty(sidecarPath, hint string) error { info, err := os.Stat(sidecarPath) if errors.Is(err, os.ErrNotExist) { @@ -389,6 +425,9 @@ func handleDBFileAWS(file string, cipher string) (*turso.DBSeed, error) { } if format == fileFormatRollback { + if err := checkSQLiteAvailable(); err != nil { + return nil, err + } // The server only accepts WAL or MVCC format files. Converting to WAL // is exactly the remediation the error message used to instruct users // to run themselves, and sqlite3 is already a hard requirement here. @@ -403,6 +442,9 @@ func handleDBFileAWS(file string, cipher string) (*turso.DBSeed, error) { switch format { case fileFormatWAL: + if err := checkSQLiteAvailable(); err != nil { + return nil, err + } if err := checkpointWALBeforeUpload(file); err != nil { return nil, err } @@ -411,27 +453,13 @@ func handleDBFileAWS(file string, cipher string) (*turso.DBSeed, error) { } case fileFormatMVCC: // sqlite3-based checks can't run on MVCC (tursodb format) files: the - // sqlite3 binary reports them as not-a-database. The server verifies - // the file with the tursodb engine after upload. + // sqlite3 binary reports them as not-a-database. if !tursoDBFlag { return nil, fmt.Errorf("%s is in tursodb (MVCC) format and can only be imported into a tursodb database", file) } if cipher != "" { return nil, errors.New("remote encryption is not supported when importing tursodb (MVCC) format files") } - // The upload ships only the main database file: a non-empty tursodb - // logical log next to it means the file is not a fully checkpointed - // snapshot and importing it would lose the log's data. - if err := checkSidecarEmpty(tursodbLogPath(file), "checkpoint the database with tursodb before importing"); err != nil { - return nil, err - } - fileInfo, err := os.Stat(file) - if err != nil { - return nil, fmt.Errorf("failed to get file info: %w", err) - } - if fileInfo.Size() > MaxAWSDBSizeBytes { - return nil, errors.New("database file size exceeds maximum allowed size of 20 GB") - } case fileFormatNotSQLite: isDump, err := checkIfDump(file) if err != nil { @@ -445,6 +473,26 @@ func handleDBFileAWS(file string, cipher string) (*turso.DBSeed, error) { return nil, fmt.Errorf("file %s has an unsupported SQLite file format", file) } + if tursoDBFlag { + if err := checkTursoDBAvailable(); err != nil { + return nil, err + } + if format != fileFormatMVCC { + fmt.Printf("Converting %s to TursoDB (MVCC) format for import.\n", file) + } + if err := prepareTursoDBFile(file); err != nil { + return nil, err + } + } + + fileInfo, err := os.Stat(file) + if err != nil { + return nil, fmt.Errorf("failed to get file info: %w", err) + } + if fileInfo.Size() > MaxAWSDBSizeBytes { + return nil, errors.New("database file size exceeds maximum allowed size of 20 GB") + } + seed := &turso.DBSeed{ Type: "database_upload", Filepath: file, @@ -457,9 +505,6 @@ func handleDBFile(client *turso.Client, file string, isAWS bool, cipher string) if err := checkFileExists(file); err != nil { return nil, err } - if err := checkSQLiteAvailable(); err != nil { - return nil, err - } if isAWS { return handleDBFileAWS(file, cipher) @@ -474,6 +519,9 @@ func handleDBFile(client *turso.Client, file string, isAWS bool, cipher string) // produce from an MVCC (tursodb format) file return nil, fmt.Errorf("%s is in tursodb (MVCC) format and can only be imported into AWS groups", file) } + if err := checkSQLiteAvailable(); err != nil { + return nil, err + } if err := checkSQLiteFile(file); err != nil { return nil, err @@ -507,6 +555,14 @@ func checkSQLiteAvailable() error { return err } +func checkTursoDBAvailable() error { + _, err := exec.LookPath("tursodb") + if errors.Is(err, exec.ErrNotFound) { + return errors.New("could not find tursodb on your system. Please install it to import into a TursoDB database") + } + return err +} + func checkSQLiteFile(file string) error { output, err := exec.Command("sqlite3", "-list", file, "pragma quick_check;").CombinedOutput() diff --git a/internal/cmd/group_flag_test.go b/internal/cmd/group_flag_test.go index 275d5d48..dd231d99 100644 --- a/internal/cmd/group_flag_test.go +++ b/internal/cmd/group_flag_test.go @@ -120,3 +120,22 @@ func TestCheckpointWALBeforeUpload(t *testing.T) { } } } + +func TestPrepareTursoDBFile(t *testing.T) { + if _, err := exec.LookPath("tursodb"); err != nil { + t.Skip("tursodb not available, skipping test") + } + + dbPath := createTestDatabase(t, 10*1024) + require.NoError(t, checkpointWALBeforeUpload(dbPath)) + require.NoError(t, prepareTursoDBFile(dbPath)) + + format, err := sniffSQLiteFileFormat(dbPath) + require.NoError(t, err) + require.Equal(t, fileFormatMVCC, format) + for _, sidecar := range []string{dbPath + "-wal", dbPath + "-journal", tursodbLogPath(dbPath)} { + if info, err := os.Stat(sidecar); err == nil { + require.Zero(t, info.Size(), "%s must be empty after checkpoint", sidecar) + } + } +} diff --git a/internal/turso/databases.go b/internal/turso/databases.go index 9c0ebb69..6726a275 100644 --- a/internal/turso/databases.go +++ b/internal/turso/databases.go @@ -242,7 +242,7 @@ func (d *DatabasesClient) Create(name, location, image, extensions, group string } if isTursoServerUpload { - if _, err = d.UploadDatabaseAWS(data, group, uploadFilepath, remoteEncryptionCipher, remoteEncryptionKey, useTursoDB, spinner); err != nil { + if _, err = d.UploadDatabaseAWS(data, group, uploadFilepath, remoteEncryptionCipher, remoteEncryptionKey, spinner); err != nil { // Clean up the database if the upload fails if deleteErr := d.Delete(data.Database.Name); deleteErr != nil { fmt.Printf("%v", deleteErr) @@ -263,7 +263,7 @@ func (d *DatabasesClient) Create(name, location, image, extensions, group string // This call happens in DatabasesClient.Create() above, after which it calls this function. // 2. This function creates a DB token for the newly-created DB, and then calls turso-server to upload the database file. // turso-server will perform validations on the file and 'activate' the db if everything is ok. -func (d *DatabasesClient) UploadDatabaseAWS(resp *CreateDatabaseResponse, group, uploadFilepath, remoteEncryptionCipher, remoteEncryptionKey string, useTursoDB bool, spinner *prompt.SpinnerT) (*CreateDatabaseResponse, error) { +func (d *DatabasesClient) UploadDatabaseAWS(resp *CreateDatabaseResponse, group, uploadFilepath, remoteEncryptionCipher, remoteEncryptionKey string, spinner *prompt.SpinnerT) (*CreateDatabaseResponse, error) { dbName := resp.Database.Name tokenTTL := 5 * time.Minute tokenProvider := func() (string, error) { @@ -282,13 +282,7 @@ func (d *DatabasesClient) UploadDatabaseAWS(resp *CreateDatabaseResponse, group, // Upload the database file spinner.Text(fmt.Sprintf("Uploading database %s in group %s, this may take a while...", internal.Emph(resp.Database.Name), internal.Emph(group))) - // TursoDB databases only accept MVCC-format uploads: the WAL file's - // format version bytes are rewritten to MVCC in the upload stream. - upload := tursoServerClient.UploadFileMultipart - if useTursoDB { - upload = tursoServerClient.UploadFileMultipartMVCC - } - err = upload(uploadFilepath, remoteEncryptionCipher, remoteEncryptionKey, func(progressPct int, uploadedBytes int64, totalBytes int64, elapsedTime time.Duration, done bool) { + err = tursoServerClient.UploadFileMultipart(uploadFilepath, remoteEncryptionCipher, remoteEncryptionKey, func(progressPct int, uploadedBytes int64, totalBytes int64, elapsedTime time.Duration, done bool) { totalSeconds := int(elapsedTime.Seconds()) minutes := totalSeconds / 60 seconds := totalSeconds % 60 diff --git a/internal/turso/tursoServer.go b/internal/turso/tursoServer.go index 8b53b0aa..3125d125 100644 --- a/internal/turso/tursoServer.go +++ b/internal/turso/tursoServer.go @@ -271,19 +271,6 @@ func (i *TursoServerClient) uploadChunkWithRetry(ctx *chunkUploadContext, maxRet // UploadFileMultipart uploads a database file using the multipart upload flow. func (i *TursoServerClient) UploadFileMultipart(filepath string, remoteEncryptionCipher, remoteEncryptionKey string, onUploadProgress func(progressPct int, uploadedBytes int64, totalBytes int64, elapsedTime time.Duration, done bool)) error { - return i.uploadFileMultipart(filepath, false, remoteEncryptionCipher, remoteEncryptionKey, onUploadProgress) -} - -// UploadFileMultipartMVCC uploads a database file like UploadFileMultipart, -// but rewrites the SQLite read/write format version bytes (header offsets -// 18/19) to 255 (MVCC) in the upload stream. TursoDB databases only accept -// MVCC-format uploads, and a checkpointed WAL file differs from tursodb -// format in exactly those two bytes; the file on disk is left untouched. -func (i *TursoServerClient) UploadFileMultipartMVCC(filepath string, remoteEncryptionCipher, remoteEncryptionKey string, onUploadProgress func(progressPct int, uploadedBytes int64, totalBytes int64, elapsedTime time.Duration, done bool)) error { - return i.uploadFileMultipart(filepath, true, remoteEncryptionCipher, remoteEncryptionKey, onUploadProgress) -} - -func (i *TursoServerClient) uploadFileMultipart(filepath string, convertToMVCC bool, remoteEncryptionCipher, remoteEncryptionKey string, onUploadProgress func(progressPct int, uploadedBytes int64, totalBytes int64, elapsedTime time.Duration, done bool)) error { file, err := os.Open(filepath) if err != nil { return fmt.Errorf("failed to open file %s: %w", filepath, err) @@ -308,12 +295,7 @@ func (i *TursoServerClient) uploadFileMultipart(filepath string, convertToMVCC b return err } - var reader io.ReadSeeker = file - if convertToMVCC { - reader = &mvccFormatReader{file: file} - } - - uploadedBytes, err := i.uploadChunks(uploadStart.UploadID, uploadStart.ChunkSize, reader, totalSize, startTime, remoteEncryptionCipher, remoteEncryptionKey, onUploadProgress) + uploadedBytes, err := i.uploadChunks(uploadStart.UploadID, uploadStart.ChunkSize, file, totalSize, startTime, remoteEncryptionCipher, remoteEncryptionKey, onUploadProgress) if err != nil { return err } @@ -373,31 +355,6 @@ func (i *TursoServerClient) startMultipartUpload(dbSize int64) (multipartUploadS return multipartUploadStart(uploadResp), nil } -// mvccFormatReader wraps a database file and rewrites the SQLite read/write -// format version bytes (offsets 18/19) to 255 (MVCC) as the data streams -// through. Seeks pass through, so chunk retries re-read patched data. -type mvccFormatReader struct { - file *os.File - pos int64 -} - -func (r *mvccFormatReader) Seek(offset int64, whence int) (int64, error) { - pos, err := r.file.Seek(offset, whence) - r.pos = pos - return pos, err -} - -func (r *mvccFormatReader) Read(p []byte) (int, error) { - n, err := r.file.Read(p) - for _, formatByteOffset := range []int64{18, 19} { - if formatByteOffset >= r.pos && formatByteOffset < r.pos+int64(n) { - p[formatByteOffset-r.pos] = 255 - } - } - r.pos += int64(n) - return n, err -} - func (i *TursoServerClient) uploadChunks(uploadID string, chunkSize int64, file io.ReadSeeker, totalSize int64, startTime time.Time, remoteEncryptionCipher, remoteEncryptionKey string, onUploadProgress func(progressPct int, uploadedBytes int64, totalBytes int64, elapsedTime time.Duration, done bool)) (int64, error) { var uploadedBytes int64 = 0 chunkID := 0 diff --git a/internal/turso/tursoServer_test.go b/internal/turso/tursoServer_test.go index 21ee6c35..a3f21c3b 100644 --- a/internal/turso/tursoServer_test.go +++ b/internal/turso/tursoServer_test.go @@ -1163,38 +1163,6 @@ func TestProgressReader_UpdatesTrackingFieldsCorrectly(t *testing.T) { }) } -func TestUploadFileMultipartMVCC_RewritesFormatBytes(t *testing.T) { - mock := NewMockTursoServer() - mock.chunkSize = 16 // tiny chunks so bytes 18/19 land mid-stream - defer mock.Close() - - content := make([]byte, 100) - for i := range content { - content[i] = byte(i) - } - content[18], content[19] = 2, 2 // WAL format version bytes - - client := createTestClient(t, mock.URL) - testFile := createTestFileWithContent(t, content) - progress := NewProgressRecorder() - - err := client.UploadFileMultipartMVCC(testFile, "", "", progress.Callback()) - require.NoError(t, err) - - uploaded := mock.GetAllChunkData() - require.Len(t, uploaded, len(content)) - require.Equal(t, byte(255), uploaded[18]) - require.Equal(t, byte(255), uploaded[19]) - // everything else must be untouched - require.Equal(t, content[:18], uploaded[:18]) - require.Equal(t, content[20:], uploaded[20:]) - - // the file on disk keeps its original bytes - onDisk, err := os.ReadFile(testFile) - require.NoError(t, err) - require.Equal(t, content, onDisk) -} - func TestUploadFileMultipart_DoesNotRewriteFormatBytes(t *testing.T) { mock := NewMockTursoServer() mock.chunkSize = 16 From d4af71bb2ff79450f4807eeeb1f039a8e9c73146 Mon Sep 17 00:00:00 2001 From: pedrocarlo Date: Tue, 28 Jul 2026 13:11:18 -0300 Subject: [PATCH 3/6] default database imports to SQLite --- internal/cmd/db_import.go | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/internal/cmd/db_import.go b/internal/cmd/db_import.go index 84ec54a3..b90630ad 100644 --- a/internal/cmd/db_import.go +++ b/internal/cmd/db_import.go @@ -9,11 +9,8 @@ import ( "github.com/spf13/cobra" ) -// Import defaults to TursoDB (MVCC) databases — the only mode the cloud -// supports for new databases. The flag is separate from createCmd's -// --tursodb (which shares the tursoDBFlag global with a false default) so -// the two commands can have different defaults; RunE copies it over before -// delegating to CreateDatabase. +// Import keeps its own flag value because it delegates database creation to +// CreateDatabase; RunE copies the selected value to the shared create flag. var importTursoDBFlag bool func init() { @@ -21,7 +18,7 @@ func init() { addGroupFlag(importCmd) addRemoteEncryptionKeyFlag(importCmd) addRemoteEncryptionCipherFlag(importCmd) - importCmd.Flags().BoolVar(&importTursoDBFlag, "tursodb", true, "Import into a TursoDB (MVCC) database.") + importCmd.Flags().BoolVar(&importTursoDBFlag, "tursodb", false, "Import into a TursoDB (MVCC) database.") } var importCmd = &cobra.Command{ From c2ed36fa333fe0562795fad5762643360d84f730 Mon Sep 17 00:00:00 2001 From: Pedro Muniz <63058033+pedrocarlo@users.noreply.github.com> Date: Wed, 29 Jul 2026 13:04:30 -0300 Subject: [PATCH 4/6] Apply suggestions from code review MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Mikaël Francoeur --- internal/cmd/db_create.go | 5 ----- internal/cmd/db_import.go | 2 -- internal/cmd/group_flag.go | 15 ++------------- 3 files changed, 2 insertions(+), 20 deletions(-) diff --git a/internal/cmd/db_create.go b/internal/cmd/db_create.go index 8ddb7504..6e971b87 100644 --- a/internal/cmd/db_create.go +++ b/internal/cmd/db_create.go @@ -155,11 +155,6 @@ func createDatabase(client *turso.Client, name, location, groupName string, seed if sizeLimitFlag != "" { return createDatabaseV2(client, name, location, groupName, seed, spinner) } - // Only fork seeds ("database") can go through the v3 API. File seeds - // ("database_upload") must use the v2 flow: the v3 branch never uploads - // the file after creating the database. (This used to compare against - // "upload", a seed type that doesn't exist, which routed uploads to v2 - // by accident.) if seed != nil && seed.Type != "database" { return createDatabaseV2(client, name, location, groupName, seed, spinner) } diff --git a/internal/cmd/db_import.go b/internal/cmd/db_import.go index b90630ad..21cb11cb 100644 --- a/internal/cmd/db_import.go +++ b/internal/cmd/db_import.go @@ -9,8 +9,6 @@ import ( "github.com/spf13/cobra" ) -// Import keeps its own flag value because it delegates database creation to -// CreateDatabase; RunE copies the selected value to the shared create flag. var importTursoDBFlag bool func init() { diff --git a/internal/cmd/group_flag.go b/internal/cmd/group_flag.go index 36a59d41..08edb259 100644 --- a/internal/cmd/group_flag.go +++ b/internal/cmd/group_flag.go @@ -341,11 +341,6 @@ func runQuickCheck(file string) error { return nil } -// checkpointWALBeforeUpload folds any pending WAL frames into the main -// database file and verifies no data is left behind in sidecar files. The -// upload ships only the main database file, so frames still sitting in -// -wal (or a hot rollback journal) would silently be missing from the -// imported database. func checkpointWALBeforeUpload(file string) error { if out, err := exec.Command("sqlite3", "-list", file, "PRAGMA wal_checkpoint(TRUNCATE);").CombinedOutput(); err != nil { return fmt.Errorf("could not checkpoint database %s: %w: %s", file, err, out) @@ -363,8 +358,8 @@ func checkpointWALBeforeUpload(file string) error { // prepareTursoDBFile asks the TursoDB engine to convert the database to MVCC, // checkpoint any logical-log entries into the main file, and validate the -// resulting database. Uploading only the main file is safe once all data-bearing -// sidecars are empty. +// resulting database. Uploading only the main file is safe once the logical log +// and WAL have been truncated. func prepareTursoDBFile(file string) error { output, err := exec.Command("tursodb", "-q", "-m", "list", file, "PRAGMA journal_mode = mvcc; PRAGMA wal_checkpoint(TRUNCATE); PRAGMA quick_check;").CombinedOutput() @@ -411,9 +406,6 @@ func checkSidecarEmpty(sidecarPath, hint string) error { return nil } -// tursodbLogPath returns the logical-log sidecar path tursodb uses for a -// database file, mirroring turso_core's `with_extension("db-log")`: the -// file's extension (if any) is replaced with "db-log". func tursodbLogPath(file string) string { return strings.TrimSuffix(file, filepath.Ext(file)) + ".db-log" } @@ -428,9 +420,6 @@ func handleDBFileAWS(file string, cipher string) (*turso.DBSeed, error) { if err := checkSQLiteAvailable(); err != nil { return nil, err } - // The server only accepts WAL or MVCC format files. Converting to WAL - // is exactly the remediation the error message used to instruct users - // to run themselves, and sqlite3 is already a hard requirement here. fmt.Printf("File %s uses a rollback journal; converting it to WAL mode for import.\n", file) if out, err := exec.Command("sqlite3", file, "PRAGMA journal_mode=WAL;").CombinedOutput(); err != nil { return nil, fmt.Errorf("could not convert %s to WAL mode: %w: %s", file, err, out) From 1b8cb413745e7da84f688f93125d466ada3d15ad Mon Sep 17 00:00:00 2001 From: pedrocarlo Date: Wed, 29 Jul 2026 13:26:35 -0300 Subject: [PATCH 5/6] apply suggestions --- internal/cmd/db_import_file.go | 395 ++++++++++++++++++ ...up_flag_test.go => db_import_file_test.go} | 42 +- internal/cmd/group_flag.go | 316 -------------- 3 files changed, 426 insertions(+), 327 deletions(-) create mode 100644 internal/cmd/db_import_file.go rename internal/cmd/{group_flag_test.go => db_import_file_test.go} (77%) diff --git a/internal/cmd/db_import_file.go b/internal/cmd/db_import_file.go new file mode 100644 index 00000000..c43c1c9f --- /dev/null +++ b/internal/cmd/db_import_file.go @@ -0,0 +1,395 @@ +package cmd + +import ( + "bufio" + "errors" + "fmt" + "log" + "os" + "os/exec" + "path/filepath" + "strconv" + "strings" + + "github.com/tursodatabase/turso-cli/internal/flags" + "github.com/tursodatabase/turso-cli/internal/prompt" + "github.com/tursodatabase/turso-cli/internal/turso" +) + +const MaxAWSDBSizeBytes = 1024 * 1024 * 1024 * 20 // 20 GB + +const databaseSettingsQuery = "select journal_mode as j, page_size as p, auto_vacuum as a, encoding as e from pragma_journal_mode, pragma_page_size, pragma_auto_vacuum, pragma_encoding;" + +type databaseFileChecker struct { + name string + binary string + journalMode string + settings func(string) (databaseSettings, error) + quickCheck func(string) error +} + +type databaseSettings struct { + journalMode string + pageSize string + autoVacuum string + encoding string +} + +func humanReadableSize(bytes int64) string { + const unit = 1024 + if bytes < unit { + return fmt.Sprintf("%d B", bytes) + } + div, exp := int64(unit), 0 + for n := bytes / unit; n >= unit; n /= unit { + div *= unit + exp++ + } + return fmt.Sprintf("%.1f %cB", float64(bytes)/float64(div), "KMGTPE"[exp]) +} + +func checkIfDump(filename string) (bool, error) { + file, err := os.Open(filename) + if err != nil { + return false, err + } + defer file.Close() + scanner := bufio.NewScanner(file) + if scanner.Scan() { + return strings.TrimSpace(scanner.Text()) == "PRAGMA foreign_keys=OFF;", nil + } + return false, scanner.Err() +} + +func validateDatabaseFileSize(file string) error { + if flags.Debug() { + log.Printf("Checking file size...") + } + fileInfo, err := os.Stat(file) + if err != nil { + return fmt.Errorf("failed to get file info: %w", err) + } + if fileInfo.Size() > MaxAWSDBSizeBytes { + return errors.New("database file size exceeds maximum allowed size of 20 GB") + } + return nil +} + +func sqliteFileIntegrityChecks(file string, cipher string) error { + return databaseFileIntegrityChecks(file, cipher, databaseFileChecker{ + name: "SQLite", + binary: "sqlite3", + journalMode: "wal", + settings: sqliteDatabaseSettings, + quickCheck: runQuickCheck, + }) +} + +func tursoDBFileIntegrityChecks(file string) error { + return databaseFileIntegrityChecks(file, "", databaseFileChecker{ + name: "TursoDB", + binary: "tursodb", + journalMode: "mvcc", + settings: tursoDBDatabaseSettings, + quickCheck: runTursoDBQuickCheck, + }) +} + +func databaseFileIntegrityChecks(file, cipher string, checker databaseFileChecker) error { + if flags.Debug() { + log.Printf("Running %s integrity checks on database file %s", checker.name, file) + log.Printf("Checking database settings...") + } + + settings, err := checker.settings(file) + if err != nil { + return err + } + if err := validateDatabaseSettings(file, settings, checker); err != nil { + return err + } + + fileInfo, err := os.Stat(file) + if err != nil { + return fmt.Errorf("failed to get file info: %w", err) + } + if flags.Debug() { + log.Printf("Running integrity check...") + } + spinner := prompt.Spinner(fmt.Sprintf("Validating database file (%s)...", humanReadableSize(fileInfo.Size()))) + err = checker.quickCheck(file) + spinner.Stop() + if err != nil { + return err + } + + if cipher != "" { + if flags.Debug() { + log.Printf("Checking reserved bytes for cipher %s...", cipher) + } + return validateReservedBytes(file, cipher) + } + + return nil +} + +func validateDatabaseSettings(file string, settings databaseSettings, checker databaseFileChecker) error { + if !strings.EqualFold(settings.journalMode, checker.journalMode) { + return fmt.Errorf("database is not in %s mode. Set it with '%s %s \"PRAGMA journal_mode = %s;\"'", strings.ToUpper(checker.journalMode), checker.binary, file, strings.ToUpper(checker.journalMode)) + } + if settings.pageSize != "4096" { + return fmt.Errorf("database must use 4KB page size. You can set it with '%s %s \"PRAGMA page_size = 4096; VACUUM;\"'", checker.binary, file) + } + if settings.autoVacuum != "0" { + return fmt.Errorf("database must have autovacuum disabled. You can set it with '%s %s \"PRAGMA auto_vacuum = 0;\"'", checker.binary, file) + } + if !strings.EqualFold(settings.encoding, "UTF-8") { + return fmt.Errorf("database must use UTF-8 encoding. You can set it with '%s %s \"PRAGMA encoding = 'UTF-8';\"'", checker.binary, file) + } + return nil +} + +func parseDatabaseSettings(output string) (databaseSettings, error) { + values := strings.Split(strings.TrimSpace(output), "|") + if len(values) != 4 { + return databaseSettings{}, fmt.Errorf("unexpected database settings output: %s", strings.TrimSpace(output)) + } + return databaseSettings{ + journalMode: strings.TrimSpace(values[0]), + pageSize: strings.TrimSpace(values[1]), + autoVacuum: strings.TrimSpace(values[2]), + encoding: strings.TrimSpace(values[3]), + }, nil +} + +func sqliteDatabaseSettings(file string) (databaseSettings, error) { + output, err := exec.Command("sqlite3", "-list", file, databaseSettingsQuery).CombinedOutput() + if err != nil { + return databaseSettings{}, fmt.Errorf("failed to check database settings with sqlite3: %w: %s", err, strings.TrimSpace(string(output))) + } + settings, err := parseDatabaseSettings(string(output)) + if err != nil { + return databaseSettings{}, fmt.Errorf("failed to parse database settings from sqlite3: %w", err) + } + return settings, nil +} + +func tursoDBDatabaseSettings(file string) (databaseSettings, error) { + output, err := exec.Command("tursodb", "-q", "-m", "list", file, databaseSettingsQuery).CombinedOutput() + if err != nil { + return databaseSettings{}, fmt.Errorf("failed to check database settings with TursoDB: %w: %s", err, strings.TrimSpace(string(output))) + } + settings, err := parseDatabaseSettings(string(output)) + if err != nil { + return databaseSettings{}, fmt.Errorf("failed to parse database settings from TursoDB: %w", err) + } + return settings, nil +} + +func runQuickCheck(file string) error { + cmd := exec.Command("sqlite3", "-list", file, "pragma quick_check;") + if err := cmd.Run(); err != nil { + return fmt.Errorf("integrity check failed: %w", err) + } + return nil +} + +func runTursoDBQuickCheck(file string) error { + output, err := exec.Command("tursodb", "-q", "-m", "list", file, "PRAGMA quick_check;").CombinedOutput() + if err != nil { + return fmt.Errorf("TursoDB integrity check failed for %s: %w: %s", file, err, strings.TrimSpace(string(output))) + } + fields := strings.Fields(string(output)) + if len(fields) == 0 || fields[len(fields)-1] != "ok" { + return fmt.Errorf("TursoDB integrity check failed for %s: %s", file, strings.TrimSpace(string(output))) + } + return nil +} + +func checkpointWALBeforeUpload(file string) error { + if out, err := exec.Command("sqlite3", "-list", file, "PRAGMA wal_checkpoint(TRUNCATE);").CombinedOutput(); err != nil { + return fmt.Errorf("could not checkpoint database %s: %w: %s", file, err, out) + } + for _, sidecar := range []struct{ suffix, hint string }{ + {"-wal", "close all connections to the database and retry the import"}, + {"-journal", "the database has a leftover rollback journal; open and cleanly close it with sqlite3 first"}, + } { + if err := checkSidecarEmpty(file+sidecar.suffix, sidecar.hint); err != nil { + return err + } + } + return nil +} + +func prepareTursoDBFile(file string) error { + output, err := exec.Command("tursodb", "-q", "-m", "list", file, + "PRAGMA journal_mode = mvcc; PRAGMA wal_checkpoint(TRUNCATE);").CombinedOutput() + if err != nil { + return fmt.Errorf("could not prepare %s for TursoDB import: %w: %s", file, err, strings.TrimSpace(string(output))) + } + + format, err := sniffSQLiteFileFormat(file) + if err != nil { + return err + } + if format != fileFormatMVCC { + return fmt.Errorf("TursoDB did not convert %s to MVCC format", file) + } + return nil +} + +func checkTursoDBSidecars(file string) error { + for _, sidecar := range []struct{ path, hint string }{ + {file + "-wal", "close all connections to the database and retry the import"}, + {file + "-journal", "the database has a leftover rollback journal; close it cleanly and retry the import"}, + {tursodbLogPath(file), "close all TursoDB connections and retry the import"}, + } { + if err := checkSidecarEmpty(sidecar.path, sidecar.hint); err != nil { + return err + } + } + return nil +} + +func checkSidecarEmpty(sidecarPath, hint string) error { + info, err := os.Stat(sidecarPath) + if errors.Is(err, os.ErrNotExist) { + return nil + } + if err != nil { + return fmt.Errorf("could not check %s: %w", sidecarPath, err) + } + if info.Size() > 0 { + return fmt.Errorf("%s is not empty, importing would lose the data it holds: %s", sidecarPath, hint) + } + return nil +} + +func tursodbLogPath(file string) string { + return strings.TrimSuffix(file, filepath.Ext(file)) + ".db-log" +} + +func handleDBFileAWS(file string, cipher string) (*turso.DBSeed, error) { + format, err := sniffSQLiteFileFormat(file) + if err != nil { + return nil, err + } + + if format == fileFormatRollback { + if err := checkSQLiteAvailable(); err != nil { + return nil, err + } + fmt.Printf("File %s uses a rollback journal; converting it to WAL mode for import.\n", file) + if out, err := exec.Command("sqlite3", file, "PRAGMA journal_mode=WAL;").CombinedOutput(); err != nil { + return nil, fmt.Errorf("could not convert %s to WAL mode: %w: %s", file, err, out) + } + if format, err = sniffSQLiteFileFormat(file); err != nil { + return nil, err + } + } + + switch format { + case fileFormatWAL: + if err := checkSQLiteAvailable(); err != nil { + return nil, err + } + if err := checkpointWALBeforeUpload(file); err != nil { + return nil, err + } + if err := sqliteFileIntegrityChecks(file, cipher); err != nil { + return nil, err + } + case fileFormatMVCC: + if !tursoDBFlag { + return nil, fmt.Errorf("%s is in tursodb (MVCC) format and can only be imported into a tursodb database", file) + } + if cipher != "" { + return nil, errors.New("remote encryption is not supported when importing tursodb (MVCC) format files") + } + case fileFormatNotSQLite: + isDump, err := checkIfDump(file) + if err != nil { + return nil, fmt.Errorf("failed to get file header: %w", err) + } + if isDump { + return nil, fmt.Errorf("%s is a sqlite3 dump, not a sqlite3 database. Please import a sqlite database", file) + } + return nil, fmt.Errorf("file %s is not a valid SQLite database file", file) + case fileFormatUnknown: + return nil, fmt.Errorf("file %s has unsupported SQLite read/write format versions", file) + default: + return nil, fmt.Errorf("file %s has an unsupported SQLite file format", file) + } + + if tursoDBFlag { + if err := checkTursoDBAvailable(); err != nil { + return nil, err + } + if format != fileFormatMVCC { + fmt.Printf("Converting %s to TursoDB (MVCC) format for import.\n", file) + } + if err := prepareTursoDBFile(file); err != nil { + return nil, err + } + if err := tursoDBFileIntegrityChecks(file); err != nil { + return nil, err + } + if err := checkTursoDBSidecars(file); err != nil { + return nil, err + } + } + + if err := validateDatabaseFileSize(file); err != nil { + return nil, err + } + + return &turso.DBSeed{ + Type: "database_upload", + Filepath: file, + }, nil +} + +func getReservedBytes(dbPath string) (int, error) { + output, err := exec.Command("sqlite3", "-list", dbPath, ".filectrl reserve_bytes").CombinedOutput() + if err != nil { + return 0, fmt.Errorf("failed to get reserved bytes: %w", err) + } + outputStr := strings.TrimSpace(string(output)) + if strings.Contains(outputStr, ":") { + parts := strings.Split(outputStr, ":") + if len(parts) >= 2 { + outputStr = strings.TrimSpace(parts[1]) + } + } + + reservedBytes, err := strconv.Atoi(outputStr) + if err != nil { + return 0, fmt.Errorf("failed to parse reserved bytes from output '%s': %w", string(output), err) + } + return reservedBytes, nil +} + +func validateReservedBytes(dbPath string, cipher string) error { + requiredBytes, ok := getRequiredReservedBytes(cipher) + if !ok { + return nil + } + + currentBytes, err := getReservedBytes(dbPath) + if err != nil { + return err + } + if currentBytes != requiredBytes { + return fmt.Errorf("database reserved bytes mismatch: found %d, but cipher '%s' requires %d reserved bytes.\nTo fix this, run:\n\n $ sqlite3 %s\n sqlite> .filectrl reserve_bytes %d\n sqlite> VACUUM;", + currentBytes, cipher, requiredBytes, dbPath, requiredBytes) + } + return nil +} + +func checkTursoDBAvailable() error { + _, err := exec.LookPath("tursodb") + if errors.Is(err, exec.ErrNotFound) { + return errors.New("could not find tursodb on your system. Please install it to import into a TursoDB database") + } + return err +} diff --git a/internal/cmd/group_flag_test.go b/internal/cmd/db_import_file_test.go similarity index 77% rename from internal/cmd/group_flag_test.go rename to internal/cmd/db_import_file_test.go index dd231d99..eec26b59 100644 --- a/internal/cmd/group_flag_test.go +++ b/internal/cmd/db_import_file_test.go @@ -18,8 +18,7 @@ func createTestDatabase(t *testing.T, sizeBytes int) string { t.Skip("sqlite3 not available, skipping test") } - tmpDir := t.TempDir() - dbPath := filepath.Join(tmpDir, "test.db") + dbPath := filepath.Join(t.TempDir(), "test.db") // Create database with correct settings for Turso cmd := exec.Command("sqlite3", "-list", dbPath, @@ -52,29 +51,48 @@ func TestRunQuickCheck(t *testing.T) { t.Run("valid database succeeds", func(t *testing.T) { dbPath := createTestDatabase(t, 10*1024) // 10KB - err := runQuickCheck(dbPath) - require.NoError(t, err) + require.NoError(t, runQuickCheck(dbPath)) }) t.Run("corrupted database returns error", func(t *testing.T) { - tmpDir := t.TempDir() - dbPath := filepath.Join(tmpDir, "corrupt.db") + dbPath := filepath.Join(t.TempDir(), "corrupt.db") // Create a file with garbage data - err := os.WriteFile(dbPath, []byte("not a valid sqlite database content here"), 0644) - require.NoError(t, err) + require.NoError(t, os.WriteFile(dbPath, []byte("not a valid sqlite database content here"), 0644)) - err = runQuickCheck(dbPath) + err := runQuickCheck(dbPath) require.Error(t, err) require.Contains(t, err.Error(), "integrity check failed") }) t.Run("nonexistent file returns error", func(t *testing.T) { - err := runQuickCheck("/nonexistent/path/db.sqlite") - require.Error(t, err) + require.Error(t, runQuickCheck("/nonexistent/path/db.sqlite")) }) } +func TestValidateDatabaseSettings(t *testing.T) { + checker := databaseFileChecker{ + binary: "tursodb", + journalMode: "mvcc", + } + settings := databaseSettings{ + journalMode: "MVCC", + pageSize: "4096", + autoVacuum: "0", + encoding: "UTF-8", + } + + require.NoError(t, validateDatabaseSettings("data.db", settings, checker)) + settings.journalMode = "WAL" + require.ErrorContains(t, validateDatabaseSettings("data.db", settings, checker), "not in MVCC mode") +} + +func TestHandleDBFileAWSRejectsUnknownFormat(t *testing.T) { + dbPath := writeHeaderFile(t, sqliteMagic, 2, 255) + _, err := handleDBFileAWS(dbPath, "") + require.ErrorContains(t, err, "unsupported SQLite read/write format versions") +} + func TestTursodbLogPath(t *testing.T) { require.Equal(t, "data.db-log", tursodbLogPath("data.db")) require.Equal(t, "data.db-log", tursodbLogPath("data.sqlite")) @@ -129,6 +147,8 @@ func TestPrepareTursoDBFile(t *testing.T) { dbPath := createTestDatabase(t, 10*1024) require.NoError(t, checkpointWALBeforeUpload(dbPath)) require.NoError(t, prepareTursoDBFile(dbPath)) + require.NoError(t, tursoDBFileIntegrityChecks(dbPath)) + require.NoError(t, checkTursoDBSidecars(dbPath)) format, err := sniffSQLiteFileFormat(dbPath) require.NoError(t, err) diff --git a/internal/cmd/group_flag.go b/internal/cmd/group_flag.go index 08edb259..20954fe5 100644 --- a/internal/cmd/group_flag.go +++ b/internal/cmd/group_flag.go @@ -5,11 +5,8 @@ import ( "bytes" "errors" "fmt" - "log" "os" "os/exec" - "path/filepath" - "strconv" "strings" "time" @@ -185,311 +182,6 @@ func countFlags(flags ...string) (count int) { return } -const MaxAWSDBSizeBytes = 1024 * 1024 * 1024 * 20 // 20 GB - -func humanReadableSize(bytes int64) string { - const unit = 1024 - if bytes < unit { - return fmt.Sprintf("%d B", bytes) - } - div, exp := int64(unit), 0 - for n := bytes / unit; n >= unit; n /= unit { - div *= unit - exp++ - } - return fmt.Sprintf("%.1f %cB", float64(bytes)/float64(div), "KMGTPE"[exp]) -} - -func checkIfDump(filename string) (bool, error) { - file, err := os.Open(filename) - if err != nil { - return false, err - } - defer file.Close() - scanner := bufio.NewScanner(file) - if scanner.Scan() { - firstLine := scanner.Text() - return strings.TrimSpace(firstLine) == "PRAGMA foreign_keys=OFF;", nil - } else { - return false, scanner.Err() - } -} - -// getReservedBytes retrieves the current reserved bytes setting from a SQLite database -func getReservedBytes(dbPath string) (int, error) { - output, err := exec.Command("sqlite3", "-list", dbPath, ".filectrl reserve_bytes").CombinedOutput() - if err != nil { - return 0, fmt.Errorf("failed to get reserved bytes: %w", err) - } - outputStr := strings.TrimSpace(string(output)) - - if strings.Contains(outputStr, ":") { - parts := strings.Split(outputStr, ":") - if len(parts) >= 2 { - outputStr = strings.TrimSpace(parts[1]) - } - } - - reservedBytes, err := strconv.Atoi(outputStr) - if err != nil { - return 0, fmt.Errorf("failed to parse reserved bytes from output '%s': %w", string(output), err) - } - - return reservedBytes, nil -} - -// validateReservedBytes checks if the database has the required reserved bytes for the given cipher -func validateReservedBytes(dbPath string, cipher string) error { - requiredBytes, ok := getRequiredReservedBytes(cipher) - if !ok { - return nil - } - - currentBytes, err := getReservedBytes(dbPath) - if err != nil { - return err - } - - if currentBytes != requiredBytes { - return fmt.Errorf("database reserved bytes mismatch: found %d, but cipher '%s' requires %d reserved bytes.\nTo fix this, run:\n\n $ sqlite3 %s\n sqlite> .filectrl reserve_bytes %d\n sqlite> VACUUM;", - currentBytes, cipher, requiredBytes, dbPath, requiredBytes) - } - - return nil -} - -func sqliteFileIntegrityChecks(file string, cipher string) error { - if flags.Debug() { - log.Printf("Running integrity checks on database file %s", file) - } - - if flags.Debug() { - log.Printf("Checking if this is a sqlite dump: common mistake!...") - } - - isDump, err := checkIfDump(file) - if err != nil { - return fmt.Errorf("failed to get file header: %w", err) - } - if isDump { - return fmt.Errorf("%s is a sqlite3 dump, not a sqlite3 database. Please import a sqlite database", file) - } - - if flags.Debug() { - log.Printf("Checking file size...") - } - fileInfo, err := os.Stat(file) - if err != nil { - return fmt.Errorf("failed to get file info: %w", err) - } - - if fileInfo.Size() > MaxAWSDBSizeBytes { - return errors.New("database file size exceeds maximum allowed size of 20 GB") - } - - if flags.Debug() { - log.Printf("Checking database settings...") - } - output, err := exec.Command("sqlite3", "-list", file, ".mode line", - "select journal_mode as j, page_size as p, auto_vacuum as a, encoding as e from pragma_journal_mode, pragma_page_size, pragma_auto_vacuum, pragma_encoding;").CombinedOutput() - if err != nil { - - return fmt.Errorf("failed to check database settings: %w", err) - } - - settings := string(output) - if !strings.Contains(settings, "j: wal") && !strings.Contains(settings, "j = wal") { - return fmt.Errorf("database is not in WAL mode. Set it with 'sqlite3 %s 'PRAGMA journal_mode = WAL'", file) - } - if !strings.Contains(settings, "p: 4096") && !strings.Contains(settings, "p = 4096") { - return fmt.Errorf("database must use 4KB page size. you can set it with 'sqlite3 %s 'PRAGMA page_size = 4096; VACUUM;' Note that this is not possible to do if your database is already in WAL mode", file) - } - if !strings.Contains(settings, "a: 0") && !strings.Contains(settings, "a = 0") { - return fmt.Errorf("database must have autovacuum disabled. you can set it with 'sqlite3 %s 'PRAGMA auto_vacuum = 0;'", file) - } - if !strings.Contains(settings, "e: UTF-8") && !strings.Contains(settings, "e = UTF-8") { - return fmt.Errorf("database must use UTF-8 encoding. you can set it with 'sqlite3 %s 'PRAGMA encoding = 'UTF-8' ", file) - } - - // run quick_check - if flags.Debug() { - log.Printf("Running integrity check...") - } - spinner := prompt.Spinner(fmt.Sprintf("Validating database file (%s)...", humanReadableSize(fileInfo.Size()))) - err = runQuickCheck(file) - spinner.Stop() - if err != nil { - return err - } - - // validate reserved bytes if encryption cipher is specified - if cipher != "" { - if flags.Debug() { - log.Printf("Checking reserved bytes for cipher %s...", cipher) - } - return validateReservedBytes(file, cipher) - } - - return nil -} - -func runQuickCheck(file string) error { - cmd := exec.Command("sqlite3", "-list", file, "pragma quick_check;") - if err := cmd.Run(); err != nil { - return fmt.Errorf("integrity check failed: %w", err) - } - return nil -} - -func checkpointWALBeforeUpload(file string) error { - if out, err := exec.Command("sqlite3", "-list", file, "PRAGMA wal_checkpoint(TRUNCATE);").CombinedOutput(); err != nil { - return fmt.Errorf("could not checkpoint database %s: %w: %s", file, err, out) - } - for _, sidecar := range []struct{ suffix, hint string }{ - {"-wal", "close all connections to the database and retry the import"}, - {"-journal", "the database has a leftover rollback journal; open and cleanly close it with sqlite3 first"}, - } { - if err := checkSidecarEmpty(file+sidecar.suffix, sidecar.hint); err != nil { - return err - } - } - return nil -} - -// prepareTursoDBFile asks the TursoDB engine to convert the database to MVCC, -// checkpoint any logical-log entries into the main file, and validate the -// resulting database. Uploading only the main file is safe once the logical log -// and WAL have been truncated. -func prepareTursoDBFile(file string) error { - output, err := exec.Command("tursodb", "-q", "-m", "list", file, - "PRAGMA journal_mode = mvcc; PRAGMA wal_checkpoint(TRUNCATE); PRAGMA quick_check;").CombinedOutput() - if err != nil { - return fmt.Errorf("could not prepare %s for TursoDB import: %w: %s", file, err, strings.TrimSpace(string(output))) - } - - lines := strings.Fields(string(output)) - if len(lines) == 0 || lines[len(lines)-1] != "ok" { - return fmt.Errorf("TursoDB integrity check failed for %s: %s", file, strings.TrimSpace(string(output))) - } - - format, err := sniffSQLiteFileFormat(file) - if err != nil { - return err - } - if format != fileFormatMVCC { - return fmt.Errorf("TursoDB did not convert %s to MVCC format", file) - } - - for _, sidecar := range []struct{ path, hint string }{ - {file + "-wal", "close all connections to the database and retry the import"}, - {file + "-journal", "the database has a leftover rollback journal; close it cleanly and retry the import"}, - {tursodbLogPath(file), "close all TursoDB connections and retry the import"}, - } { - if err := checkSidecarEmpty(sidecar.path, sidecar.hint); err != nil { - return err - } - } - return nil -} - -func checkSidecarEmpty(sidecarPath, hint string) error { - info, err := os.Stat(sidecarPath) - if errors.Is(err, os.ErrNotExist) { - return nil - } - if err != nil { - return fmt.Errorf("could not check %s: %w", sidecarPath, err) - } - if info.Size() > 0 { - return fmt.Errorf("%s is not empty, importing would lose the data it holds: %s", sidecarPath, hint) - } - return nil -} - -func tursodbLogPath(file string) string { - return strings.TrimSuffix(file, filepath.Ext(file)) + ".db-log" -} - -func handleDBFileAWS(file string, cipher string) (*turso.DBSeed, error) { - format, err := sniffSQLiteFileFormat(file) - if err != nil { - return nil, err - } - - if format == fileFormatRollback { - if err := checkSQLiteAvailable(); err != nil { - return nil, err - } - fmt.Printf("File %s uses a rollback journal; converting it to WAL mode for import.\n", file) - if out, err := exec.Command("sqlite3", file, "PRAGMA journal_mode=WAL;").CombinedOutput(); err != nil { - return nil, fmt.Errorf("could not convert %s to WAL mode: %w: %s", file, err, out) - } - if format, err = sniffSQLiteFileFormat(file); err != nil { - return nil, err - } - } - - switch format { - case fileFormatWAL: - if err := checkSQLiteAvailable(); err != nil { - return nil, err - } - if err := checkpointWALBeforeUpload(file); err != nil { - return nil, err - } - if err := sqliteFileIntegrityChecks(file, cipher); err != nil { - return nil, err - } - case fileFormatMVCC: - // sqlite3-based checks can't run on MVCC (tursodb format) files: the - // sqlite3 binary reports them as not-a-database. - if !tursoDBFlag { - return nil, fmt.Errorf("%s is in tursodb (MVCC) format and can only be imported into a tursodb database", file) - } - if cipher != "" { - return nil, errors.New("remote encryption is not supported when importing tursodb (MVCC) format files") - } - case fileFormatNotSQLite: - isDump, err := checkIfDump(file) - if err != nil { - return nil, fmt.Errorf("failed to get file header: %w", err) - } - if isDump { - return nil, fmt.Errorf("%s is a sqlite3 dump, not a sqlite3 database. Please import a sqlite database", file) - } - return nil, fmt.Errorf("file %s is not a valid SQLite database file", file) - default: - return nil, fmt.Errorf("file %s has an unsupported SQLite file format", file) - } - - if tursoDBFlag { - if err := checkTursoDBAvailable(); err != nil { - return nil, err - } - if format != fileFormatMVCC { - fmt.Printf("Converting %s to TursoDB (MVCC) format for import.\n", file) - } - if err := prepareTursoDBFile(file); err != nil { - return nil, err - } - } - - fileInfo, err := os.Stat(file) - if err != nil { - return nil, fmt.Errorf("failed to get file info: %w", err) - } - if fileInfo.Size() > MaxAWSDBSizeBytes { - return nil, errors.New("database file size exceeds maximum allowed size of 20 GB") - } - - seed := &turso.DBSeed{ - Type: "database_upload", - Filepath: file, - } - - return seed, nil -} - func handleDBFile(client *turso.Client, file string, isAWS bool, cipher string) (*turso.DBSeed, error) { if err := checkFileExists(file); err != nil { return nil, err @@ -544,14 +236,6 @@ func checkSQLiteAvailable() error { return err } -func checkTursoDBAvailable() error { - _, err := exec.LookPath("tursodb") - if errors.Is(err, exec.ErrNotFound) { - return errors.New("could not find tursodb on your system. Please install it to import into a TursoDB database") - } - return err -} - func checkSQLiteFile(file string) error { output, err := exec.Command("sqlite3", "-list", file, "pragma quick_check;").CombinedOutput() From 3e94c7868f47275768e87a931a6efc0ad35b5f91 Mon Sep 17 00:00:00 2001 From: pedrocarlo Date: Wed, 29 Jul 2026 17:31:11 -0300 Subject: [PATCH 6/6] use embedded Turso Go driver for imports --- go.mod | 5 +- go.sum | 10 +- internal/cmd/db_import_file.go | 144 +++++++++++++++++++++------- internal/cmd/db_import_file_test.go | 16 ++-- internal/cmd/group_flag.go | 1 - 5 files changed, 132 insertions(+), 44 deletions(-) diff --git a/go.mod b/go.mod index b9482b6f..1aebf94a 100644 --- a/go.mod +++ b/go.mod @@ -25,6 +25,7 @@ require ( github.com/stretchr/testify v1.11.1 golang.org/x/exp v0.0.0-20240716160929-1d5bc16f04a8 golang.org/x/sync v0.19.0 + turso.tech/database/tursogo v0.7.1 ) require ( @@ -34,11 +35,13 @@ require ( github.com/charmbracelet/x/cellbuf v0.0.13-0.20250311204145-2c3ea96c31dd // indirect github.com/charmbracelet/x/term v0.2.1 // indirect github.com/davecgh/go-spew v1.1.1 // indirect + github.com/ebitengine/purego v0.9.1 // indirect github.com/erikgeiser/coninput v0.0.0-20211004153227-1c3628e74d0f // indirect github.com/go-viper/mapstructure/v2 v2.4.0 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/sagikazarmark/locafero v0.11.0 // indirect github.com/sourcegraph/conc v0.3.1-0.20240121214520-5f936abd7ae8 // indirect + github.com/tursodatabase/turso-go-platform-libs v0.7.1 // indirect github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect go.yaml.in/yaml/v3 v3.0.4 // indirect ) @@ -58,7 +61,7 @@ require ( github.com/mattn/go-isatty v0.0.20 github.com/mattn/go-localereader v0.0.1 // indirect github.com/mattn/go-runewidth v0.0.16 // indirect - github.com/mattn/go-sqlite3 v1.14.16 // indirect + github.com/mattn/go-sqlite3 v1.14.42 // indirect github.com/muesli/ansi v0.0.0-20230316100256-276c6243b2f6 // indirect github.com/muesli/cancelreader v0.2.2 // indirect github.com/muesli/termenv v0.16.0 // indirect diff --git a/go.sum b/go.sum index 285fe41c..84176d47 100644 --- a/go.sum +++ b/go.sum @@ -43,6 +43,8 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= +github.com/ebitengine/purego v0.9.1 h1:a/k2f2HQU3Pi399RPW1MOaZyhKJL9w/xFpKAg4q1s0A= +github.com/ebitengine/purego v0.9.1/go.mod h1:iIjxzd6CiRiOG0UyXP+V1+jWqUXVjPKLAI0mRfJZTmQ= github.com/erikgeiser/coninput v0.0.0-20211004153227-1c3628e74d0f h1:Y/CXytFA4m6baUTXGLOoWe4PQhGxaX0KpnayAqC48p4= github.com/erikgeiser/coninput v0.0.0-20211004153227-1c3628e74d0f/go.mod h1:vw97MGsxSvLiUE2X8qFplwetxpGLQrlU1Q9AUEIzCaM= github.com/fatih/color v1.18.0 h1:S8gINlzdQ840/4pfAwic/ZE0djQEH3wM94VfqLTZcOM= @@ -113,8 +115,8 @@ github.com/mattn/go-runewidth v0.0.7/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= github.com/mattn/go-runewidth v0.0.16 h1:E5ScNMtiwvlvB5paMFdw9p4kSQzbXFikJ5SQO6TULQc= github.com/mattn/go-runewidth v0.0.16/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= -github.com/mattn/go-sqlite3 v1.14.16 h1:yOQRA0RpS5PFz/oikGwBEqvAWhWg5ufRz4ETLjwpU1Y= -github.com/mattn/go-sqlite3 v1.14.16/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S2DGjv9HUNg= +github.com/mattn/go-sqlite3 v1.14.42 h1:MigqEP4ZmHw3aIdIT7T+9TLa90Z6smwcthx+Azv4Cgo= +github.com/mattn/go-sqlite3 v1.14.42/go.mod h1:pjEuOr8IwzLJP2MfGeTb0A35jauH+C2kbHKBr7yXKVQ= github.com/mitchellh/copystructure v1.2.0 h1:vpKXTN4ewci03Vljg/q9QvCGUDttBOGBIa15WveJJGw= github.com/mitchellh/copystructure v1.2.0/go.mod h1:qLl+cE2AmVv+CoeAwDPye/v+N2HKCj9FbZEVFJRxO9s= github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= @@ -166,6 +168,8 @@ github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8 github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU= github.com/tursodatabase/libsql-client-go v0.0.0-20260514053736-a9a8fadfe885 h1:YssVXwM/9nUAjGNmUWdgvb05JVcsaBrDn5yr+MaJTn0= github.com/tursodatabase/libsql-client-go v0.0.0-20260514053736-a9a8fadfe885/go.mod h1:08inkKyguB6CGGssc/JzhmQWwBgFQBgjlYFjxjRh7nU= +github.com/tursodatabase/turso-go-platform-libs v0.7.1 h1:sQ2aupg/dhilMWRRhIIjd9w8ikue4vIYGxHFMtWhYZE= +github.com/tursodatabase/turso-go-platform-libs v0.7.1/go.mod h1:bo+Lpv5OYOX1gRV9L5DLKMsYxmDs56SkZwnCOLEFcxU= github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e h1:JVG44RsyaB9T2KIHavMF/ppJZNG9ZpyihvCd0w101no= github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e/go.mod h1:RbqR21r5mrJuqunuUZ/Dhy/avygyECGrLceyNeo4LiM= go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc= @@ -198,3 +202,5 @@ gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EV gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +turso.tech/database/tursogo v0.7.1 h1:xCVENss9TeC5dR2Xs0saZZpVg05+1hW9pL1ei8KSAQ8= +turso.tech/database/tursogo v0.7.1/go.mod h1:sDMlDKBFfVdU1zi3qOd+HqU0Nnu/84UYGR6sZGqBIAM= diff --git a/internal/cmd/db_import_file.go b/internal/cmd/db_import_file.go index c43c1c9f..bee68be5 100644 --- a/internal/cmd/db_import_file.go +++ b/internal/cmd/db_import_file.go @@ -2,6 +2,7 @@ package cmd import ( "bufio" + "database/sql" "errors" "fmt" "log" @@ -14,15 +15,41 @@ import ( "github.com/tursodatabase/turso-cli/internal/flags" "github.com/tursodatabase/turso-cli/internal/prompt" "github.com/tursodatabase/turso-cli/internal/turso" + _ "turso.tech/database/tursogo" ) const MaxAWSDBSizeBytes = 1024 * 1024 * 1024 * 20 // 20 GB const databaseSettingsQuery = "select journal_mode as j, page_size as p, auto_vacuum as a, encoding as e from pragma_journal_mode, pragma_page_size, pragma_auto_vacuum, pragma_encoding;" +type databaseEngine uint8 + +const ( + databaseEngineUnknown databaseEngine = iota + databaseEngineSQLite + databaseEngineTursoDB +) + +func (e databaseEngine) name() string { + switch e { + case databaseEngineSQLite: + return "SQLite" + case databaseEngineTursoDB: + return "TursoDB" + default: + return "unknown" + } +} + +func (e databaseEngine) settingsCommand() (string, bool) { + if e == databaseEngineSQLite { + return "sqlite3", true + } + return "", false +} + type databaseFileChecker struct { - name string - binary string + engine databaseEngine journalMode string settings func(string) (databaseSettings, error) quickCheck func(string) error @@ -77,8 +104,7 @@ func validateDatabaseFileSize(file string) error { func sqliteFileIntegrityChecks(file string, cipher string) error { return databaseFileIntegrityChecks(file, cipher, databaseFileChecker{ - name: "SQLite", - binary: "sqlite3", + engine: databaseEngineSQLite, journalMode: "wal", settings: sqliteDatabaseSettings, quickCheck: runQuickCheck, @@ -87,8 +113,7 @@ func sqliteFileIntegrityChecks(file string, cipher string) error { func tursoDBFileIntegrityChecks(file string) error { return databaseFileIntegrityChecks(file, "", databaseFileChecker{ - name: "TursoDB", - binary: "tursodb", + engine: databaseEngineTursoDB, journalMode: "mvcc", settings: tursoDBDatabaseSettings, quickCheck: runTursoDBQuickCheck, @@ -97,7 +122,7 @@ func tursoDBFileIntegrityChecks(file string) error { func databaseFileIntegrityChecks(file, cipher string, checker databaseFileChecker) error { if flags.Debug() { - log.Printf("Running %s integrity checks on database file %s", checker.name, file) + log.Printf("Running %s integrity checks on database file %s", checker.engine.name(), file) log.Printf("Checking database settings...") } @@ -134,17 +159,30 @@ func databaseFileIntegrityChecks(file, cipher string, checker databaseFileChecke } func validateDatabaseSettings(file string, settings databaseSettings, checker databaseFileChecker) error { + settingsCommand, hasSettingsCommand := checker.engine.settingsCommand() if !strings.EqualFold(settings.journalMode, checker.journalMode) { - return fmt.Errorf("database is not in %s mode. Set it with '%s %s \"PRAGMA journal_mode = %s;\"'", strings.ToUpper(checker.journalMode), checker.binary, file, strings.ToUpper(checker.journalMode)) + if !hasSettingsCommand { + return fmt.Errorf("database is not in %s mode", strings.ToUpper(checker.journalMode)) + } + return fmt.Errorf("database is not in %s mode. Set it with '%s %s \"PRAGMA journal_mode = %s;\"'", strings.ToUpper(checker.journalMode), settingsCommand, file, strings.ToUpper(checker.journalMode)) } if settings.pageSize != "4096" { - return fmt.Errorf("database must use 4KB page size. You can set it with '%s %s \"PRAGMA page_size = 4096; VACUUM;\"'", checker.binary, file) + if !hasSettingsCommand { + return errors.New("database must use 4KB page size") + } + return fmt.Errorf("database must use 4KB page size. You can set it with '%s %s \"PRAGMA page_size = 4096; VACUUM;\"'", settingsCommand, file) } if settings.autoVacuum != "0" { - return fmt.Errorf("database must have autovacuum disabled. You can set it with '%s %s \"PRAGMA auto_vacuum = 0;\"'", checker.binary, file) + if !hasSettingsCommand { + return errors.New("database must have autovacuum disabled") + } + return fmt.Errorf("database must have autovacuum disabled. You can set it with '%s %s \"PRAGMA auto_vacuum = 0;\"'", settingsCommand, file) } if !strings.EqualFold(settings.encoding, "UTF-8") { - return fmt.Errorf("database must use UTF-8 encoding. You can set it with '%s %s \"PRAGMA encoding = 'UTF-8';\"'", checker.binary, file) + if !hasSettingsCommand { + return errors.New("database must use UTF-8 encoding") + } + return fmt.Errorf("database must use UTF-8 encoding. You can set it with '%s %s \"PRAGMA encoding = 'UTF-8';\"'", settingsCommand, file) } return nil } @@ -175,17 +213,38 @@ func sqliteDatabaseSettings(file string) (databaseSettings, error) { } func tursoDBDatabaseSettings(file string) (databaseSettings, error) { - output, err := exec.Command("tursodb", "-q", "-m", "list", file, databaseSettingsQuery).CombinedOutput() + db, err := openTursoDB(file) if err != nil { - return databaseSettings{}, fmt.Errorf("failed to check database settings with TursoDB: %w: %s", err, strings.TrimSpace(string(output))) + return databaseSettings{}, fmt.Errorf("failed to check database settings with TursoDB: %w", err) } - settings, err := parseDatabaseSettings(string(output)) + defer db.Close() + + var settings databaseSettings + err = db.QueryRow(databaseSettingsQuery).Scan( + &settings.journalMode, + &settings.pageSize, + &settings.autoVacuum, + &settings.encoding, + ) if err != nil { - return databaseSettings{}, fmt.Errorf("failed to parse database settings from TursoDB: %w", err) + return databaseSettings{}, fmt.Errorf("failed to query database settings with TursoDB: %w", err) } return settings, nil } +func openTursoDB(file string) (*sql.DB, error) { + db, err := sql.Open("turso", file) + if err != nil { + return nil, err + } + db.SetMaxOpenConns(1) + if err := db.Ping(); err != nil { + db.Close() + return nil, err + } + return db, nil +} + func runQuickCheck(file string) error { cmd := exec.Command("sqlite3", "-list", file, "pragma quick_check;") if err := cmd.Run(); err != nil { @@ -195,13 +254,34 @@ func runQuickCheck(file string) error { } func runTursoDBQuickCheck(file string) error { - output, err := exec.Command("tursodb", "-q", "-m", "list", file, "PRAGMA quick_check;").CombinedOutput() + db, err := openTursoDB(file) + if err != nil { + return fmt.Errorf("TursoDB integrity check failed for %s: %w", file, err) + } + defer db.Close() + + rows, err := db.Query("PRAGMA quick_check;") if err != nil { - return fmt.Errorf("TursoDB integrity check failed for %s: %w: %s", file, err, strings.TrimSpace(string(output))) + return fmt.Errorf("TursoDB integrity check failed for %s: %w", file, err) } - fields := strings.Fields(string(output)) - if len(fields) == 0 || fields[len(fields)-1] != "ok" { - return fmt.Errorf("TursoDB integrity check failed for %s: %s", file, strings.TrimSpace(string(output))) + defer rows.Close() + + checked := false + for rows.Next() { + var result string + if err := rows.Scan(&result); err != nil { + return fmt.Errorf("TursoDB integrity check failed for %s: %w", file, err) + } + checked = true + if result != "ok" { + return fmt.Errorf("TursoDB integrity check failed for %s: %s", file, result) + } + } + if err := rows.Err(); err != nil { + return fmt.Errorf("TursoDB integrity check failed for %s: %w", file, err) + } + if !checked { + return fmt.Errorf("TursoDB integrity check failed for %s: no result", file) } return nil } @@ -222,10 +302,17 @@ func checkpointWALBeforeUpload(file string) error { } func prepareTursoDBFile(file string) error { - output, err := exec.Command("tursodb", "-q", "-m", "list", file, - "PRAGMA journal_mode = mvcc; PRAGMA wal_checkpoint(TRUNCATE);").CombinedOutput() + db, err := openTursoDB(file) if err != nil { - return fmt.Errorf("could not prepare %s for TursoDB import: %w: %s", file, err, strings.TrimSpace(string(output))) + return fmt.Errorf("could not prepare %s for TursoDB import: %w", file, err) + } + _, execErr := db.Exec("PRAGMA journal_mode = mvcc; PRAGMA wal_checkpoint(TRUNCATE);") + closeErr := db.Close() + if execErr != nil { + return fmt.Errorf("could not prepare %s for TursoDB import: %w", file, execErr) + } + if closeErr != nil { + return fmt.Errorf("could not close %s after preparing it for TursoDB import: %w", file, closeErr) } format, err := sniffSQLiteFileFormat(file) @@ -322,9 +409,6 @@ func handleDBFileAWS(file string, cipher string) (*turso.DBSeed, error) { } if tursoDBFlag { - if err := checkTursoDBAvailable(); err != nil { - return nil, err - } if format != fileFormatMVCC { fmt.Printf("Converting %s to TursoDB (MVCC) format for import.\n", file) } @@ -385,11 +469,3 @@ func validateReservedBytes(dbPath string, cipher string) error { } return nil } - -func checkTursoDBAvailable() error { - _, err := exec.LookPath("tursodb") - if errors.Is(err, exec.ErrNotFound) { - return errors.New("could not find tursodb on your system. Please install it to import into a TursoDB database") - } - return err -} diff --git a/internal/cmd/db_import_file_test.go b/internal/cmd/db_import_file_test.go index eec26b59..26f56e38 100644 --- a/internal/cmd/db_import_file_test.go +++ b/internal/cmd/db_import_file_test.go @@ -72,7 +72,7 @@ func TestRunQuickCheck(t *testing.T) { func TestValidateDatabaseSettings(t *testing.T) { checker := databaseFileChecker{ - binary: "tursodb", + engine: databaseEngineTursoDB, journalMode: "mvcc", } settings := databaseSettings{ @@ -84,7 +84,14 @@ func TestValidateDatabaseSettings(t *testing.T) { require.NoError(t, validateDatabaseSettings("data.db", settings, checker)) settings.journalMode = "WAL" - require.ErrorContains(t, validateDatabaseSettings("data.db", settings, checker), "not in MVCC mode") + err := validateDatabaseSettings("data.db", settings, checker) + require.ErrorContains(t, err, "not in MVCC mode") + require.NotContains(t, err.Error(), "tursodb") + + checker.engine = databaseEngineSQLite + checker.journalMode = "wal" + settings.journalMode = "MVCC" + require.ErrorContains(t, validateDatabaseSettings("data.db", settings, checker), "sqlite3") } func TestHandleDBFileAWSRejectsUnknownFormat(t *testing.T) { @@ -140,12 +147,9 @@ func TestCheckpointWALBeforeUpload(t *testing.T) { } func TestPrepareTursoDBFile(t *testing.T) { - if _, err := exec.LookPath("tursodb"); err != nil { - t.Skip("tursodb not available, skipping test") - } - dbPath := createTestDatabase(t, 10*1024) require.NoError(t, checkpointWALBeforeUpload(dbPath)) + t.Setenv("PATH", t.TempDir()) // TursoDB preparation must not require an external binary. require.NoError(t, prepareTursoDBFile(dbPath)) require.NoError(t, tursoDBFileIntegrityChecks(dbPath)) require.NoError(t, checkTursoDBSidecars(dbPath)) diff --git a/internal/cmd/group_flag.go b/internal/cmd/group_flag.go index 20954fe5..c180ae69 100644 --- a/internal/cmd/group_flag.go +++ b/internal/cmd/group_flag.go @@ -7,7 +7,6 @@ import ( "fmt" "os" "os/exec" - "strings" "time" "github.com/Clever/csvlint"