Support TursoDB database imports

internal/cmd/db_create.go

@@ -155,7 +155,12 @@ func createDatabase(client *turso.Client, name, location, groupName string, seed
 	if sizeLimitFlag != "" {
 		return createDatabaseV2(client, name, location, groupName, seed, spinner)
 	}
-	if seed != nil && seed.Type != "database" && seed.Type != "upload" {
+	// Only fork seeds ("database") can go through the v3 API. File seeds
+	// ("database_upload") must use the v2 flow: the v3 branch never uploads
+	// the file after creating the database. (This used to compare against
+	// "upload", a seed type that doesn't exist, which routed uploads to v2
+	// by accident.)
+	if seed != nil && seed.Type != "database" {
 		return createDatabaseV2(client, name, location, groupName, seed, spinner)
 	}
 	orgID, err := tryResolveOrgID(client)

internal/cmd/db_import.go

@@ -9,11 +9,16 @@ import (
 	"github.com/spf13/cobra"
 )
 
+// Import keeps its own flag value because it delegates database creation to
+// CreateDatabase; RunE copies the selected value to the shared create flag.
+var importTursoDBFlag bool
+
 func init() {
 	dbCmd.AddCommand(importCmd)
 	addGroupFlag(importCmd)
 	addRemoteEncryptionKeyFlag(importCmd)
 	addRemoteEncryptionCipherFlag(importCmd)
+	importCmd.Flags().BoolVar(&importTursoDBFlag, "tursodb", false, "Import into a TursoDB (MVCC) database.")
 }
 
 var importCmd = &cobra.Command{
@@ -43,6 +48,7 @@ var importCmd = &cobra.Command{
 		}
 
 		fromFileFlag = filename
+		tursoDBFlag = importTursoDBFlag
 		name := sanitizeDatabaseName(filename)
 		return CreateDatabase(name)
 	},

internal/cmd/group_flag.go

@@ -8,6 +8,7 @@ import (
 	"log"
 	"os"
 	"os/exec"
+	"path/filepath"
 	"strconv"
 	"strings"
 	"time"
@@ -340,11 +341,158 @@ func runQuickCheck(file string) error {
 	return nil
 }
 
+// checkpointWALBeforeUpload folds any pending WAL frames into the main
+// database file and verifies no data is left behind in sidecar files. The
+// upload ships only the main database file, so frames still sitting in
+// <file>-wal (or a hot rollback journal) would silently be missing from the
+// imported database.
+func checkpointWALBeforeUpload(file string) error {
+	if out, err := exec.Command("sqlite3", "-list", file, "PRAGMA wal_checkpoint(TRUNCATE);").CombinedOutput(); err != nil {
+		return fmt.Errorf("could not checkpoint database %s: %w: %s", file, err, out)
+	}
+	for _, sidecar := range []struct{ suffix, hint string }{
+		{"-wal", "close all connections to the database and retry the import"},
+		{"-journal", "the database has a leftover rollback journal; open and cleanly close it with sqlite3 first"},
+	} {
+		if err := checkSidecarEmpty(file+sidecar.suffix, sidecar.hint); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+// prepareTursoDBFile asks the TursoDB engine to convert the database to MVCC,
+// checkpoint any logical-log entries into the main file, and validate the
+// resulting database. Uploading only the main file is safe once all data-bearing
+// sidecars are empty.
+func prepareTursoDBFile(file string) error {
+	output, err := exec.Command("tursodb", "-q", "-m", "list", file,
+		"PRAGMA journal_mode = mvcc; PRAGMA wal_checkpoint(TRUNCATE); PRAGMA quick_check;").CombinedOutput()
+	if err != nil {
+		return fmt.Errorf("could not prepare %s for TursoDB import: %w: %s", file, err, strings.TrimSpace(string(output)))
+	}
+
+	lines := strings.Fields(string(output))
+	if len(lines) == 0 || lines[len(lines)-1] != "ok" {
+		return fmt.Errorf("TursoDB integrity check failed for %s: %s", file, strings.TrimSpace(string(output)))
+	}
+
+	format, err := sniffSQLiteFileFormat(file)
+	if err != nil {
+		return err
+	}
+	if format != fileFormatMVCC {
+		return fmt.Errorf("TursoDB did not convert %s to MVCC format", file)
+	}
+
+	for _, sidecar := range []struct{ path, hint string }{
+		{file + "-wal", "close all connections to the database and retry the import"},
+		{file + "-journal", "the database has a leftover rollback journal; close it cleanly and retry the import"},
+		{tursodbLogPath(file), "close all TursoDB connections and retry the import"},
+	} {
+		if err := checkSidecarEmpty(sidecar.path, sidecar.hint); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func checkSidecarEmpty(sidecarPath, hint string) error {
+	info, err := os.Stat(sidecarPath)
+	if errors.Is(err, os.ErrNotExist) {
+		return nil
+	}
+	if err != nil {
+		return fmt.Errorf("could not check %s: %w", sidecarPath, err)
+	}
+	if info.Size() > 0 {
+		return fmt.Errorf("%s is not empty, importing would lose the data it holds: %s", sidecarPath, hint)
+	}
+	return nil
+}
+
+// tursodbLogPath returns the logical-log sidecar path tursodb uses for a
+// database file, mirroring turso_core's `with_extension("db-log")`: the
+// file's extension (if any) is replaced with "db-log".
+func tursodbLogPath(file string) string {
+	return strings.TrimSuffix(file, filepath.Ext(file)) + ".db-log"
+}
+
 func handleDBFileAWS(file string, cipher string) (*turso.DBSeed, error) {
-	if err := sqliteFileIntegrityChecks(file, cipher); err != nil {
+	format, err := sniffSQLiteFileFormat(file)
+	if err != nil {
 		return nil, err
 	}
 
+	if format == fileFormatRollback {
+		if err := checkSQLiteAvailable(); err != nil {
+			return nil, err
+		}
+		// The server only accepts WAL or MVCC format files. Converting to WAL
+		// is exactly the remediation the error message used to instruct users
+		// to run themselves, and sqlite3 is already a hard requirement here.
+		fmt.Printf("File %s uses a rollback journal; converting it to WAL mode for import.\n", file)
+		if out, err := exec.Command("sqlite3", file, "PRAGMA journal_mode=WAL;").CombinedOutput(); err != nil {
+			return nil, fmt.Errorf("could not convert %s to WAL mode: %w: %s", file, err, out)
+		}
+		if format, err = sniffSQLiteFileFormat(file); err != nil {
+			return nil, err
+		}
+	}
+
+	switch format {
+	case fileFormatWAL:
+		if err := checkSQLiteAvailable(); err != nil {
+			return nil, err
+		}
+		if err := checkpointWALBeforeUpload(file); err != nil {
+			return nil, err
+		}
+		if err := sqliteFileIntegrityChecks(file, cipher); err != nil {
+			return nil, err
+		}
+	case fileFormatMVCC:
+		// sqlite3-based checks can't run on MVCC (tursodb format) files: the
+		// sqlite3 binary reports them as not-a-database.
+		if !tursoDBFlag {
+			return nil, fmt.Errorf("%s is in tursodb (MVCC) format and can only be imported into a tursodb database", file)
+		}
+		if cipher != "" {
+			return nil, errors.New("remote encryption is not supported when importing tursodb (MVCC) format files")
+		}
+	case fileFormatNotSQLite:
+		isDump, err := checkIfDump(file)
+		if err != nil {
+			return nil, fmt.Errorf("failed to get file header: %w", err)
+		}
+		if isDump {
+			return nil, fmt.Errorf("%s is a sqlite3 dump, not a sqlite3 database. Please import a sqlite database", file)
+		}
+		return nil, fmt.Errorf("file %s is not a valid SQLite database file", file)
+	default:
+		return nil, fmt.Errorf("file %s has an unsupported SQLite file format", file)
+	}
+
+	if tursoDBFlag {
+		if err := checkTursoDBAvailable(); err != nil {
+			return nil, err
+		}
+		if format != fileFormatMVCC {
+			fmt.Printf("Converting %s to TursoDB (MVCC) format for import.\n", file)
+		}
+		if err := prepareTursoDBFile(file); err != nil {
+			return nil, err
+		}
+	}
+
+	fileInfo, err := os.Stat(file)
+	if err != nil {
+		return nil, fmt.Errorf("failed to get file info: %w", err)
+	}
+	if fileInfo.Size() > MaxAWSDBSizeBytes {
+		return nil, errors.New("database file size exceeds maximum allowed size of 20 GB")
+	}
+
 	seed := &turso.DBSeed{
 		Type:     "database_upload",
 		Filepath: file,
@@ -357,14 +505,24 @@ func handleDBFile(client *turso.Client, file string, isAWS bool, cipher string)
 	if err := checkFileExists(file); err != nil {
 		return nil, err
 	}
-	if err := checkSQLiteAvailable(); err != nil {
-		return nil, err
-	}
 
 	if isAWS {
 		return handleDBFileAWS(file, cipher)
 	}
 
+	format, err := sniffSQLiteFileFormat(file)
+	if err != nil {
+		return nil, err
+	}
+	if format == fileFormatMVCC {
+		// non-AWS groups are seeded by replaying a .dump, which sqlite3 cannot
+		// produce from an MVCC (tursodb format) file
+		return nil, fmt.Errorf("%s is in tursodb (MVCC) format and can only be imported into AWS groups", file)
+	}
+	if err := checkSQLiteAvailable(); err != nil {
+		return nil, err
+	}
+
 	if err := checkSQLiteFile(file); err != nil {
 		return nil, err
 	}
@@ -397,6 +555,14 @@ func checkSQLiteAvailable() error {
 	return err
 }
 
+func checkTursoDBAvailable() error {
+	_, err := exec.LookPath("tursodb")
+	if errors.Is(err, exec.ErrNotFound) {
+		return errors.New("could not find tursodb on your system. Please install it to import into a TursoDB database")
+	}
+	return err
+}
+
 func checkSQLiteFile(file string) error {
 	output, err := exec.Command("sqlite3", "-list", file, "pragma quick_check;").CombinedOutput()
 

internal/cmd/group_flag_test.go

@@ -74,3 +74,68 @@ func TestRunQuickCheck(t *testing.T) {
 		require.Error(t, err)
 	})
 }
+
+func TestTursodbLogPath(t *testing.T) {
+	require.Equal(t, "data.db-log", tursodbLogPath("data.db"))
+	require.Equal(t, "data.db-log", tursodbLogPath("data.sqlite"))
+	require.Equal(t, "/some/dir/mydb.db-log", tursodbLogPath("/some/dir/mydb.db"))
+	require.Equal(t, "noext.db-log", tursodbLogPath("noext"))
+}
+
+func TestCheckSidecarEmpty(t *testing.T) {
+	dir := t.TempDir()
+
+	t.Run("missing sidecar is fine", func(t *testing.T) {
+		require.NoError(t, checkSidecarEmpty(filepath.Join(dir, "missing-wal"), "hint"))
+	})
+
+	t.Run("empty sidecar is fine", func(t *testing.T) {
+		path := filepath.Join(dir, "empty-wal")
+		require.NoError(t, os.WriteFile(path, nil, 0644))
+		require.NoError(t, checkSidecarEmpty(path, "hint"))
+	})
+
+	t.Run("non-empty sidecar errors with hint", func(t *testing.T) {
+		path := filepath.Join(dir, "full-wal")
+		require.NoError(t, os.WriteFile(path, []byte("frames"), 0644))
+		err := checkSidecarEmpty(path, "close all connections")
+		require.Error(t, err)
+		require.Contains(t, err.Error(), "importing would lose the data it holds")
+		require.Contains(t, err.Error(), "close all connections")
+	})
+}
+
+func TestCheckpointWALBeforeUpload(t *testing.T) {
+	if _, err := exec.LookPath("sqlite3"); err != nil {
+		t.Skip("sqlite3 not available, skipping test")
+	}
+
+	dbPath := createTestDatabase(t, 10*1024)
+	require.NoError(t, checkpointWALBeforeUpload(dbPath))
+
+	// after the checkpoint no data-bearing sidecars may remain
+	for _, suffix := range []string{"-wal", "-journal"} {
+		if info, err := os.Stat(dbPath + suffix); err == nil {
+			require.Zero(t, info.Size(), "%s must be empty after checkpoint", dbPath+suffix)
+		}
+	}
+}
+
+func TestPrepareTursoDBFile(t *testing.T) {
+	if _, err := exec.LookPath("tursodb"); err != nil {
+		t.Skip("tursodb not available, skipping test")
+	}
+
+	dbPath := createTestDatabase(t, 10*1024)
+	require.NoError(t, checkpointWALBeforeUpload(dbPath))
+	require.NoError(t, prepareTursoDBFile(dbPath))
+
+	format, err := sniffSQLiteFileFormat(dbPath)
+	require.NoError(t, err)
+	require.Equal(t, fileFormatMVCC, format)
+	for _, sidecar := range []string{dbPath + "-wal", dbPath + "-journal", tursodbLogPath(dbPath)} {
+		if info, err := os.Stat(sidecar); err == nil {
+			require.Zero(t, info.Size(), "%s must be empty after checkpoint", sidecar)
+		}
+	}
+}

internal/cmd/sqlite_header.go

@@ -0,0 +1,52 @@
+package cmd
+
+import (
+	"io"
+	"os"
+)
+
+type sqliteFileFormat int
+
+const (
+	fileFormatNotSQLite sqliteFileFormat = iota
+	fileFormatRollback
+	fileFormatWAL
+	fileFormatMVCC
+	fileFormatUnknown
+)
+
+const sqliteMagic = "SQLite format 3\x00"
+
+// sniffSQLiteFileFormat classifies a database file by its SQLite header: the
+// 16-byte magic string plus the read/write format version bytes at offsets
+// 18/19 (1 = rollback journal, 2 = WAL, 255 = MVCC, i.e. tursodb format).
+//
+// This must run before any sqlite3 shellout: the sqlite3 binary reports
+// MVCC-format files as not-a-database, so they have to be routed around the
+// sqlite3-based checks entirely.
+func sniffSQLiteFileFormat(path string) (sqliteFileFormat, error) {
+	file, err := os.Open(path)
+	if err != nil {
+		return fileFormatNotSQLite, err
+	}
+	defer file.Close()
+
+	header := make([]byte, 20)
+	if _, err := io.ReadFull(file, header); err != nil {
+		// too short to hold a SQLite header
+		return fileFormatNotSQLite, nil
+	}
+	if string(header[:len(sqliteMagic)]) != sqliteMagic {
+		return fileFormatNotSQLite, nil
+	}
+	readVersion, writeVersion := header[18], header[19]
+	switch {
+	case readVersion == 1 && writeVersion == 1:
+		return fileFormatRollback, nil
+	case readVersion == 2 && writeVersion == 2:
+		return fileFormatWAL, nil
+	case readVersion == 255 && writeVersion == 255:
+		return fileFormatMVCC, nil
+	}
+	return fileFormatUnknown, nil
+}