Merge pull request #935 from tursodatabase/aws-upload-v2

Turso CLI database upload flow for AWS

Author: jussisaurio

internal/cmd/db_create.go

@@ -2,6 +2,7 @@ package cmd
 
 import (
 	"fmt"
+	"strings"
 	"time"
 
 	"github.com/athoscouto/codename"
@@ -62,7 +63,7 @@ var createCmd = &cobra.Command{
 			return err
 		}
 
-		group, err := groupFromFlag(client)
+		group, isDefault, err := groupFromFlag(client)
 		if err != nil {
 			return err
 		}
@@ -72,7 +73,8 @@ var createCmd = &cobra.Command{
 			return err
 		}
 
-		seed, err := parseDBSeedFlags(client)
+		isAWS := strings.HasPrefix(group.Primary, "aws-")
+		seed, err := parseDBSeedFlags(client, isAWS)
 		if err != nil {
 			return err
 		}
@@ -82,21 +84,26 @@ var createCmd = &cobra.Command{
 			version = "canary"
 		}
 
-		if err := ensureGroup(client, group, location, version); err != nil {
+		groupName := group.Name
+		if isDefault {
+			groupName = "default"
+		}
+
+		if err := ensureGroup(client, groupName, location, version); err != nil {
 			return err
 		}
 
 		start := time.Now()
-		spinner := prompt.Spinner(fmt.Sprintf("Creating database %s in group %s...", internal.Emph(name), internal.Emph(group)))
+		spinner := prompt.Spinner(fmt.Sprintf("Creating database %s in group %s...", internal.Emph(name), internal.Emph(groupName)))
 		defer spinner.Stop()
 
-		if _, err = client.Databases.Create(name, location, "", "", group, schemaFlag, typeFlag == "schema", seed, sizeLimitFlag); err != nil {
+		if _, err = client.Databases.Create(name, location, "", "", groupName, schemaFlag, typeFlag == "schema", seed, sizeLimitFlag, spinner); err != nil {
 			return fmt.Errorf("could not create database %s: %w", name, err)
 		}
 
 		spinner.Stop()
 		elapsed := time.Since(start)
-		fmt.Printf("Created database %s at group %s in %s.\n\n", internal.Emph(name), internal.Emph(group), elapsed.Round(time.Millisecond).String())
+		fmt.Printf("Created database %s at group %s in %s.\n\n", internal.Emph(name), internal.Emph(groupName), elapsed.Round(time.Millisecond).String())
 
 		fmt.Printf("Start an interactive SQL shell with:\n\n")
 		fmt.Printf("   %s\n\n", internal.Emph("turso db shell "+name))
@@ -131,26 +138,32 @@ func getDatabaseName(args []string) (string, error) {
 	return codename.Generate(rng, 0), nil
 }
 
-func groupFromFlag(client *turso.Client) (string, error) {
+// Returns (group, isDefault, error)
+func groupFromFlag(client *turso.Client) (turso.Group, bool, error) {
 	groups, err := getGroups(client)
 	if err != nil {
-		return "", err
+		return turso.Group{}, false, err
 	}
 
 	if groupFlag != "" {
 		if !groupExists(groups, groupFlag) {
-			return "", fmt.Errorf("group %s does not exist", groupFlag)
+			return turso.Group{}, false, fmt.Errorf("group %s does not exist", groupFlag)
+		}
+		for _, group := range groups {
+			if group.Name == groupFlag {
+				return group, false, nil
+			}
 		}
-		return groupFlag, nil
+		return turso.Group{}, false, fmt.Errorf("group %s does not exist", groupFlag)
 	}
 
 	switch {
 	case len(groups) == 0:
-		return "default", nil
+		return turso.Group{Name: "default"}, true, nil
 	case len(groups) == 1:
-		return groups[0].Name, nil
+		return groups[0], true, nil
 	default:
-		return "", fmt.Errorf("you have more than one database group. Please specify one with %s", internal.Emph("--group"))
+		return turso.Group{}, false, fmt.Errorf("you have more than one database group. Please specify one with %s", internal.Emph("--group"))
 
 	}
 }

internal/cmd/group_flag.go

@@ -5,8 +5,10 @@ import (
 	"bytes"
 	"errors"
 	"fmt"
+	"log"
 	"os"
 	"os/exec"
+	"strings"
 	"time"
 
 	"github.com/Clever/csvlint"
@@ -62,7 +64,7 @@ func parseTimestampFlag() (*time.Time, error) {
 	return &timestamp, nil
 }
 
-func parseDBSeedFlags(client *turso.Client) (*turso.DBSeed, error) {
+func parseDBSeedFlags(client *turso.Client, isAWS bool) (*turso.DBSeed, error) {
 	if countFlags(fromDBFlag, fromDumpFlag, fromFileFlag, fromDumpURLFlag, fromCSVFlag) > 1 {
 		return nil, fmt.Errorf("only one of --from prefixed flags can be used at a time")
 	}
@@ -83,7 +85,7 @@ func parseDBSeedFlags(client *turso.Client) (*turso.DBSeed, error) {
 	}
 
 	if fromFileFlag != "" {
-		return handleDBFile(client, fromFileFlag)
+		return handleDBFile(client, fromFileFlag, isAWS)
 	}
 
 	if fromDumpFlag != "" {
@@ -187,14 +189,84 @@ func countFlags(flags ...string) (count int) {
 	return
 }
 
-func handleDBFile(client *turso.Client, file string) (*turso.DBSeed, error) {
+const MaxAWSDBSizeBytes = 1024 * 1024 * 1024 * 20 // 20 GB
+func sqliteFileIntegrityChecks(file string) error {
+	if flags.Debug() {
+		log.Printf("Running integrity checks on database file %s", file)
+	}
+	if flags.Debug() {
+		log.Printf("Checking file size...")
+	}
+	fileInfo, err := os.Stat(file)
+	if err != nil {
+		return fmt.Errorf("failed to get file info: %w", err)
+	}
+
+	if fileInfo.Size() > MaxAWSDBSizeBytes {
+		return fmt.Errorf("database file size exceeds maximum allowed size of 20 GB")
+	}
+
+	if flags.Debug() {
+		log.Printf("Checking database settings...")
+	}
+	output, err := exec.Command("sqlite3", file, ".mode line",
+		"select journal_mode as j, page_size as p, auto_vacuum as a, encoding as e from pragma_journal_mode, pragma_page_size, pragma_auto_vacuum, pragma_encoding;").CombinedOutput()
+	if err != nil {
+		return fmt.Errorf("failed to check database settings: %w", err)
+	}
+
+	settings := string(output)
+	if !strings.Contains(settings, "j = wal") {
+		return fmt.Errorf("database is not in WAL mode. Set it with 'sqlite3 %s 'PRAGMA journal_mode = WAL'", file)
+	}
+
+	if !strings.Contains(settings, "p = 4096") {
+		return fmt.Errorf("database must use 4KB page size. you can set it with 'sqlite3 %s 'PRAGMA page_size = 4096; VACUUM;' Note that this is not possible to do if your database is already in WAL mode", file)
+	}
+	if !strings.Contains(settings, "a = 0") {
+		return fmt.Errorf("database must have autovacuum disabled. you can set it with 'sqlite3 %s 'PRAGMA auto_vacuum = 0;'", file)
+	}
+	if !strings.Contains(settings, "e = UTF-8") {
+		return fmt.Errorf("database must use UTF-8 encoding. you can set it with 'sqlite3 %s 'PRAGMA encoding = 'UTF-8'	", file)
+	}
+
+	// run quick_check
+	if flags.Debug() {
+		log.Printf("Running integrity check...")
+	}
+	_, err = exec.Command("sqlite3", file, "pragma quick_check;").CombinedOutput()
+	if err != nil {
+		return fmt.Errorf("integrity check on database failed: %w", err)
+	}
+
+	return nil
+}
+
+func handleDBFileAWS(file string) (*turso.DBSeed, error) {
+	if err := sqliteFileIntegrityChecks(file); err != nil {
+		return nil, err
+	}
+
+	seed := &turso.DBSeed{
+		Type:     "database_upload",
+		Filepath: file,
+	}
+
+	return seed, nil
+}
+
+func handleDBFile(client *turso.Client, file string, isAWS bool) (*turso.DBSeed, error) {
 	if err := checkFileExists(file); err != nil {
 		return nil, err
 	}
 	if err := checkSQLiteAvailable(); err != nil {
 		return nil, err
 	}
 
+	if isAWS {
+		return handleDBFileAWS(file)
+	}
+
 	if err := checkSQLiteFile(file); err != nil {
 		return nil, err
 	}
@@ -292,7 +364,7 @@ func handleCSVFile(client *turso.Client, file, csvTableName string, separator ru
 		return nil, err
 	}
 
-	seed, err := handleDBFile(client, tempDB.Name())
+	seed, err := handleDBFile(client, tempDB.Name(), false)
 	if err != nil {
 		return nil, err
 	}

internal/prompt/spinner.go

@@ -8,7 +8,7 @@ import (
 	tea "github.com/charmbracelet/bubbletea"
 )
 
-type spinner struct {
+type SpinnerT struct {
 	spinner   spn.Model
 	prefix    string
 	suffix    string
@@ -17,17 +17,17 @@ type spinner struct {
 	done      chan bool
 }
 
-func newSpinner(prefix, suffix string) *spinner {
+func newSpinner(prefix, suffix string) *SpinnerT {
 	s := spn.New()
 	s.Spinner = spn.Dot
-	return &spinner{spinner: s, prefix: prefix, suffix: suffix}
+	return &SpinnerT{spinner: s, prefix: prefix, suffix: suffix}
 }
 
-func (m *spinner) Init() tea.Cmd {
+func (m *SpinnerT) Init() tea.Cmd {
 	return m.spinner.Tick
 }
 
-func (m *spinner) Update(msg tea.Msg) (tea.Model, tea.Cmd) {
+func (m *SpinnerT) Update(msg tea.Msg) (tea.Model, tea.Cmd) {
 	if m.quitting {
 		return m, tea.Quit
 	}
@@ -48,25 +48,25 @@ func (m *spinner) Update(msg tea.Msg) (tea.Model, tea.Cmd) {
 	}
 }
 
-func (m *spinner) View() string {
+func (m *SpinnerT) View() string {
 	if m.quitting {
 		return ""
 	}
 	return fmt.Sprintf("%s%s %s", m.prefix, m.spinner.View(), m.suffix)
 }
 
-func (m *spinner) Stop() {
+func (m *SpinnerT) Stop() {
 	m.quitting = true
 	if m.done != nil {
 		<-m.done
 	}
 }
 
-func (m *spinner) Text(t string) {
+func (m *SpinnerT) Text(t string) {
 	m.suffix = t
 }
 
-func (m *spinner) Start() {
+func (m *SpinnerT) Start() {
 	if !isInteractive {
 		fmt.Println(m.View())
 		return
@@ -85,12 +85,12 @@ func (m *spinner) Start() {
 	}()
 }
 
-func StoppedSpinner(text string) *spinner {
+func StoppedSpinner(text string) *SpinnerT {
 	spinner := newSpinner("", text)
 	return spinner
 }
 
-func Spinner(text string) *spinner {
+func Spinner(text string) *SpinnerT {
 	spinner := StoppedSpinner(text)
 	spinner.Start()
 	return spinner