Various upload improvements (#1012)

## Summary

This PR contains various improvements to the multipart upload flow. I
realize that it's a lot in the same PR, sorry about that.

* always use multipart upload
* retry chunks when they fail. I expect that this will increase
stability compared to the previous flow.
* use the new endpoints, since there was a breaking change (there's an
upload_id that's now exposed)
* refresh token between each chunk and before `/finalize`. This was
necessary because the new flow can include several requests over more
than an hour.
* Show a spinner with the DB size during validation. The first thing we
do when we start an upload is that we check that the DB is valid. For
large uploads, this could make it look like the CLI is hanging.
* Add debug logging for multipart upload, using the
`TURSO_DEBUG_UPLOAD=1` environment variable. I wasn't able to use the
existing `--debug` flag, because that one logs the bodies of the PUT
requests, and they can contain control characters that will wreck your
terminal.
* Update the progress tracker at least every 2 seconds. Previously, it
was updated every 1% of an upload. For very large uploads, this made it
look like the CLI was hanging.

## Testing

I uploaded a very small databases (few KB), one that was ~100MB, and one
that was 1GB. There are also some automated tests in the PR.

## Failing CI test

The integration tests are failing in the CI. It's because multipart
upload isn't deployed to us-east-1 yet. I changed the integration tests
to use `eu-north-1`. You can see a passing run in that PR
(tursodatabase/tursotest#93).

## AI use

The code was mostly written by Claude, but I iterated a lot with it.

Author: LeMikaelF

go.mod

@@ -1,8 +1,8 @@
 module github.com/tursodatabase/turso-cli
 
-go 1.22
+go 1.25
 
-toolchain go1.22.0
+toolchain go1.25.0
 
 require (
 	github.com/Clever/csvlint v0.3.0

internal/cmd/db_create.go

@@ -113,7 +113,7 @@ func CreateDatabase(name string) error {
 		return err
 	}
 
-	seed, err := parseDBSeedFlags(client, isAWS, remoteEncryptionCipherFlag, multipartFlag)
+	seed, err := parseDBSeedFlags(client, isAWS, remoteEncryptionCipherFlag)
 	if err != nil {
 		return err
 	}

internal/cmd/db_import.go

@@ -8,14 +8,11 @@ import (
 	"github.com/spf13/cobra"
 )
 
-var multipartFlag bool
-
 func init() {
 	dbCmd.AddCommand(importCmd)
 	addGroupFlag(importCmd)
 	addRemoteEncryptionKeyFlag(importCmd)
 	addRemoteEncryptionCipherFlag(importCmd)
-	importCmd.Flags().BoolVar(&multipartFlag, "multipart", false, "force multipart upload")
 }
 
 var importCmd = &cobra.Command{

internal/cmd/db_shell.go

@@ -219,7 +219,7 @@ var shellCmd = &cobra.Command{
 		}
 
 		if db != nil && db.Sleeping {
-			return fmt.Errorf("Your DB might be archived. Please run `turso group unarchive " + db.Group + "` to unarchive it")
+			return fmt.Errorf("your DB might be archived, please run `turso group unarchive %s` to unarchive it", db.Group)
 		}
 		connectionInfo := getConnectionInfo(urlString, db)
 		schemaDb := false

internal/cmd/group_flag.go

@@ -65,7 +65,7 @@ func parseTimestampFlag() (*time.Time, error) {
 	return &timestamp, nil
 }
 
-func parseDBSeedFlags(client *turso.Client, isAWS bool, cipher string, multipart bool) (*turso.DBSeed, error) {
+func parseDBSeedFlags(client *turso.Client, isAWS bool, cipher string) (*turso.DBSeed, error) {
 	if countFlags(fromDBFlag, fromDumpFlag, fromFileFlag, fromDumpURLFlag, fromCSVFlag) > 1 {
 		return nil, errors.New("only one of --from prefixed flags can be used at a time")
 	}
@@ -86,7 +86,7 @@ func parseDBSeedFlags(client *turso.Client, isAWS bool, cipher string, multipart
 	}
 
 	if fromFileFlag != "" {
-		return handleDBFile(client, fromFileFlag, isAWS, cipher, multipart)
+		return handleDBFile(client, fromFileFlag, isAWS, cipher)
 	}
 
 	if fromDumpFlag != "" {
@@ -98,7 +98,7 @@ func parseDBSeedFlags(client *turso.Client, isAWS bool, cipher string, multipart
 		if err != nil {
 			return nil, err
 		}
-		return handleCSVFile(client, fromCSVFlag, csvTableNameFlag, csvSeparator, cipher, multipart)
+		return handleCSVFile(client, fromCSVFlag, csvTableNameFlag, csvSeparator, cipher)
 	}
 	if fromDumpURLFlag != "" {
 		return handleDumpURL(fromDumpURLFlag)
@@ -191,6 +191,20 @@ func countFlags(flags ...string) (count int) {
 }
 
 const MaxAWSDBSizeBytes = 1024 * 1024 * 1024 * 20 // 20 GB
+
+func humanReadableSize(bytes int64) string {
+	const unit = 1024
+	if bytes < unit {
+		return fmt.Sprintf("%d B", bytes)
+	}
+	div, exp := int64(unit), 0
+	for n := bytes / unit; n >= unit; n /= unit {
+		div *= unit
+		exp++
+	}
+	return fmt.Sprintf("%.1f %cB", float64(bytes)/float64(div), "KMGTPE"[exp])
+}
+
 func checkIfDump(filename string) (bool, error) {
 	file, err := os.Open(filename)
 	if err != nil {
@@ -307,9 +321,11 @@ func sqliteFileIntegrityChecks(file string, cipher string) error {
 	if flags.Debug() {
 		log.Printf("Running integrity check...")
 	}
-	_, err = exec.Command("sqlite3", file, "pragma quick_check;").CombinedOutput()
+	spinner := prompt.Spinner(fmt.Sprintf("Validating database file (%s)...", humanReadableSize(fileInfo.Size())))
+	err = runQuickCheck(file)
+	spinner.Stop()
 	if err != nil {
-		return fmt.Errorf("integrity check on database failed: %w", err)
+		return err
 	}
 
 	// validate reserved bytes if encryption cipher is specified
@@ -323,21 +339,28 @@ func sqliteFileIntegrityChecks(file string, cipher string) error {
 	return nil
 }
 
-func handleDBFileAWS(file string, cipher string, multipart bool) (*turso.DBSeed, error) {
+func runQuickCheck(file string) error {
+	cmd := exec.Command("sqlite3", file, "pragma quick_check;")
+	if err := cmd.Run(); err != nil {
+		return fmt.Errorf("integrity check failed: %w", err)
+	}
+	return nil
+}
+
+func handleDBFileAWS(file string, cipher string) (*turso.DBSeed, error) {
 	if err := sqliteFileIntegrityChecks(file, cipher); err != nil {
 		return nil, err
 	}
 
 	seed := &turso.DBSeed{
-		Type:      "database_upload",
-		Filepath:  file,
-		Multipart: multipart,
+		Type:     "database_upload",
+		Filepath: file,
 	}
 
 	return seed, nil
 }
 
-func handleDBFile(client *turso.Client, file string, isAWS bool, cipher string, multipart bool) (*turso.DBSeed, error) {
+func handleDBFile(client *turso.Client, file string, isAWS bool, cipher string) (*turso.DBSeed, error) {
 	if err := checkFileExists(file); err != nil {
 		return nil, err
 	}
@@ -346,7 +369,7 @@ func handleDBFile(client *turso.Client, file string, isAWS bool, cipher string,
 	}
 
 	if isAWS {
-		return handleDBFileAWS(file, cipher, multipart)
+		return handleDBFileAWS(file, cipher)
 	}
 
 	if err := checkSQLiteFile(file); err != nil {
@@ -416,7 +439,7 @@ func dumpSQLiteDatabase(database string, dump *os.File) error {
 	return nil
 }
 
-func handleCSVFile(client *turso.Client, file, csvTableName string, separator rune, cipher string, multipart bool) (*turso.DBSeed, error) {
+func handleCSVFile(client *turso.Client, file, csvTableName string, separator rune, cipher string) (*turso.DBSeed, error) {
 	if err := checkFileExists(file); err != nil {
 		return nil, err
 	}
@@ -446,7 +469,7 @@ func handleCSVFile(client *turso.Client, file, csvTableName string, separator ru
 		return nil, err
 	}
 
-	seed, err := handleDBFile(client, tempDB.Name(), false, cipher, multipart)
+	seed, err := handleDBFile(client, tempDB.Name(), false, cipher)
 	if err != nil {
 		return nil, err
 	}

internal/cmd/group_flag_test.go

@@ -0,0 +1,76 @@
+package cmd
+
+import (
+	"fmt"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+// createTestDatabase creates a valid WAL-mode test database with approximately the given size
+func createTestDatabase(t *testing.T, sizeBytes int) string {
+	t.Helper()
+
+	if _, err := exec.LookPath("sqlite3"); err != nil {
+		t.Skip("sqlite3 not available, skipping test")
+	}
+
+	tmpDir := t.TempDir()
+	dbPath := filepath.Join(tmpDir, "test.db")
+
+	// Create database with correct settings for Turso
+	cmd := exec.Command("sqlite3", dbPath,
+		"PRAGMA page_size=4096;",
+		"PRAGMA journal_mode=WAL;",
+		"CREATE TABLE data (id INTEGER PRIMARY KEY, blob BLOB);")
+	require.NoError(t, cmd.Run(), "failed to create test database")
+
+	// Fill with data to reach target size
+	if sizeBytes > 0 {
+		rowSize := 1000 // ~1KB per row
+		numRows := sizeBytes / rowSize
+		if numRows < 1 {
+			numRows = 1
+		}
+		for i := 0; i < numRows; i++ {
+			cmd = exec.Command("sqlite3", dbPath,
+				fmt.Sprintf("INSERT INTO data (blob) VALUES (randomblob(%d));", rowSize))
+			require.NoError(t, cmd.Run())
+		}
+	}
+
+	return dbPath
+}
+
+func TestRunQuickCheck(t *testing.T) {
+	if _, err := exec.LookPath("sqlite3"); err != nil {
+		t.Skip("sqlite3 not available, skipping test")
+	}
+
+	t.Run("valid database succeeds", func(t *testing.T) {
+		dbPath := createTestDatabase(t, 10*1024) // 10KB
+		err := runQuickCheck(dbPath)
+		require.NoError(t, err)
+	})
+
+	t.Run("corrupted database returns error", func(t *testing.T) {
+		tmpDir := t.TempDir()
+		dbPath := filepath.Join(tmpDir, "corrupt.db")
+
+		// Create a file with garbage data
+		err := os.WriteFile(dbPath, []byte("not a valid sqlite database content here"), 0644)
+		require.NoError(t, err)
+
+		err = runQuickCheck(dbPath)
+		require.Error(t, err)
+		require.Contains(t, err.Error(), "integrity check failed")
+	})
+
+	t.Run("nonexistent file returns error", func(t *testing.T) {
+		err := runQuickCheck("/nonexistent/path/db.sqlite")
+		require.Error(t, err)
+	})
+}

internal/turso/databases.go

@@ -15,8 +15,6 @@ import (
 	"github.com/tursodatabase/turso-cli/internal/prompt"
 )
 
-const multipartUploadThresholdBytes = 100 * 1024 * 1024 // 100MB
-
 type Database struct {
 	ID               string `json:"dbId" mapstructure:"dbId"`
 	Name             string
@@ -133,8 +131,7 @@ type DBSeed struct {
 	Timestamp *time.Time `json:"timestamp,omitempty"`
 	// This is only used locally when uploading a database file and
 	// never passed to the control plane as JSON.
-	Filepath  string `json:"-"`
-	Multipart bool   `json:"-"`
+	Filepath string `json:"-"`
 }
 
 type RemoteEncryption struct {
@@ -158,17 +155,14 @@ type CreateDatabaseBody struct {
 func (d *DatabasesClient) Create(name, location, image, extensions, group string, schema string, isSchema bool, seed *DBSeed, sizeLimit, remoteEncryptionCipher, remoteEncryptionKey string, spinner *prompt.SpinnerT) (*CreateDatabaseResponse, error) {
 	isTursoServerUpload := seed != nil && seed.Type == "database_upload" && seed.Filepath != ""
 	var uploadFilepath string
-	var useMultipart bool
 	var params CreateDatabaseBody
 	if isTursoServerUpload {
 		uploadFilepath = seed.Filepath
-		useMultipart = seed.Multipart
 		// Clear the unused seed parameters, only Type=database_upload is used.
 		seed.Filepath = ""
 		seed.Name = ""
 		seed.URL = ""
 		seed.Timestamp = nil
-		seed.Multipart = false
 		params = CreateDatabaseBody{
 			Name:     name,
 			Location: location,
@@ -216,7 +210,7 @@ func (d *DatabasesClient) Create(name, location, image, extensions, group string
 	}
 
 	if isTursoServerUpload {
-		if _, err = d.UploadDatabaseAWS(data, group, uploadFilepath, remoteEncryptionCipher, remoteEncryptionKey, useMultipart, spinner); err != nil {
+		if _, err = d.UploadDatabaseAWS(data, group, uploadFilepath, remoteEncryptionCipher, remoteEncryptionKey, spinner); err != nil {
 			// Clean up the database if the upload fails
 			if deleteErr := d.Delete(data.Database.Name); deleteErr != nil {
 				fmt.Printf("%v", deleteErr)
@@ -237,36 +231,26 @@ func (d *DatabasesClient) Create(name, location, image, extensions, group string
 //     This call happens in DatabasesClient.Create() above, after which it calls this function.
 //  2. This function creates a DB token for the newly-created DB, and then calls turso-server to upload the database file.
 //     turso-server will perform validations on the file and 'activate' the db if everything is ok.
-func (d *DatabasesClient) UploadDatabaseAWS(resp *CreateDatabaseResponse, group, uploadFilepath, remoteEncryptionCipher, remoteEncryptionKey string, useMultipart bool, spinner *prompt.SpinnerT) (*CreateDatabaseResponse, error) {
-	// Create a short-lived DB token for the newly created database to facilitate the upload
-	token, err := d.Token(resp.Database.Name, "1h", false, nil, nil)
-	if err != nil {
-		return nil, fmt.Errorf("could not create database token: %w", err)
+func (d *DatabasesClient) UploadDatabaseAWS(resp *CreateDatabaseResponse, group, uploadFilepath, remoteEncryptionCipher, remoteEncryptionKey string, spinner *prompt.SpinnerT) (*CreateDatabaseResponse, error) {
+	dbName := resp.Database.Name
+	tokenTTL := 5 * time.Minute
+	tokenProvider := func() (string, error) {
+		return d.Token(dbName, "5m", false, nil, nil)
 	}
 
 	baseURL, err := url.Parse(fmt.Sprintf("https://%s", resp.Database.Hostname))
 	if err != nil {
 		return nil, fmt.Errorf("unable to create TursoServerClient: %v", err)
 	}
-	tursoServerClient, err := NewTursoServerClient(baseURL, token, d.client.cliVersion, d.client.Org)
+	tursoServerClient, err := NewTursoServerClient(baseURL, tokenProvider, tokenTTL, d.client.cliVersion, d.client.Org)
 	if err != nil {
 		return nil, fmt.Errorf("could not create Turso server client: %w", err)
 	}
 
 	// Upload the database file
 	spinner.Text(fmt.Sprintf("Uploading database %s in group %s, this may take a while...", internal.Emph(resp.Database.Name), internal.Emph(group)))
 
-	stat, err := os.Stat(uploadFilepath)
-	if err != nil {
-		return nil, fmt.Errorf("failed to fetch file size %s: %w", uploadFilepath, err)
-	}
-
-	uploadFunc := tursoServerClient.UploadFileSinglePart
-	if useMultipart || stat.Size() > multipartUploadThresholdBytes {
-		uploadFunc = tursoServerClient.UploadFileMultipart
-	}
-
-	err = uploadFunc(uploadFilepath, remoteEncryptionCipher, remoteEncryptionKey, func(progressPct int, uploadedBytes int64, totalBytes int64, elapsedTime time.Duration, done bool) {
+	err = tursoServerClient.UploadFileMultipart(uploadFilepath, remoteEncryptionCipher, remoteEncryptionKey, func(progressPct int, uploadedBytes int64, totalBytes int64, elapsedTime time.Duration, done bool) {
 		totalSeconds := int(elapsedTime.Seconds())
 		minutes := totalSeconds / 60
 		seconds := totalSeconds % 60
@@ -304,15 +288,14 @@ func (d *DatabasesClient) Export(dbName, dbUrl, outputFile string, withMetadata
 			return fmt.Errorf("file %s already exists, use `--overwrite` flag to overwrite it", outputFile)
 		}
 	}
-	token, err := d.Token(dbName, "1h", false, nil, nil)
-	if err != nil {
-		return fmt.Errorf("could not create database token: %w", err)
+	tokenProvider := func() (string, error) {
+		return d.Token(dbName, "1h", false, nil, nil)
 	}
 	baseURL, err := url.Parse(dbUrl)
 	if err != nil {
 		return fmt.Errorf("could not parse database URL: %w", err)
 	}
-	tursoServerClient, err := NewTursoServerClient(baseURL, token, d.client.cliVersion, d.client.Org)
+	tursoServerClient, err := NewTursoServerClient(baseURL, tokenProvider, time.Hour, d.client.cliVersion, d.client.Org)
 	if err != nil {
 		return fmt.Errorf("could not create Turso server client: %w", err)
 	}

internal/turso/turso.go

@@ -63,6 +63,10 @@ func New(base *url.URL, token string, cliVersion string, org string) *Client {
 	return c
 }
 
+func (t *Client) SetToken(token string) {
+	t.token = token
+}
+
 func (t *Client) newRequest(method, urlPath string, body io.Reader, extraHeaders map[string]string) (*http.Request, error) {
 	if _, exists := extraHeaders["Content-Type"]; !exists {
 		return nil, errors.New("content type is required")