Add operations supporting encrypted databases on Cloud (#1000)

This patch adds operations for encrypted databases:

- create (fresh db, upload)
- fork
- shell

Author: avinassh

go.mod

@@ -15,7 +15,7 @@ require (
 	github.com/google/uuid v1.3.0
 	github.com/hashicorp/go-version v1.6.0
 	github.com/kirsle/configdir v0.0.0-20170128060238-e45d2f54772f
-	github.com/libsql/libsql-shell-go v0.10.6
+	github.com/libsql/libsql-shell-go v0.10.7-0.20251205123613-0342ea2584be
 	github.com/manifoldco/promptui v0.9.0
 	github.com/mitchellh/mapstructure v1.5.0
 	github.com/olekukonko/tablewriter v0.0.5
@@ -58,7 +58,7 @@ require (
 	github.com/spf13/jwalterweatherman v1.1.0 // indirect
 	github.com/spf13/pflag v1.0.5 // indirect
 	github.com/subosito/gotenv v1.4.2 // indirect
-	github.com/tursodatabase/libsql-client-go v0.0.0-20240902231107-85af5b9d094d // indirect
+	github.com/tursodatabase/libsql-client-go v0.0.0-20251205113610-b69dd6e475fc // indirect
 	golang.org/x/crypto v0.14.0 // indirect
 	golang.org/x/net v0.17.0 // indirect
 	golang.org/x/sys v0.13.0 // indirect

go.sum

@@ -203,8 +203,8 @@ github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
 github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw=
 github.com/leodido/go-urn v1.2.4 h1:XlAE/cm/ms7TE/VMVoduSpNBoyc2dOxHs5MZSwAN63Q=
 github.com/leodido/go-urn v1.2.4/go.mod h1:7ZrI8mTSeBSHl/UaRyKQW1qZeMgak41ANeCNaVckg+4=
-github.com/libsql/libsql-shell-go v0.10.6 h1:Ck/OAqdYnp4vAsBFyaS73lV8Kg6+mp60sv4XhQpLF/Y=
-github.com/libsql/libsql-shell-go v0.10.6/go.mod h1:CzfyMwVmQhHxs85xglrNgkdBasrcd8XN5QTpg4A6JFQ=
+github.com/libsql/libsql-shell-go v0.10.7-0.20251205123613-0342ea2584be h1:01jRtOlo3emaVJIgPvyvTSanxt8Q9oe7u+DLQBGpVtY=
+github.com/libsql/libsql-shell-go v0.10.7-0.20251205123613-0342ea2584be/go.mod h1:hO7V4+aqYxjLP742up3TN8hcS88xlbdCiLHWjtWAtE8=
 github.com/lucasb-eyer/go-colorful v1.0.3/go.mod h1:R4dSotOR9KMtayYi1e77YzuveK+i7ruzyGqttikkLy0=
 github.com/lucasb-eyer/go-colorful v1.2.0 h1:1nnpGOrhyZZuNyfu1QjKiUICQ74+3FNCN69Aj6K7nkY=
 github.com/lucasb-eyer/go-colorful v1.2.0/go.mod h1:R4dSotOR9KMtayYi1e77YzuveK+i7ruzyGqttikkLy0=
@@ -294,8 +294,8 @@ github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsT
 github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
 github.com/subosito/gotenv v1.4.2 h1:X1TuBLAMDFbaTAChgCBLu3DU3UPyELpnF2jjJ2cz/S8=
 github.com/subosito/gotenv v1.4.2/go.mod h1:ayKnFf/c6rvx/2iiLrJUk1e6plDbT3edrFNGqEflhK0=
-github.com/tursodatabase/libsql-client-go v0.0.0-20240902231107-85af5b9d094d h1:dOMI4+zEbDI37KGb0TI44GUAwxHF9cMsIoDTJ7UmgfU=
-github.com/tursodatabase/libsql-client-go v0.0.0-20240902231107-85af5b9d094d/go.mod h1:l8xTsYB90uaVdMHXMCxKKLSgw5wLYBwBKKefNIUnm9s=
+github.com/tursodatabase/libsql-client-go v0.0.0-20251205113610-b69dd6e475fc h1:uhpFwk9G+wp9JpPnaABzwyIUz1P4EYIkEKKivyJVO14=
+github.com/tursodatabase/libsql-client-go v0.0.0-20251205113610-b69dd6e475fc/go.mod h1:08inkKyguB6CGGssc/JzhmQWwBgFQBgjlYFjxjRh7nU=
 github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=

internal/cmd/db_create.go

@@ -1,6 +1,7 @@
 package cmd
 
 import (
+	"encoding/base64"
 	"fmt"
 	"strings"
 	"time"
@@ -37,6 +38,8 @@ func init() {
 	addSchemaFlag(createCmd)
 	addTypeFlag(createCmd)
 	addSizeLimitFlag(createCmd)
+	addRemoteEncryptionCipherFlag(createCmd)
+	addRemoteEncryptionKeyFlag(createCmd)
 }
 
 var createCmd = &cobra.Command{
@@ -93,6 +96,10 @@ func CreateDatabase(name string) error {
 		version = "canary"
 	}
 
+	if err = validateEncryptionFlags(); err != nil {
+		return err
+	}
+
 	if err := ensureGroup(client, groupName, groups, location, version); err != nil {
 		return err
 	}
@@ -101,7 +108,7 @@ func CreateDatabase(name string) error {
 	spinner := prompt.Spinner(fmt.Sprintf("Creating database %s in group %s...", internal.Emph(name), internal.Emph(groupName)))
 	defer spinner.Stop()
 
-	if _, err = client.Databases.Create(name, location, "", "", groupName, schemaFlag, typeFlag == "schema", seed, sizeLimitFlag, spinner); err != nil {
+	if _, err = client.Databases.Create(name, location, "", "", groupName, schemaFlag, typeFlag == "schema", seed, sizeLimitFlag, remoteEncryptionCipherFlag, remoteEncryptionKeyFlag(), spinner); err != nil {
 		return fmt.Errorf("could not create database %s: %w", name, err)
 	}
 
@@ -209,3 +216,30 @@ func shouldAutoCreateGroup(name string, groups []turso.Group) bool {
 	// we only create the default group automatically
 	return name == "default" && len(groups) == 0
 }
+
+func validateEncryptionFlags() error {
+	remoteEncryptionKey := remoteEncryptionKeyFlag()
+	if remoteEncryptionKey == "" && remoteEncryptionCipherFlag == "" {
+		return nil
+	}
+	// if key flag is empty, then user passed only the cipher, which is invalid
+	if remoteEncryptionKey == "" {
+		return fmt.Errorf("remote encryption key must be provided when remote encryption cipher is set")
+	}
+
+	// if key is provided, lets verify its in base64 encoded
+	_, err := base64.StdEncoding.DecodeString(remoteEncryptionKey)
+	if err != nil {
+		return fmt.Errorf("encryption key (%s) is not valid base64: %w", remoteEncryptionKey, err)
+	}
+
+	if remoteEncryptionCipherFlag != "" {
+		return nil
+	}
+
+	// if cipher is empty, then it is only valid in case of forks and for everything else we need to have it set
+	if fromDBFlag == "" {
+		return fmt.Errorf("remote encryption cipher must be provided when remote encryption key is set")
+	}
+	return nil
+}

internal/cmd/db_export.go

@@ -50,7 +50,7 @@ func ExportDatabase(dbName, outputFile string, withMetadata bool, overwrite bool
 		return fmt.Errorf("failed to find database: %w", err)
 	}
 	dbUrl := getDatabaseHttpUrl(&db)
-	err = client.Databases.Export(dbName, dbUrl, outputFile, withMetadata, overwrite)
+	err = client.Databases.Export(dbName, dbUrl, outputFile, withMetadata, overwrite, remoteEncryptionKeyFlag())
 	if err != nil {
 		return err
 	}
@@ -61,5 +61,6 @@ func init() {
 	exportCmd.Flags().BoolVar(&withMetadata, "with-metadata", false, "Include metadata in the export.")
 	exportCmd.Flags().BoolVar(&overwriteExport, "overwrite", false, "Overwrite output file if it exists.")
 	exportCmd.Flags().StringVar(&outputFile, "output-file", "", "Specify the output file name (default: <database>.db)")
+	addRemoteEncryptionKeyFlag(exportCmd)
 	dbCmd.AddCommand(exportCmd)
 }

internal/cmd/db_shell.go

@@ -27,6 +27,7 @@ func init() {
 	dbCmd.AddCommand(shellCmd)
 	addInstanceFlag(shellCmd, "Connect to the database at the specified instance.")
 	addLocationFlag(shellCmd, "Connect to the database at the specified location.")
+	addRemoteEncryptionKeyFlag(shellCmd)
 	shellCmd.Flags().StringVar(&proxy, "proxy", "", "Proxy to use for the connection.")
 	shellCmd.RegisterFlagCompletionFunc("proxy", func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
 		return []string{}, cobra.ShellCompDirectiveNoFileComp
@@ -227,15 +228,16 @@ var shellCmd = &cobra.Command{
 		}
 
 		shellConfig := shell.ShellConfig{
-			DbUri:          dbUrl,
-			Proxy:          proxy,
-			AuthToken:      authToken,
-			InF:            cmd.InOrStdin(),
-			OutF:           cmd.OutOrStdout(),
-			ErrF:           cmd.ErrOrStderr(),
-			HistoryMode:    enums.PerDatabaseHistory,
-			HistoryName:    "turso",
-			WelcomeMessage: &connectionInfo,
+			DbUri:               dbUrl,
+			Proxy:               proxy,
+			AuthToken:           authToken,
+			RemoteEncryptionKey: remoteEncryptionKeyFlag(),
+			InF:                 cmd.InOrStdin(),
+			OutF:                cmd.OutOrStdout(),
+			ErrF:                cmd.ErrOrStderr(),
+			HistoryMode:         enums.PerDatabaseHistory,
+			HistoryName:         "turso",
+			WelcomeMessage:      &connectionInfo,
 			AfterDbConnectionCallback: func() {
 				spinner.Stop()
 			},

internal/cmd/db_show.go

@@ -149,6 +149,9 @@ var showCmd = &cobra.Command{
 		if db.Schema != "" {
 			fmt.Println("Schema:            ", db.Schema)
 		}
+		if db.EncryptionCipher != "" {
+			fmt.Println("Encryption:        ", db.EncryptionCipher)
+		}
 
 		fmt.Println()
 

internal/cmd/encryption_flag.go

@@ -0,0 +1,26 @@
+package cmd
+
+import (
+	"os"
+
+	"github.com/spf13/cobra"
+)
+
+// unexported, don't use directly. Consider using remoteEncryptionKeyFlag()
+var remoteEncryptionKeyArg string
+var remoteEncryptionCipherFlag string
+
+func addRemoteEncryptionKeyFlag(cmd *cobra.Command) {
+	cmd.Flags().StringVar(&remoteEncryptionKeyArg, "remote-encryption-key", "", "Encryption key (in base64) for accessing encrypted databases on Turso cloud")
+}
+
+func addRemoteEncryptionCipherFlag(cmd *cobra.Command) {
+	cmd.Flags().StringVar(&remoteEncryptionCipherFlag, "remote-encryption-cipher", "", "Cipher to use for database encryption")
+}
+
+func remoteEncryptionKeyFlag() string {
+	if remoteEncryptionKeyArg != "" {
+		return remoteEncryptionKeyArg
+	}
+	return os.Getenv("TURSO_DB_REMOTE_ENCRYPTION_KEY")
+}

internal/turso/databases.go

@@ -16,17 +16,18 @@ import (
 )
 
 type Database struct {
-	ID            string `json:"dbId" mapstructure:"dbId"`
-	Name          string
-	Regions       []string
-	PrimaryRegion string
-	Hostname      string
-	Version       string
-	Group         string
-	Sleeping      bool
-	Schema        string
-	IsSchema      bool      `json:"is_schema" mapstructure:"is_schema"`
-	Parent        *Database `json:"parent,omitempty"`
+	ID               string `json:"dbId" mapstructure:"dbId"`
+	Name             string
+	Regions          []string
+	PrimaryRegion    string
+	Hostname         string
+	Version          string
+	Group            string
+	Sleeping         bool
+	Schema           string
+	IsSchema         bool      `json:"is_schema" mapstructure:"is_schema"`
+	Parent           *Database `json:"parent,omitempty"`
+	EncryptionCipher string    `json:"encryption_cipher,omitempty"`
 }
 
 type DatabasesClient client
@@ -133,19 +134,25 @@ type DBSeed struct {
 	Filepath string `json:"-"`
 }
 
+type RemoteEncryption struct {
+	EncryptionKey    string `json:"encryption_key"`
+	EncryptionCipher string `json:"encryption_cipher"`
+}
+
 type CreateDatabaseBody struct {
-	Name       string  `json:"name"`
-	Location   string  `json:"location"`
-	Image      string  `json:"image,omitempty"`
-	Extensions string  `json:"extensions,omitempty"`
-	Group      string  `json:"group,omitempty"`
-	Seed       *DBSeed `json:"seed,omitempty"`
-	Schema     string  `json:"schema,omitempty"`
-	IsSchema   bool    `json:"is_schema,omitempty"`
-	SizeLimit  string  `json:"size_limit,omitempty"`
-}
-
-func (d *DatabasesClient) Create(name, location, image, extensions, group string, schema string, isSchema bool, seed *DBSeed, sizeLimit string, spinner *prompt.SpinnerT) (*CreateDatabaseResponse, error) {
+	Name             string            `json:"name"`
+	Location         string            `json:"location"`
+	Image            string            `json:"image,omitempty"`
+	Extensions       string            `json:"extensions,omitempty"`
+	Group            string            `json:"group,omitempty"`
+	Seed             *DBSeed           `json:"seed,omitempty"`
+	Schema           string            `json:"schema,omitempty"`
+	IsSchema         bool              `json:"is_schema,omitempty"`
+	SizeLimit        string            `json:"size_limit,omitempty"`
+	RemoteEncryption *RemoteEncryption `json:"remote_encryption,omitempty"`
+}
+
+func (d *DatabasesClient) Create(name, location, image, extensions, group string, schema string, isSchema bool, seed *DBSeed, sizeLimit, remoteEncryptionCipher, remoteEncryptionKey string, spinner *prompt.SpinnerT) (*CreateDatabaseResponse, error) {
 	isTursoServerUpload := seed != nil && seed.Type == "database_upload" && seed.Filepath != ""
 	var uploadFilepath string
 	var params CreateDatabaseBody
@@ -163,7 +170,14 @@ func (d *DatabasesClient) Create(name, location, image, extensions, group string
 			Seed:     seed,
 		}
 	} else {
-		params = CreateDatabaseBody{name, location, image, extensions, group, seed, schema, isSchema, sizeLimit}
+		params = CreateDatabaseBody{name, location, image, extensions, group, seed, schema, isSchema, sizeLimit, nil}
+	}
+
+	if remoteEncryptionKey != "" {
+		params.RemoteEncryption = &RemoteEncryption{
+			EncryptionKey:    remoteEncryptionKey,
+			EncryptionCipher: remoteEncryptionCipher,
+		}
 	}
 
 	body, err := marshal(params)
@@ -196,7 +210,7 @@ func (d *DatabasesClient) Create(name, location, image, extensions, group string
 	}
 
 	if isTursoServerUpload {
-		if _, err = d.UploadDatabaseAWS(data, group, uploadFilepath, spinner); err != nil {
+		if _, err = d.UploadDatabaseAWS(data, group, uploadFilepath, remoteEncryptionCipher, remoteEncryptionKey, spinner); err != nil {
 			// Clean up the database if the upload fails
 			if deleteErr := d.Delete(data.Database.Name); deleteErr != nil {
 				fmt.Printf("%v", deleteErr)
@@ -217,7 +231,7 @@ func (d *DatabasesClient) Create(name, location, image, extensions, group string
 //     This call happens in DatabasesClient.Create() above, after which it calls this function.
 //  2. This function creates a DB token for the newly-created DB, and then calls turso-server to upload the database file.
 //     turso-server will perform validations on the file and 'activate' the db if everything is ok.
-func (d *DatabasesClient) UploadDatabaseAWS(resp *CreateDatabaseResponse, group string, uploadFilepath string, spinner *prompt.SpinnerT) (*CreateDatabaseResponse, error) {
+func (d *DatabasesClient) UploadDatabaseAWS(resp *CreateDatabaseResponse, group, uploadFilepath, remoteEncryptionCipher, remoteEncryptionKey string, spinner *prompt.SpinnerT) (*CreateDatabaseResponse, error) {
 	// Create a short-lived DB token for the newly created database to facilitate the upload
 	token, err := d.Token(resp.Database.Name, "1h", false, nil, nil)
 	if err != nil {
@@ -235,7 +249,7 @@ func (d *DatabasesClient) UploadDatabaseAWS(resp *CreateDatabaseResponse, group
 
 	// Upload the database file
 	spinner.Text(fmt.Sprintf("Uploading database %s in group %s, this may take a while...", internal.Emph(resp.Database.Name), internal.Emph(group)))
-	err = tursoServerClient.UploadFile(uploadFilepath, func(progressPct int, uploadedBytes int64, totalBytes int64, elapsedTime time.Duration, done bool) {
+	err = tursoServerClient.UploadFile(uploadFilepath, remoteEncryptionCipher, remoteEncryptionKey, func(progressPct int, uploadedBytes int64, totalBytes int64, elapsedTime time.Duration, done bool) {
 		totalSeconds := int(elapsedTime.Seconds())
 		minutes := totalSeconds / 60
 		seconds := totalSeconds % 60
@@ -267,7 +281,7 @@ func (d *DatabasesClient) UploadDatabaseAWS(resp *CreateDatabaseResponse, group
 	return resp, nil
 }
 
-func (d *DatabasesClient) Export(dbName, dbUrl, outputFile string, withMetadata bool, overwrite bool) error {
+func (d *DatabasesClient) Export(dbName, dbUrl, outputFile string, withMetadata bool, overwrite bool, remoteEncryptionKey string) error {
 	if !overwrite {
 		if _, err := os.Stat(outputFile); err == nil {
 			return fmt.Errorf("file %s already exists, use `--overwrite` flag to overwrite it", outputFile)
@@ -285,7 +299,7 @@ func (d *DatabasesClient) Export(dbName, dbUrl, outputFile string, withMetadata
 	if err != nil {
 		return fmt.Errorf("could not create Turso server client: %w", err)
 	}
-	return tursoServerClient.Export(outputFile, withMetadata)
+	return tursoServerClient.Export(outputFile, withMetadata, remoteEncryptionKey)
 }
 
 func (d *DatabasesClient) Seed(name string, dbFile *os.File) error {

internal/turso/turso.go

@@ -148,6 +148,9 @@ func (t *Client) Get(path string, body io.Reader) (*http.Response, error) {
 }
 
 func (t *Client) GetWithHeaders(path string, body io.Reader, headers map[string]string) (*http.Response, error) {
+	if headers == nil {
+		headers = make(map[string]string)
+	}
 	headers["Content-Type"] = "application/json"
 	return t.do("GET", path, body, headers)
 }
@@ -156,8 +159,12 @@ func (t *Client) Post(path string, body io.Reader) (*http.Response, error) {
 	return t.do("POST", path, body, Header("Content-Type", "application/json"))
 }
 
-func (t *Client) PostBinary(path string, body io.Reader) (*http.Response, error) {
-	return t.do("POST", path, body, Header("Content-Type", "application/octet-stream"))
+func (t *Client) PostBinary(path string, body io.Reader, headers map[string]string) (*http.Response, error) {
+	if headers == nil {
+		headers = make(map[string]string)
+	}
+	headers["Content-Type"] = "application/octet-stream"
+	return t.do("POST", path, body, headers)
 }
 
 func (t *Client) Patch(path string, body io.Reader) (*http.Response, error) {