Skip to content

Commit b4721e9

Browse files
committed
use embedded Turso Go driver for imports
1 parent 1b8cb41 commit b4721e9

5 files changed

Lines changed: 88 additions & 33 deletions

File tree

go.mod

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@ require (
2525
github.com/stretchr/testify v1.11.1
2626
golang.org/x/exp v0.0.0-20240716160929-1d5bc16f04a8
2727
golang.org/x/sync v0.19.0
28+
turso.tech/database/tursogo v0.7.1
2829
)
2930

3031
require (
@@ -34,11 +35,13 @@ require (
3435
github.com/charmbracelet/x/cellbuf v0.0.13-0.20250311204145-2c3ea96c31dd // indirect
3536
github.com/charmbracelet/x/term v0.2.1 // indirect
3637
github.com/davecgh/go-spew v1.1.1 // indirect
38+
github.com/ebitengine/purego v0.9.1 // indirect
3739
github.com/erikgeiser/coninput v0.0.0-20211004153227-1c3628e74d0f // indirect
3840
github.com/go-viper/mapstructure/v2 v2.4.0 // indirect
3941
github.com/pmezard/go-difflib v1.0.0 // indirect
4042
github.com/sagikazarmark/locafero v0.11.0 // indirect
4143
github.com/sourcegraph/conc v0.3.1-0.20240121214520-5f936abd7ae8 // indirect
44+
github.com/tursodatabase/turso-go-platform-libs v0.7.1 // indirect
4245
github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect
4346
go.yaml.in/yaml/v3 v3.0.4 // indirect
4447
)
@@ -58,7 +61,7 @@ require (
5861
github.com/mattn/go-isatty v0.0.20
5962
github.com/mattn/go-localereader v0.0.1 // indirect
6063
github.com/mattn/go-runewidth v0.0.16 // indirect
61-
github.com/mattn/go-sqlite3 v1.14.16 // indirect
64+
github.com/mattn/go-sqlite3 v1.14.42 // indirect
6265
github.com/muesli/ansi v0.0.0-20230316100256-276c6243b2f6 // indirect
6366
github.com/muesli/cancelreader v0.2.2 // indirect
6467
github.com/muesli/termenv v0.16.0 // indirect

go.sum

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -43,6 +43,8 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c
4343
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
4444
github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
4545
github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
46+
github.com/ebitengine/purego v0.9.1 h1:a/k2f2HQU3Pi399RPW1MOaZyhKJL9w/xFpKAg4q1s0A=
47+
github.com/ebitengine/purego v0.9.1/go.mod h1:iIjxzd6CiRiOG0UyXP+V1+jWqUXVjPKLAI0mRfJZTmQ=
4648
github.com/erikgeiser/coninput v0.0.0-20211004153227-1c3628e74d0f h1:Y/CXytFA4m6baUTXGLOoWe4PQhGxaX0KpnayAqC48p4=
4749
github.com/erikgeiser/coninput v0.0.0-20211004153227-1c3628e74d0f/go.mod h1:vw97MGsxSvLiUE2X8qFplwetxpGLQrlU1Q9AUEIzCaM=
4850
github.com/fatih/color v1.18.0 h1:S8gINlzdQ840/4pfAwic/ZE0djQEH3wM94VfqLTZcOM=
@@ -113,8 +115,8 @@ github.com/mattn/go-runewidth v0.0.7/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m
113115
github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI=
114116
github.com/mattn/go-runewidth v0.0.16 h1:E5ScNMtiwvlvB5paMFdw9p4kSQzbXFikJ5SQO6TULQc=
115117
github.com/mattn/go-runewidth v0.0.16/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=
116-
github.com/mattn/go-sqlite3 v1.14.16 h1:yOQRA0RpS5PFz/oikGwBEqvAWhWg5ufRz4ETLjwpU1Y=
117-
github.com/mattn/go-sqlite3 v1.14.16/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S2DGjv9HUNg=
118+
github.com/mattn/go-sqlite3 v1.14.42 h1:MigqEP4ZmHw3aIdIT7T+9TLa90Z6smwcthx+Azv4Cgo=
119+
github.com/mattn/go-sqlite3 v1.14.42/go.mod h1:pjEuOr8IwzLJP2MfGeTb0A35jauH+C2kbHKBr7yXKVQ=
118120
github.com/mitchellh/copystructure v1.2.0 h1:vpKXTN4ewci03Vljg/q9QvCGUDttBOGBIa15WveJJGw=
119121
github.com/mitchellh/copystructure v1.2.0/go.mod h1:qLl+cE2AmVv+CoeAwDPye/v+N2HKCj9FbZEVFJRxO9s=
120122
github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY=
@@ -166,6 +168,8 @@ github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8
166168
github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU=
167169
github.com/tursodatabase/libsql-client-go v0.0.0-20260514053736-a9a8fadfe885 h1:YssVXwM/9nUAjGNmUWdgvb05JVcsaBrDn5yr+MaJTn0=
168170
github.com/tursodatabase/libsql-client-go v0.0.0-20260514053736-a9a8fadfe885/go.mod h1:08inkKyguB6CGGssc/JzhmQWwBgFQBgjlYFjxjRh7nU=
171+
github.com/tursodatabase/turso-go-platform-libs v0.7.1 h1:sQ2aupg/dhilMWRRhIIjd9w8ikue4vIYGxHFMtWhYZE=
172+
github.com/tursodatabase/turso-go-platform-libs v0.7.1/go.mod h1:bo+Lpv5OYOX1gRV9L5DLKMsYxmDs56SkZwnCOLEFcxU=
169173
github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e h1:JVG44RsyaB9T2KIHavMF/ppJZNG9ZpyihvCd0w101no=
170174
github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e/go.mod h1:RbqR21r5mrJuqunuUZ/Dhy/avygyECGrLceyNeo4LiM=
171175
go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc=
@@ -198,3 +202,5 @@ gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EV
198202
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
199203
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
200204
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
205+
turso.tech/database/tursogo v0.7.1 h1:xCVENss9TeC5dR2Xs0saZZpVg05+1hW9pL1ei8KSAQ8=
206+
turso.tech/database/tursogo v0.7.1/go.mod h1:sDMlDKBFfVdU1zi3qOd+HqU0Nnu/84UYGR6sZGqBIAM=

internal/cmd/db_import_file.go

Lines changed: 75 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@ package cmd
22

33
import (
44
"bufio"
5+
"database/sql"
56
"errors"
67
"fmt"
78
"log"
@@ -14,6 +15,7 @@ import (
1415
"github.com/tursodatabase/turso-cli/internal/flags"
1516
"github.com/tursodatabase/turso-cli/internal/prompt"
1617
"github.com/tursodatabase/turso-cli/internal/turso"
18+
_ "turso.tech/database/tursogo"
1719
)
1820

1921
const MaxAWSDBSizeBytes = 1024 * 1024 * 1024 * 20 // 20 GB
@@ -88,7 +90,6 @@ func sqliteFileIntegrityChecks(file string, cipher string) error {
8890
func tursoDBFileIntegrityChecks(file string) error {
8991
return databaseFileIntegrityChecks(file, "", databaseFileChecker{
9092
name: "TursoDB",
91-
binary: "tursodb",
9293
journalMode: "mvcc",
9394
settings: tursoDBDatabaseSettings,
9495
quickCheck: runTursoDBQuickCheck,
@@ -135,15 +136,27 @@ func databaseFileIntegrityChecks(file, cipher string, checker databaseFileChecke
135136

136137
func validateDatabaseSettings(file string, settings databaseSettings, checker databaseFileChecker) error {
137138
if !strings.EqualFold(settings.journalMode, checker.journalMode) {
139+
if checker.binary == "" {
140+
return fmt.Errorf("database is not in %s mode", strings.ToUpper(checker.journalMode))
141+
}
138142
return fmt.Errorf("database is not in %s mode. Set it with '%s %s \"PRAGMA journal_mode = %s;\"'", strings.ToUpper(checker.journalMode), checker.binary, file, strings.ToUpper(checker.journalMode))
139143
}
140144
if settings.pageSize != "4096" {
145+
if checker.binary == "" {
146+
return errors.New("database must use 4KB page size")
147+
}
141148
return fmt.Errorf("database must use 4KB page size. You can set it with '%s %s \"PRAGMA page_size = 4096; VACUUM;\"'", checker.binary, file)
142149
}
143150
if settings.autoVacuum != "0" {
151+
if checker.binary == "" {
152+
return errors.New("database must have autovacuum disabled")
153+
}
144154
return fmt.Errorf("database must have autovacuum disabled. You can set it with '%s %s \"PRAGMA auto_vacuum = 0;\"'", checker.binary, file)
145155
}
146156
if !strings.EqualFold(settings.encoding, "UTF-8") {
157+
if checker.binary == "" {
158+
return errors.New("database must use UTF-8 encoding")
159+
}
147160
return fmt.Errorf("database must use UTF-8 encoding. You can set it with '%s %s \"PRAGMA encoding = 'UTF-8';\"'", checker.binary, file)
148161
}
149162
return nil
@@ -175,17 +188,38 @@ func sqliteDatabaseSettings(file string) (databaseSettings, error) {
175188
}
176189

177190
func tursoDBDatabaseSettings(file string) (databaseSettings, error) {
178-
output, err := exec.Command("tursodb", "-q", "-m", "list", file, databaseSettingsQuery).CombinedOutput()
191+
db, err := openTursoDB(file)
179192
if err != nil {
180-
return databaseSettings{}, fmt.Errorf("failed to check database settings with TursoDB: %w: %s", err, strings.TrimSpace(string(output)))
193+
return databaseSettings{}, fmt.Errorf("failed to check database settings with TursoDB: %w", err)
181194
}
182-
settings, err := parseDatabaseSettings(string(output))
195+
defer db.Close()
196+
197+
var settings databaseSettings
198+
err = db.QueryRow(databaseSettingsQuery).Scan(
199+
&settings.journalMode,
200+
&settings.pageSize,
201+
&settings.autoVacuum,
202+
&settings.encoding,
203+
)
183204
if err != nil {
184-
return databaseSettings{}, fmt.Errorf("failed to parse database settings from TursoDB: %w", err)
205+
return databaseSettings{}, fmt.Errorf("failed to query database settings with TursoDB: %w", err)
185206
}
186207
return settings, nil
187208
}
188209

210+
func openTursoDB(file string) (*sql.DB, error) {
211+
db, err := sql.Open("turso", file)
212+
if err != nil {
213+
return nil, err
214+
}
215+
db.SetMaxOpenConns(1)
216+
if err := db.Ping(); err != nil {
217+
db.Close()
218+
return nil, err
219+
}
220+
return db, nil
221+
}
222+
189223
func runQuickCheck(file string) error {
190224
cmd := exec.Command("sqlite3", "-list", file, "pragma quick_check;")
191225
if err := cmd.Run(); err != nil {
@@ -195,13 +229,34 @@ func runQuickCheck(file string) error {
195229
}
196230

197231
func runTursoDBQuickCheck(file string) error {
198-
output, err := exec.Command("tursodb", "-q", "-m", "list", file, "PRAGMA quick_check;").CombinedOutput()
232+
db, err := openTursoDB(file)
233+
if err != nil {
234+
return fmt.Errorf("TursoDB integrity check failed for %s: %w", file, err)
235+
}
236+
defer db.Close()
237+
238+
rows, err := db.Query("PRAGMA quick_check;")
199239
if err != nil {
200-
return fmt.Errorf("TursoDB integrity check failed for %s: %w: %s", file, err, strings.TrimSpace(string(output)))
240+
return fmt.Errorf("TursoDB integrity check failed for %s: %w", file, err)
241+
}
242+
defer rows.Close()
243+
244+
checked := false
245+
for rows.Next() {
246+
var result string
247+
if err := rows.Scan(&result); err != nil {
248+
return fmt.Errorf("TursoDB integrity check failed for %s: %w", file, err)
249+
}
250+
checked = true
251+
if result != "ok" {
252+
return fmt.Errorf("TursoDB integrity check failed for %s: %s", file, result)
253+
}
254+
}
255+
if err := rows.Err(); err != nil {
256+
return fmt.Errorf("TursoDB integrity check failed for %s: %w", file, err)
201257
}
202-
fields := strings.Fields(string(output))
203-
if len(fields) == 0 || fields[len(fields)-1] != "ok" {
204-
return fmt.Errorf("TursoDB integrity check failed for %s: %s", file, strings.TrimSpace(string(output)))
258+
if !checked {
259+
return fmt.Errorf("TursoDB integrity check failed for %s: no result", file)
205260
}
206261
return nil
207262
}
@@ -222,10 +277,17 @@ func checkpointWALBeforeUpload(file string) error {
222277
}
223278

224279
func prepareTursoDBFile(file string) error {
225-
output, err := exec.Command("tursodb", "-q", "-m", "list", file,
226-
"PRAGMA journal_mode = mvcc; PRAGMA wal_checkpoint(TRUNCATE);").CombinedOutput()
280+
db, err := openTursoDB(file)
227281
if err != nil {
228-
return fmt.Errorf("could not prepare %s for TursoDB import: %w: %s", file, err, strings.TrimSpace(string(output)))
282+
return fmt.Errorf("could not prepare %s for TursoDB import: %w", file, err)
283+
}
284+
_, execErr := db.Exec("PRAGMA journal_mode = mvcc; PRAGMA wal_checkpoint(TRUNCATE);")
285+
closeErr := db.Close()
286+
if execErr != nil {
287+
return fmt.Errorf("could not prepare %s for TursoDB import: %w", file, execErr)
288+
}
289+
if closeErr != nil {
290+
return fmt.Errorf("could not close %s after preparing it for TursoDB import: %w", file, closeErr)
229291
}
230292

231293
format, err := sniffSQLiteFileFormat(file)
@@ -322,9 +384,6 @@ func handleDBFileAWS(file string, cipher string) (*turso.DBSeed, error) {
322384
}
323385

324386
if tursoDBFlag {
325-
if err := checkTursoDBAvailable(); err != nil {
326-
return nil, err
327-
}
328387
if format != fileFormatMVCC {
329388
fmt.Printf("Converting %s to TursoDB (MVCC) format for import.\n", file)
330389
}
@@ -385,11 +444,3 @@ func validateReservedBytes(dbPath string, cipher string) error {
385444
}
386445
return nil
387446
}
388-
389-
func checkTursoDBAvailable() error {
390-
_, err := exec.LookPath("tursodb")
391-
if errors.Is(err, exec.ErrNotFound) {
392-
return errors.New("could not find tursodb on your system. Please install it to import into a TursoDB database")
393-
}
394-
return err
395-
}

internal/cmd/db_import_file_test.go

Lines changed: 1 addition & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -72,7 +72,6 @@ func TestRunQuickCheck(t *testing.T) {
7272

7373
func TestValidateDatabaseSettings(t *testing.T) {
7474
checker := databaseFileChecker{
75-
binary: "tursodb",
7675
journalMode: "mvcc",
7776
}
7877
settings := databaseSettings{
@@ -140,12 +139,9 @@ func TestCheckpointWALBeforeUpload(t *testing.T) {
140139
}
141140

142141
func TestPrepareTursoDBFile(t *testing.T) {
143-
if _, err := exec.LookPath("tursodb"); err != nil {
144-
t.Skip("tursodb not available, skipping test")
145-
}
146-
147142
dbPath := createTestDatabase(t, 10*1024)
148143
require.NoError(t, checkpointWALBeforeUpload(dbPath))
144+
t.Setenv("PATH", t.TempDir()) // TursoDB preparation must not require an external binary.
149145
require.NoError(t, prepareTursoDBFile(dbPath))
150146
require.NoError(t, tursoDBFileIntegrityChecks(dbPath))
151147
require.NoError(t, checkTursoDBSidecars(dbPath))

internal/cmd/group_flag.go

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,6 @@ import (
77
"fmt"
88
"os"
99
"os/exec"
10-
"strings"
1110
"time"
1211

1312
"github.com/Clever/csvlint"

0 commit comments

Comments
 (0)