Skip to content

Commit f80b706

Browse files
penbergpenberg
authored
db export: Export database WAL too (#990)
Fixes #977
2 parents d2a0244 + 00760fc commit f80b706

2 files changed

Lines changed: 175 additions & 36 deletions

File tree

‎internal/cmd/db_export.go‎

Lines changed: 4 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -13,13 +13,12 @@ var outputFile string
1313

1414
var exportCmd = &cobra.Command{
1515
Use: "export <database>",
16-
Short: "Export a database snapshot from Turso to SQLite file.",
17-
Long: `Export a database snapshot from Turso to SQLite file.
16+
Short: "Export a database snapshot and WAL from Turso to SQLite files.",
17+
Long: `Export a database snapshot and WAL from Turso to SQLite files.
1818
1919
This command exports a snapshot of the current generation of a Turso database
20-
to a local SQLite file. Note that the exported file may not contain the
21-
latest changes. Use SDK to sync the database after exporting to ensure you
22-
have the most recent version.`,
20+
to a local SQLite file, along with any WAL (Write-Ahead Log) frames. The WAL
21+
file will be saved as <database>.db-wal alongside the main database file.`,
2322
Args: cobra.ExactArgs(1),
2423
RunE: func(cmd *cobra.Command, args []string) error {
2524
cmd.SilenceUsage = true

‎internal/turso/tursoServer.go‎

Lines changed: 171 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
package turso
22

33
import (
4+
"crypto/rand"
45
"encoding/binary"
56
"encoding/json"
67
"errors"
@@ -98,6 +99,10 @@ func (i *TursoServerClient) UploadFile(filepath string, onUploadProgress func(pr
9899
return nil
99100
}
100101

102+
type ExportInfo struct {
103+
CurrentGeneration int `json:"current_generation"`
104+
}
105+
101106
func (i *TursoServerClient) Export(outputFile string, withMetadata bool) error {
102107
res, err := i.client.Get("/info", nil)
103108
if err != nil {
@@ -107,9 +112,7 @@ func (i *TursoServerClient) Export(outputFile string, withMetadata bool) error {
107112
if res.StatusCode != http.StatusOK {
108113
return parseResponseError(res)
109114
}
110-
var info struct {
111-
CurrentGeneration int `json:"current_generation"`
112-
}
115+
var info ExportInfo
113116
if err := json.NewDecoder(res.Body).Decode(&info); err != nil {
114117
return fmt.Errorf("failed to decode /info response: %w", err)
115118
}
@@ -132,39 +135,176 @@ func (i *TursoServerClient) Export(outputFile string, withMetadata bool) error {
132135
return fmt.Errorf("failed to write export to file: %w", err)
133136
}
134137

138+
lastFrameNo, err := i.ExportWAL(outputFile, &info)
139+
if err != nil {
140+
return fmt.Errorf("failed to export WAL: %w", err)
141+
}
135142
if withMetadata {
136-
out, err := os.Create(outputFile + "-info")
143+
if err := i.ExportMetadata(outputFile, &info, lastFrameNo); err != nil {
144+
return fmt.Errorf("failed to export metadata: %w", err)
145+
}
146+
}
147+
148+
return nil
149+
}
150+
151+
func (i *TursoServerClient) ExportWAL(outputFile string, info *ExportInfo) (int, error) {
152+
walFile := outputFile + "-wal"
153+
walOut, err := os.Create(walFile)
154+
if err != nil {
155+
return 0, fmt.Errorf("failed to create WAL file: %w", err)
156+
}
157+
defer walOut.Close()
158+
159+
var saltBytes [8]byte
160+
if _, err := rand.Read(saltBytes[:]); err != nil {
161+
return 0, fmt.Errorf("failed to generate random salt values: %w", err)
162+
}
163+
salt1 := binary.BigEndian.Uint32(saltBytes[0:4]) // Random salt-1
164+
salt2 := binary.BigEndian.Uint32(saltBytes[4:8]) // Random salt-2
165+
166+
walHeader := make([]byte, 32)
167+
binary.BigEndian.PutUint32(walHeader[0:4], 0x377f0682) // Magic number
168+
binary.BigEndian.PutUint32(walHeader[4:8], 3007000) // File format version
169+
binary.BigEndian.PutUint32(walHeader[8:12], 4096) // Database page size
170+
binary.BigEndian.PutUint32(walHeader[12:16], 0) // Checkpoint sequence number
171+
binary.BigEndian.PutUint32(walHeader[16:20], salt1) // Salt-1 (must match frames)
172+
binary.BigEndian.PutUint32(walHeader[20:24], salt2) // Salt-2 (must match frames)
173+
174+
s0 := uint32(0)
175+
s1 := uint32(0)
176+
177+
for i := 0; i < 24; i += 8 {
178+
x0 := binary.LittleEndian.Uint32(walHeader[i : i+4])
179+
x1 := binary.LittleEndian.Uint32(walHeader[i+4 : i+8])
180+
s0 += x0 + s1
181+
s1 += x1 + s0
182+
}
183+
184+
binary.BigEndian.PutUint32(walHeader[24:28], s0)
185+
binary.BigEndian.PutUint32(walHeader[28:32], s1)
186+
187+
if _, err := walOut.Write(walHeader); err != nil {
188+
return 0, fmt.Errorf("failed to write WAL header: %w", err)
189+
}
190+
191+
const batchSize = 128
192+
frameNo := 1
193+
lastFrameNo := 0
194+
195+
for {
196+
walRes, err := i.client.Get(fmt.Sprintf("/sync/%d/%d/%d", info.CurrentGeneration, frameNo, frameNo+batchSize), nil)
137197
if err != nil {
138-
return fmt.Errorf("failed to create info file: %w", err)
198+
if frameNo == 1 {
199+
break
200+
}
201+
return lastFrameNo, fmt.Errorf("failed to fetch WAL frames: %w", err)
139202
}
140-
defer out.Close()
141-
142-
hasher := crc32.New(crc32.MakeTable(crc32.IEEE))
143-
var versionBytes [4]byte
144-
var durableFrameNumBytes [4]byte
145-
var generationBytes [4]byte
146-
binary.LittleEndian.PutUint32(versionBytes[:], 0)
147-
binary.LittleEndian.PutUint32(durableFrameNumBytes[:], 0)
148-
binary.LittleEndian.PutUint32(generationBytes[:], uint32(info.CurrentGeneration))
149-
hasher.Write(versionBytes[:])
150-
hasher.Write(durableFrameNumBytes[:])
151-
hasher.Write(generationBytes[:])
152-
hash := int(hasher.Sum32())
153-
154-
metadata := struct {
155-
Hash int `json:"hash"`
156-
Version int `json:"version"`
157-
DurableFrameNum int `json:"durable_frame_num"`
158-
Generation int `json:"generation"`
159-
}{
160-
Hash: hash,
161-
Version: 0,
162-
DurableFrameNum: 0,
163-
Generation: info.CurrentGeneration,
203+
204+
if walRes.StatusCode == http.StatusBadRequest || walRes.StatusCode == http.StatusInternalServerError {
205+
walRes.Body.Close()
206+
break
164207
}
165-
if err := json.NewEncoder(out).Encode(metadata); err != nil {
166-
return fmt.Errorf("failed to write metadata to file: %w", err)
208+
if walRes.StatusCode != http.StatusOK {
209+
walRes.Body.Close()
210+
if frameNo == 1 {
211+
break
212+
}
213+
return lastFrameNo, parseResponseError(walRes)
214+
}
215+
216+
frames, err := io.ReadAll(walRes.Body)
217+
walRes.Body.Close()
218+
if err != nil {
219+
return lastFrameNo, fmt.Errorf("failed to read WAL frames: %w", err)
167220
}
221+
222+
if len(frames) == 0 {
223+
break
224+
}
225+
226+
frameSize := 4120
227+
framesInBatch := len(frames) / frameSize
228+
229+
for i := 0; i < framesInBatch; i++ {
230+
offset := i * frameSize
231+
if offset+frameSize > len(frames) {
232+
return lastFrameNo, fmt.Errorf("invalid frame data: expected %d bytes, got %d", frameSize, len(frames)-offset)
233+
}
234+
frame := frames[offset : offset+frameSize]
235+
236+
binary.BigEndian.PutUint32(frame[8:12], salt1)
237+
binary.BigEndian.PutUint32(frame[12:16], salt2)
238+
239+
x0 := binary.LittleEndian.Uint32(frame[0:4])
240+
x1 := binary.LittleEndian.Uint32(frame[4:8])
241+
s0 += x0 + s1
242+
s1 += x1 + s0
243+
244+
for j := 24; j < frameSize; j += 8 {
245+
x0 := binary.LittleEndian.Uint32(frame[j : j+4])
246+
x1 := binary.LittleEndian.Uint32(frame[j+4 : j+8])
247+
s0 += x0 + s1
248+
s1 += x1 + s0
249+
}
250+
251+
binary.BigEndian.PutUint32(frame[16:20], s0)
252+
binary.BigEndian.PutUint32(frame[20:24], s1)
253+
254+
if _, err := walOut.Write(frame); err != nil {
255+
return lastFrameNo, fmt.Errorf("failed to write WAL frame: %w", err)
256+
}
257+
258+
lastFrameNo = frameNo + i
259+
}
260+
261+
if framesInBatch < batchSize {
262+
break
263+
}
264+
265+
frameNo += framesInBatch
168266
}
267+
268+
if err := walOut.Sync(); err != nil {
269+
return lastFrameNo, fmt.Errorf("failed to sync WAL file: %w", err)
270+
}
271+
272+
return lastFrameNo, nil
273+
}
274+
275+
func (i *TursoServerClient) ExportMetadata(outputFile string, info *ExportInfo, durableFrameNum int) error {
276+
out, err := os.Create(outputFile + "-info")
277+
if err != nil {
278+
return fmt.Errorf("failed to create info file: %w", err)
279+
}
280+
defer out.Close()
281+
282+
hasher := crc32.New(crc32.MakeTable(crc32.IEEE))
283+
var versionBytes [4]byte
284+
var durableFrameNumBytes [4]byte
285+
var generationBytes [4]byte
286+
binary.LittleEndian.PutUint32(versionBytes[:], 0)
287+
binary.LittleEndian.PutUint32(durableFrameNumBytes[:], uint32(durableFrameNum))
288+
binary.LittleEndian.PutUint32(generationBytes[:], uint32(info.CurrentGeneration))
289+
hasher.Write(versionBytes[:])
290+
hasher.Write(durableFrameNumBytes[:])
291+
hasher.Write(generationBytes[:])
292+
hash := int(hasher.Sum32())
293+
294+
metadata := struct {
295+
Hash int `json:"hash"`
296+
Version int `json:"version"`
297+
DurableFrameNum int `json:"durable_frame_num"`
298+
Generation int `json:"generation"`
299+
}{
300+
Hash: hash,
301+
Version: 0,
302+
DurableFrameNum: durableFrameNum,
303+
Generation: info.CurrentGeneration,
304+
}
305+
if err := json.NewEncoder(out).Encode(metadata); err != nil {
306+
return fmt.Errorf("failed to write metadata to file: %w", err)
307+
}
308+
169309
return nil
170310
}

0 commit comments

Comments
 (0)