Skip to content

Commit fe1498f

Browse files
committed
migrate: add support for new Go io/fs.FS
This patch adds FromFS function and lets you use the new go:embed directive to add the migration files to the binary. Also, migration tests use an in memory FS impl instead of working with tmp directories.
1 parent 7980a95 commit fe1498f

File tree

6 files changed

+73
-107
lines changed

6 files changed

+73
-107
lines changed

go.mod

+2-2
Original file line numberDiff line numberDiff line change
@@ -3,10 +3,10 @@ module github.com/scylladb/gocqlx/v2
33
go 1.16
44

55
require (
6-
github.com/davecgh/go-spew v1.1.1 // indirect
76
github.com/gocql/gocql v0.0.0-20200131111108-92af2e088537
87
github.com/golang/snappy v0.0.1 // indirect
9-
github.com/google/go-cmp v0.2.0
8+
github.com/google/go-cmp v0.5.4
9+
github.com/psanford/memfs v0.0.0-20210214183328-a001468d78ef
1010
github.com/scylladb/go-reflectx v1.0.1
1111
golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a
1212
gopkg.in/inf.v0 v0.9.1

go.sum

+6-8
Original file line numberDiff line numberDiff line change
@@ -2,33 +2,31 @@ github.com/bitly/go-hostpool v0.0.0-20171023180738-a3a6125de932 h1:mXoPYz/Ul5HYE
22
github.com/bitly/go-hostpool v0.0.0-20171023180738-a3a6125de932/go.mod h1:NOuUCSz6Q9T7+igc/hlvDOUdtWKryOrtFyIVABv/p7k=
33
github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869 h1:DDGfHa7BWjL4YnC6+E63dPcxHo2sUxDIu8g3QgEJdRY=
44
github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869/go.mod h1:Ekp36dRnpXw/yCqJaO+ZrUyxD+3VXMFFr56k5XYrpB4=
5-
github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8=
65
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
7-
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
8-
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
96
github.com/gocql/gocql v0.0.0-20200131111108-92af2e088537 h1:NaMut1fdw76YYX/TPinSAbai4DShF5tPort3bHpET6g=
107
github.com/gocql/gocql v0.0.0-20200131111108-92af2e088537/go.mod h1:DL0ekTmBSTdlNF25Orwt/JMzqIq3EJ4MVa/J/uK64OY=
11-
github.com/golang/snappy v0.0.0-20170215233205-553a64147049 h1:K9KHZbXKpGydfDN0aZrsoHpLJlZsBrGMFWbgLDGnPZk=
128
github.com/golang/snappy v0.0.0-20170215233205-553a64147049/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
139
github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4=
1410
github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
15-
github.com/google/go-cmp v0.2.0 h1:+dTQ8DZQJz0Mb/HjFlkptS1FeQ4cWSnN941F8aEG4SQ=
16-
github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
11+
github.com/google/go-cmp v0.5.4 h1:L8R9j+yAqZuZjsqh/z+F1NCffTKKLShY6zXTItVIZ8M=
12+
github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
1713
github.com/hailocab/go-hostpool v0.0.0-20160125115350-e80d13ce29ed h1:5upAirOpQc1Q53c0bnx2ufif5kANL7bfZWcc6VJWJd8=
1814
github.com/hailocab/go-hostpool v0.0.0-20160125115350-e80d13ce29ed/go.mod h1:tMWxXQ9wFIaZeTI9F+hmhFiGpFmhOHzyShyFUhRm0H4=
1915
github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
2016
github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
2117
github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
2218
github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
2319
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
24-
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
2520
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
21+
github.com/psanford/memfs v0.0.0-20210214183328-a001468d78ef h1:NKxTG6GVGbfMXc2mIk+KphcH6hagbVXhcFkbTgYleTI=
22+
github.com/psanford/memfs v0.0.0-20210214183328-a001468d78ef/go.mod h1:tcaRap0jS3eifrEEllL6ZMd9dg8IlDpi2S1oARrQ+NI=
2623
github.com/scylladb/go-reflectx v1.0.1 h1:b917wZM7189pZdlND9PbIJ6NQxfDPfBvUaQ7cjj1iZQ=
2724
github.com/scylladb/go-reflectx v1.0.1/go.mod h1:rWnOfDIRWBGN0miMLIcoPt/Dhi2doCMZqwMCJ3KupFc=
2825
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
29-
github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q=
3026
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
3127
golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a h1:WXEvlFVvvGxCJLG6REjsT03iWnKLEWinaScsxF2Vm2o=
3228
golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
29+
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4=
30+
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
3331
gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc=
3432
gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw=

migrate/checksum.go

+5-5
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@ import (
88
"crypto/md5"
99
"encoding/hex"
1010
"io"
11-
"os"
11+
"io/fs"
1212
)
1313

1414
var encode = hex.EncodeToString
@@ -18,15 +18,15 @@ func checksum(b []byte) string {
1818
return encode(v[:])
1919
}
2020

21-
func fileChecksum(path string) (string, error) {
22-
f, err := os.Open(path)
21+
func fileChecksum(f fs.FS, path string) (string, error) {
22+
file, err := f.Open(path)
2323
if err != nil {
2424
return "", nil
2525
}
26-
defer f.Close()
26+
defer file.Close()
2727

2828
h := md5.New()
29-
if _, err := io.Copy(h, f); err != nil {
29+
if _, err := io.Copy(h, file); err != nil {
3030
return "", err
3131
}
3232
v := h.Sum(nil)

migrate/checksum_test.go

+5-2
Original file line numberDiff line numberDiff line change
@@ -4,10 +4,13 @@
44

55
package migrate
66

7-
import "testing"
7+
import (
8+
"os"
9+
"testing"
10+
)
811

912
func TestFileChecksum(t *testing.T) {
10-
c, err := fileChecksum("testdata/file")
13+
c, err := fileChecksum(os.DirFS("testdata"), "file")
1114
if err != nil {
1215
t.Fatal(err)
1316
}

migrate/migrate.go

+27-16
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@ import (
1010
"errors"
1111
"fmt"
1212
"io"
13+
"io/fs"
1314
"io/ioutil"
1415
"os"
1516
"path/filepath"
@@ -90,37 +91,47 @@ func ensureInfoTable(ctx context.Context, session gocqlx.Session) error {
9091
return session.ContextQuery(ctx, infoSchema, nil).ExecRelease()
9192
}
9293

93-
// Migrate reads the cql files from a directory and applies required migrations.
94-
// It also supports code based migrations, see Callback and CallbackFunc.
95-
// Any comment in form `-- CALL <name>;` will trigger an CallComment callback.
94+
// Migrate is a wrapper around FromFS.
95+
// It executes migrations from a directory on disk.
96+
//
97+
// Deprecated: use FromFS instead
9698
func Migrate(ctx context.Context, session gocqlx.Session, dir string) error {
99+
return FromFS(ctx, session, os.DirFS(dir))
100+
}
101+
102+
// FromFS executes new CQL files from a file system abstraction (io/fs.FS).
103+
// The provided FS has to be a flat directory containing *.cql files.
104+
//
105+
// It supports code based migrations, see Callback and CallbackFunc.
106+
// Any comment in form `-- CALL <name>;` will trigger an CallComment callback.
107+
func FromFS(ctx context.Context, session gocqlx.Session, f fs.FS) error {
97108
// get database migrations
98109
dbm, err := List(ctx, session)
99110
if err != nil {
100111
return fmt.Errorf("list migrations: %s", err)
101112
}
102113

103114
// get file migrations
104-
fm, err := filepath.Glob(filepath.Join(dir, "*.cql"))
115+
fm, err := fs.Glob(f, "*.cql")
105116
if err != nil {
106-
return fmt.Errorf("list migrations in %q: %s", dir, err)
117+
return fmt.Errorf("list migrations: %w", err)
107118
}
108119
if len(fm) == 0 {
109-
return fmt.Errorf("no migration files found in %q", dir)
120+
return fmt.Errorf("no migration files found")
110121
}
111122
sort.Strings(fm)
112123

113124
// verify migrations
114125
if len(dbm) > len(fm) {
115-
return fmt.Errorf("database is ahead of %q", dir)
126+
return fmt.Errorf("database is ahead")
116127
}
117128

118129
for i := 0; i < len(dbm); i++ {
119-
if dbm[i].Name != filepath.Base(fm[i]) {
120-
fmt.Println(dbm[i].Name, filepath.Base(fm[i]), i)
130+
if dbm[i].Name != fm[i] {
131+
fmt.Println(dbm[i].Name, fm[i], i)
121132
return errors.New("inconsistent migrations")
122133
}
123-
c, err := fileChecksum(fm[i])
134+
c, err := fileChecksum(f, fm[i])
124135
if err != nil {
125136
return fmt.Errorf("calculate checksum for %q: %s", fm[i], err)
126137
}
@@ -132,13 +143,13 @@ func Migrate(ctx context.Context, session gocqlx.Session, dir string) error {
132143
// apply migrations
133144
if len(dbm) > 0 {
134145
last := len(dbm) - 1
135-
if err := applyMigration(ctx, session, fm[last], dbm[last].Done); err != nil {
146+
if err := applyMigration(ctx, session, f, fm[last], dbm[last].Done); err != nil {
136147
return fmt.Errorf("apply migration %q: %s", fm[last], err)
137148
}
138149
}
139150

140151
for i := len(dbm); i < len(fm); i++ {
141-
if err := applyMigration(ctx, session, fm[i], 0); err != nil {
152+
if err := applyMigration(ctx, session, f, fm[i], 0); err != nil {
142153
return fmt.Errorf("apply migration %q: %s", fm[i], err)
143154
}
144155
}
@@ -150,14 +161,14 @@ func Migrate(ctx context.Context, session gocqlx.Session, dir string) error {
150161
return nil
151162
}
152163

153-
func applyMigration(ctx context.Context, session gocqlx.Session, path string, done int) error {
154-
f, err := os.Open(path)
164+
func applyMigration(ctx context.Context, session gocqlx.Session, f fs.FS, path string, done int) error {
165+
file, err := f.Open(path)
155166
if err != nil {
156167
return err
157168
}
158169

159-
b, err := ioutil.ReadAll(f)
160-
f.Close()
170+
b, err := ioutil.ReadAll(file)
171+
file.Close()
161172
if err != nil {
162173
return err
163174
}

migrate/migrate_test.go

+28-74
Original file line numberDiff line numberDiff line change
@@ -9,12 +9,11 @@ package migrate_test
99
import (
1010
"context"
1111
"fmt"
12-
"io/ioutil"
13-
"os"
14-
"path/filepath"
12+
"io/fs"
1513
"strings"
1614
"testing"
1715

16+
"github.com/psanford/memfs"
1817
"github.com/scylladb/gocqlx/v2"
1918
. "github.com/scylladb/gocqlx/v2/gocqlxtest"
2019
"github.com/scylladb/gocqlx/v2/migrate"
@@ -52,10 +51,7 @@ func TestMigration(t *testing.T) {
5251
ctx := context.Background()
5352

5453
t.Run("init", func(t *testing.T) {
55-
dir := makeMigrationDir(t, 2)
56-
defer os.Remove(dir)
57-
58-
if err := migrate.Migrate(ctx, session, dir); err != nil {
54+
if err := migrate.FromFS(ctx, session, makeTestFS(2)); err != nil {
5955
t.Fatal(err)
6056
}
6157
if c := countMigrations(t, session); c != 2 {
@@ -64,10 +60,7 @@ func TestMigration(t *testing.T) {
6460
})
6561

6662
t.Run("update", func(t *testing.T) {
67-
dir := makeMigrationDir(t, 4)
68-
defer os.Remove(dir)
69-
70-
if err := migrate.Migrate(ctx, session, dir); err != nil {
63+
if err := migrate.FromFS(ctx, session, makeTestFS(4)); err != nil {
7164
t.Fatal(err)
7265
}
7366
if c := countMigrations(t, session); c != 4 {
@@ -76,23 +69,19 @@ func TestMigration(t *testing.T) {
7669
})
7770

7871
t.Run("ahead", func(t *testing.T) {
79-
dir := makeMigrationDir(t, 2)
80-
defer os.Remove(dir)
81-
82-
if err := migrate.Migrate(ctx, session, dir); err == nil || !strings.Contains(err.Error(), "ahead") {
72+
err := migrate.FromFS(ctx, session, makeTestFS(2))
73+
if err == nil || !strings.Contains(err.Error(), "ahead") {
8374
t.Fatal("expected error")
8475
} else {
8576
t.Log(err)
8677
}
8778
})
8879

8980
t.Run("tempered with file", func(t *testing.T) {
90-
dir := makeMigrationDir(t, 4)
91-
defer os.Remove(dir)
92-
93-
appendMigrationFile(t, dir, 3, "\nSELECT * FROM bla;\n")
81+
f := makeTestFS(4)
82+
writeFile(f, 3, "SELECT * FROM bla;")
9483

95-
if err := migrate.Migrate(ctx, session, dir); err == nil || !strings.Contains(err.Error(), "tempered") {
84+
if err := migrate.FromFS(ctx, session, f); err == nil || !strings.Contains(err.Error(), "tempered") {
9685
t.Fatal("expected error")
9786
} else {
9887
t.Log(err)
@@ -109,19 +98,11 @@ func TestMigrationNoSemicolon(t *testing.T) {
10998
t.Fatal(err)
11099
}
111100

112-
ctx := context.Background()
113-
114-
dir := makeMigrationDir(t, 1)
115-
defer os.Remove(dir)
101+
f := makeTestFS(0)
102+
f.WriteFile("0.cql", []byte(fmt.Sprintf(insertMigrate, 0)+";"+fmt.Sprintf(insertMigrate, 1)), fs.ModePerm)
116103

117-
f, err := os.OpenFile(filepath.Join(dir, "0.cql"), os.O_WRONLY|os.O_APPEND, 0)
118-
if err != nil {
119-
t.Fatal(err)
120-
}
121-
fmt.Fprintf(f, insertMigrate, 0) // note no ; at the end
122-
f.Close()
123-
124-
if err := migrate.Migrate(ctx, session, dir); err != nil {
104+
ctx := context.Background()
105+
if err := migrate.FromFS(ctx, session, f); err != nil {
125106
t.Fatal(err)
126107
}
127108
if c := countMigrations(t, session); c != 2 {
@@ -231,62 +212,38 @@ func TestMigrationCallback(t *testing.T) {
231212
ctx := context.Background()
232213

233214
t.Run("init", func(t *testing.T) {
234-
dir := makeMigrationDir(t, 2)
235-
defer os.Remove(dir)
215+
f := makeTestFS(2)
236216
reset()
237217

238-
if err := migrate.Migrate(ctx, session, dir); err != nil {
218+
if err := migrate.FromFS(ctx, session, f); err != nil {
239219
t.Fatal(err)
240220
}
241221
assertCallbacks(t, 2, 2, 0)
242222
})
243223

244224
t.Run("no duplicate calls", func(t *testing.T) {
245-
dir := makeMigrationDir(t, 4)
246-
defer os.Remove(dir)
225+
f := makeTestFS(4)
247226
reset()
248227

249-
if err := migrate.Migrate(ctx, session, dir); err != nil {
228+
if err := migrate.FromFS(ctx, session, f); err != nil {
250229
t.Fatal(err)
251230
}
252231
assertCallbacks(t, 2, 2, 0)
253232
})
254233

255234
t.Run("in calls", func(t *testing.T) {
256-
dir := makeMigrationDir(t, 4)
257-
defer os.Remove(dir)
235+
f := makeTestFS(4)
236+
writeFile(f, 4, "\n-- CALL Foo;\n")
237+
writeFile(f, 5, "\n-- CALL Bar;\n")
258238
reset()
259239

260-
appendMigrationFile(t, dir, 4, "\n-- CALL Foo;\n")
261-
appendMigrationFile(t, dir, 5, "\n-- CALL Bar;\n")
262-
263-
if err := migrate.Migrate(ctx, session, dir); err != nil {
240+
if err := migrate.FromFS(ctx, session, f); err != nil {
264241
t.Fatal(err)
265242
}
266243
assertCallbacks(t, 2, 2, 2)
267244
})
268245
}
269246

270-
func makeMigrationDir(tb testing.TB, n int) (dir string) {
271-
tb.Helper()
272-
273-
dir, err := ioutil.TempDir("", "gocqlx_migrate")
274-
if err != nil {
275-
tb.Fatal(err)
276-
}
277-
278-
for i := 0; i < n; i++ {
279-
path := migrateFilePath(dir, i)
280-
cql := []byte(fmt.Sprintf(insertMigrate, i) + ";")
281-
if err := ioutil.WriteFile(path, cql, os.ModePerm); err != nil {
282-
os.Remove(dir)
283-
tb.Fatal(err)
284-
}
285-
}
286-
287-
return dir
288-
}
289-
290247
func countMigrations(tb testing.TB, session gocqlx.Session) int {
291248
tb.Helper()
292249

@@ -297,17 +254,14 @@ func countMigrations(tb testing.TB, session gocqlx.Session) int {
297254
return v
298255
}
299256

300-
func appendMigrationFile(tb testing.TB, dir string, i int, text string) {
301-
path := migrateFilePath(dir, i)
302-
f, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_APPEND, os.ModePerm)
303-
if err != nil {
304-
tb.Fatal(err)
305-
}
306-
if _, err := f.WriteString(text); err != nil {
307-
tb.Fatal(err)
257+
func makeTestFS(n int) *memfs.FS {
258+
f := memfs.New()
259+
for i := 0; i < n; i++ {
260+
writeFile(f, i, fmt.Sprintf(insertMigrate, i)+";")
308261
}
262+
return f
309263
}
310264

311-
func migrateFilePath(dir string, i int) string {
312-
return filepath.Join(dir, fmt.Sprint(i, ".cql"))
265+
func writeFile(f *memfs.FS, i int, text string) {
266+
f.WriteFile(fmt.Sprint(i, ".cql"), []byte(text), fs.ModePerm)
313267
}

0 commit comments

Comments
 (0)