Skip to content

Commit 0befb81

Browse files
rk1274rk1274
committed
Small PR fixes, and optimisation of GetFilesInDir
1 parent 3579f8b commit 0befb81

File tree

8 files changed

+58
-27
lines changed

8 files changed

+58
-27
lines changed

‎put/baton.go

+3-3
Original file line numberDiff line numberDiff line change
@@ -502,7 +502,7 @@ func metaToAVUs(meta map[string]string) []ex.AVU {
502502
return avus
503503
}
504504

505-
func (b *Baton) removeFile(path string) error {
505+
func (b *Baton) RemoveFile(path string) error {
506506
it := remotePathToRodsItem(path)
507507

508508
err := timeoutOp(func() error {
@@ -514,7 +514,7 @@ func (b *Baton) removeFile(path string) error {
514514
return err
515515
}
516516

517-
func (b *Baton) queryMeta(dirToSearch string, meta map[string]string) ([]string, error) {
517+
func (b *Baton) QueryMeta(dirToSearch string, meta map[string]string) ([]string, error) {
518518
it := &ex.RodsItem{
519519
IPath: dirToSearch,
520520
IAVUs: metaToAVUs(meta),
@@ -539,7 +539,7 @@ func (b *Baton) queryMeta(dirToSearch string, meta map[string]string) ([]string,
539539
return paths, err
540540
}
541541

542-
// RemoveDirFromIRODS removes the given directory from iRODS given it is empty.
542+
// RemoveDir removes the given directory from iRODS given it is empty.
543543
func (b *Baton) RemoveDir(path string) error {
544544
it := &ex.RodsItem{
545545
IPath: path,

‎put/mock.go

+2-2
Original file line numberDiff line numberDiff line change
@@ -258,13 +258,13 @@ func (l *LocalHandler) RemoveDir(path string) error {
258258
return os.Remove(path)
259259
}
260260

261-
func (l *LocalHandler) removeFile(path string) error {
261+
func (l *LocalHandler) RemoveFile(path string) error {
262262
delete(l.meta, path)
263263

264264
return os.Remove(path)
265265
}
266266

267-
func (l *LocalHandler) queryMeta(dirToSearch string, meta map[string]string) ([]string, error) {
267+
func (l *LocalHandler) QueryMeta(dirToSearch string, meta map[string]string) ([]string, error) {
268268
var objects []string
269269

270270
for path, pathMeta := range l.meta {

‎put/put.go

+4-4
Original file line numberDiff line numberDiff line change
@@ -124,11 +124,11 @@ type Handler interface {
124124
// RemoveDir deletes a given empty folder
125125
RemoveDir(path string) error
126126

127-
// removeFile deletes a given file
128-
removeFile(path string) error
127+
// RemoveFile deletes a given file
128+
RemoveFile(path string) error
129129

130-
// queryMeta return paths to all objects with given metadata
131-
queryMeta(dirToSearch string, meta map[string]string) ([]string, error)
130+
// QueryMeta return paths to all objects with given metadata
131+
QueryMeta(dirToSearch string, meta map[string]string) ([]string, error)
132132
}
133133

134134
// FileReadTester is a function that attempts to open and read the given path,

‎put/put_test.go

+4-4
Original file line numberDiff line numberDiff line change
@@ -173,7 +173,7 @@ func TestPutMock(t *testing.T) {
173173
Convey("RemoveFile removes a file", func() {
174174
filePath := requests[0].Remote
175175

176-
err = lh.removeFile(filePath)
176+
err = lh.RemoveFile(filePath)
177177
So(err, ShouldBeNil)
178178

179179
_, err = os.Stat(filePath)
@@ -193,18 +193,18 @@ func TestPutMock(t *testing.T) {
193193
})
194194

195195
Convey("queryMeta returns all paths with matching metadata", func() {
196-
paths, errq := lh.queryMeta("", map[string]string{"a": "1"})
196+
paths, errq := lh.QueryMeta("", map[string]string{"a": "1"})
197197
So(errq, ShouldBeNil)
198198

199199
So(len(paths), ShouldEqual, 5)
200200

201-
paths, err = lh.queryMeta("", map[string]string{MetaKeyRequester: requests[0].Requester})
201+
paths, err = lh.QueryMeta("", map[string]string{MetaKeyRequester: requests[0].Requester})
202202
So(err, ShouldBeNil)
203203

204204
So(len(paths), ShouldEqual, 1)
205205

206206
Convey("queryMeta only returns paths in the provided scope", func() {
207-
paths, errq := lh.queryMeta(expectedCollections[1], map[string]string{"a": "1"})
207+
paths, errq := lh.QueryMeta(expectedCollections[1], map[string]string{"a": "1"})
208208
So(errq, ShouldBeNil)
209209

210210
So(len(paths), ShouldEqual, 2)

‎put/remove.go

+3-3
Original file line numberDiff line numberDiff line change
@@ -67,7 +67,7 @@ func RemovePathFromSetInIRODS(handler Handler, transformer PathTransformer, path
6767
// same file, if not, it removes the file.
6868
func handleHardlinkAndRemoveFromIRODS(handler Handler, path string, transformer PathTransformer,
6969
meta map[string]string) error {
70-
err := handler.removeFile(path)
70+
err := handler.RemoveFile(path)
7171
if err != nil {
7272
return err
7373
}
@@ -81,7 +81,7 @@ func handleHardlinkAndRemoveFromIRODS(handler Handler, path string, transformer
8181
return err
8282
}
8383

84-
items, err := handler.queryMeta(dirToSearch, map[string]string{MetaKeyRemoteHardlink: meta[MetaKeyRemoteHardlink]})
84+
items, err := handler.QueryMeta(dirToSearch, map[string]string{MetaKeyRemoteHardlink: meta[MetaKeyRemoteHardlink]})
8585
if err != nil {
8686
return err
8787
}
@@ -90,7 +90,7 @@ func handleHardlinkAndRemoveFromIRODS(handler Handler, path string, transformer
9090
return nil
9191
}
9292

93-
return handler.removeFile(meta[MetaKeyRemoteHardlink])
93+
return handler.RemoveFile(meta[MetaKeyRemoteHardlink])
9494
}
9595

9696
// RemoveDirFromIRODS removes the remote path of a given directory from iRODS.

‎server/server.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,7 @@ package server
2929

3030
import (
3131
"context"
32+
"encoding/json"
3233
"errors"
3334
"fmt"
3435
"io"
@@ -40,7 +41,6 @@ import (
4041
jqs "github.com/VertebrateResequencing/wr/jobqueue/scheduler"
4142
"github.com/VertebrateResequencing/wr/queue"
4243
"github.com/gammazero/workerpool"
43-
"github.com/goccy/go-json"
4444
"github.com/inconshreveable/log15"
4545
gas "github.com/wtsi-hgi/go-authserver"
4646
"github.com/wtsi-hgi/ibackup/put"

‎set/db.go

+28-10
Original file line numberDiff line numberDiff line change
@@ -369,23 +369,41 @@ func (d *DB) RemoveDirEntry(setID string, path string) error {
369369

370370
// GetFilesInDir returns all file paths from inside the given directory (and all
371371
// nested inside) for the given set using the db.
372-
func (d *DB) GetFilesInDir(setID string, dirpath string) ([]string, error) {
373-
var filepaths []string
374-
375-
entries, err := d.getEntries(setID, discoveredBucket)
372+
func (d *DBRO) GetFilesInDir(setID string, dirpath string) ([]string, error) {
373+
filepaths, err := d.getPathsWithPrefix(setID, discoveredBucket, dirpath)
376374
if err != nil {
377375
return nil, err
378376
}
379377

380-
for _, entry := range entries {
381-
path := entry.Path
378+
return filepaths, nil
379+
}
380+
381+
// getPathsWithPrefix returns all the filepaths for the given set from the given sub
382+
// bucket prefix, that have the given prefix.
383+
func (d *DBRO) getPathsWithPrefix(setID, bucketName, prefix string) ([]string, error) {
384+
var entries []string
382385

383-
if strings.HasPrefix(path, dirpath) {
384-
filepaths = append(filepaths, path)
386+
err := d.db.View(func(tx *bolt.Tx) error {
387+
subBucketName := []byte(bucketName + separator + setID)
388+
setsBucket := tx.Bucket([]byte(setsBucket))
389+
390+
entriesBucket := setsBucket.Bucket(subBucketName)
391+
if entriesBucket == nil {
392+
return nil
385393
}
386-
}
387394

388-
return filepaths, nil
395+
c := entriesBucket.Cursor()
396+
397+
prefix := []byte(prefix)
398+
399+
for k, _ := c.Seek(prefix); k != nil && bytes.HasPrefix(k, prefix); k, _ = c.Next() {
400+
entries = append(entries, string(k))
401+
}
402+
403+
return nil
404+
})
405+
406+
return entries, err
389407
}
390408

391409
// SetFileEntries sets the file paths for the given backup set. Only supply

‎set/set_test.go

+13
Original file line numberDiff line numberDiff line change
@@ -358,6 +358,19 @@ func TestSetDB(t *testing.T) {
358358
err = db.SetFileEntries(set2.ID(), []string{"/a/b.txt", "/c/k.txt"})
359359
So(err, ShouldBeNil)
360360

361+
Convey("You can get all paths containing a prefix", func() {
362+
err = db.SetFileEntries(set2.ID(), []string{"/a/a/j.txt", "/a/b/c/k.txt",
363+
"/a/b/c/l.txt", "/a/b/d/m.txt", "/c/n.txt"})
364+
So(err, ShouldBeNil)
365+
366+
files, errp := db.getPathsWithPrefix(set2.ID(), fileBucket, "/a/b/c/")
367+
So(errp, ShouldBeNil)
368+
369+
So(len(files), ShouldEqual, 2)
370+
So(files, ShouldContain, "/a/b/c/k.txt")
371+
So(files, ShouldContain, "/a/b/c/l.txt")
372+
})
373+
361374
Convey("Then remove files and dirs from the sets", func() {
362375
err = db.removeEntry(set.ID(), "/a/b.txt", fileBucket)
363376
So(err, ShouldBeNil)

0 commit comments

Comments
 (0)