Skip to content
This repository was archived by the owner on Aug 18, 2025. It is now read-only.

Commit a414feb

Browse files
gtoppergtopper
authored
Fix paging when listing a v3io directory. (#2)
1 parent 79e7400 commit a414feb

File tree

3 files changed

+75
-52
lines changed

3 files changed

+75
-52
lines changed

‎backends/v3io.go‎

Lines changed: 46 additions & 40 deletions
Original file line numberDiff line numberDiff line change
@@ -13,8 +13,6 @@ import (
1313
"time"
1414
)
1515

16-
const defaultWorkers = 0
17-
1816
const (
1917
V3ioPathEnvironmentVariable = "V3IO_API"
2018
V3ioUserEnvironmentVariable = "V3IO_USERNAME"
@@ -81,54 +79,62 @@ func (c *V3ioClient) ListDir(fileChan chan *FileDetails, task *ListDirTask, summ
8179

8280
func (c *V3ioClient) getDir(path string, fileChan chan *FileDetails, summary *ListSummary) error {
8381

84-
resp, err := c.container.GetContainerContentsSync(&v3io.GetContainerContentsInput{Path: path})
85-
if err != nil {
86-
c.logger.ErrorWith("ListBucket failed", "endpoint", c.params.Endpoint, "container",
87-
c.params.Bucket, "path", path)
88-
return errors.Wrap(err, "failed v3io ListBucket")
89-
}
90-
result := resp.Output.(*v3io.GetContainerContentsOutput)
82+
req := v3io.GetContainerContentsInput{Path: path}
83+
for {
84+
resp, err := c.container.GetContainerContentsSync(&req)
85+
if err != nil {
86+
c.logger.ErrorWith("ListBucket failed", "endpoint", c.params.Endpoint, "container",
87+
c.params.Bucket, "path", path)
88+
return errors.Wrap(err, "failed v3io ListBucket")
89+
}
90+
result := resp.Output.(*v3io.GetContainerContentsOutput)
9191

92-
for _, obj := range result.Contents {
92+
for _, obj := range result.Contents {
9393

94-
if strings.HasSuffix(obj.Key, "/") {
95-
continue
96-
}
94+
if strings.HasSuffix(obj.Key, "/") {
95+
continue
96+
}
9797

98-
t, err := time.Parse(time.RFC3339, obj.LastModified)
99-
if err != nil {
100-
return errors.Wrap(err, "Invalid object time string - not an RFC 3339 time format.")
101-
}
102-
var size int64
103-
if obj.Size != nil {
104-
size = int64(*obj.Size)
105-
}
106-
_, name := filepath.Split(obj.Key)
98+
t, err := time.Parse(time.RFC3339, obj.LastModified)
99+
if err != nil {
100+
return errors.Wrap(err, "Invalid object time string - not an RFC 3339 time format.")
101+
}
102+
var size int64
103+
if obj.Size != nil {
104+
size = int64(*obj.Size)
105+
}
106+
_, name := filepath.Split(obj.Key)
107107

108-
if !IsMatch(c.task, name, t, size) {
109-
continue
110-
}
108+
if !IsMatch(c.task, name, t, size) {
109+
continue
110+
}
111111

112-
c.logger.DebugWith("List dir:", "key", obj.Key, "modified", obj.LastModified, "size", obj.Size)
113-
fileDetails := &FileDetails{
114-
Key: obj.Key, Size: size, //Mtime: obj.LastModified,
115-
}
112+
c.logger.DebugWith("List dir:", "key", obj.Key, "modified", obj.LastModified, "size", obj.Size)
113+
fileDetails := &FileDetails{
114+
Key: obj.Key, Size: size, //Mtime: obj.LastModified,
115+
}
116116

117-
summary.TotalBytes += size
118-
summary.TotalFiles += 1
119-
fileChan <- fileDetails
120-
}
117+
summary.TotalBytes += size
118+
summary.TotalFiles += 1
119+
fileChan <- fileDetails
120+
}
121121

122-
if c.task.Recursive {
123-
for _, val := range result.CommonPrefixes {
124-
_, name := filepath.Split(val.Prefix[0 : len(val.Prefix)-1])
125-
if c.task.Hidden || !strings.HasPrefix(name, ".") {
126-
err = c.getDir(val.Prefix, fileChan, summary)
127-
if err != nil {
128-
return err
122+
if c.task.Recursive {
123+
for _, val := range result.CommonPrefixes {
124+
_, name := filepath.Split(val.Prefix[0 : len(val.Prefix)-1])
125+
if c.task.Hidden || !strings.HasPrefix(name, ".") {
126+
err = c.getDir(val.Prefix, fileChan, summary)
127+
if err != nil {
128+
return err
129+
}
129130
}
130131
}
131132
}
133+
134+
if !result.IsTruncated {
135+
break
136+
}
137+
req.Marker = result.NextMarker
132138
}
133139
return nil
134140
}

‎go.mod‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -12,10 +12,10 @@ require (
1212
github.com/pkg/errors v0.8.1
1313
github.com/smartystreets/goconvey v0.0.0-20190731233626-505e41936337 // indirect
1414
github.com/stretchr/testify v1.4.0
15-
github.com/v3io/v3io-go v0.0.0-20190804122140-7a7baa9fe04ff8591cb4b22270d598b36fc0d49a
15+
github.com/tinylib/msgp v1.1.6 // indirect
16+
github.com/v3io/v3io-go v0.0.0-20190804122140-7a7baa9fe04f
1617
github.com/valyala/fasthttp v1.4.0 // indirect
1718
go.uber.org/atomic v1.4.0 // indirect
18-
golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4
19-
golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7 // indirect
19+
golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9
2020
gopkg.in/ini.v1 v1.46.0 // indirect
2121
)

‎go.sum‎

Lines changed: 26 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,3 @@
1-
github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8=
21
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
32
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
43
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
@@ -34,6 +33,8 @@ github.com/nuclio/zap v0.0.2 h1:rY5PkMOl8CTkqRqIPuxziBiKK6Mq/8oEurfgRnNtqf0=
3433
github.com/nuclio/zap v0.0.2/go.mod h1:SUxPsgePvlyjx6c5MtGdB50pf0IQThtlyLwISLboeuc=
3534
github.com/pavius/zap v1.4.2-0.20180228181622-8d52692529b8 h1:WqLgmr/wj9TO5Sc6oYPQRAJBxuHE0NTeuVeFnT+FZVo=
3635
github.com/pavius/zap v1.4.2-0.20180228181622-8d52692529b8/go.mod h1:6FWOCx06uh50GClv8S2cfk3asqTJs3qq3ZNRtLZE77I=
36+
github.com/philhofer/fwd v1.1.1 h1:GdGcTjf5RNAxwS4QLsiMzJYj5KEvPJD3Abr261yRQXQ=
37+
github.com/philhofer/fwd v1.1.1/go.mod h1:gk3iGcWd9+svBvR0sR+KPcfE+RNWozjowpeBVG3ZVNU=
3738
github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I=
3839
github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
3940
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
@@ -46,34 +47,50 @@ github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+
4647
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
4748
github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk=
4849
github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
49-
github.com/v3io/v3io-go v0.0.0-20190804122140-7a7baa9fe04ff8591cb4b22270d598b36fc0d49a h1:3xiuabjd2Qy1auDKeqJfkaASmsn0UEtrtwUeJKYZ5O4=
50-
github.com/v3io/v3io-go v0.0.0-20190804122140-7a7baa9fe04ff8591cb4b22270d598b36fc0d49a/go.mod h1:IFb6dJiyvJnOjXUoCoPJ5UViaYjgVYmqJb4fD1qDeLk=
50+
github.com/tinylib/msgp v1.1.6 h1:i+SbKraHhnrf9M5MYmvQhFnbLhAXSDWF8WWsuyRdocw=
51+
github.com/tinylib/msgp v1.1.6/go.mod h1:75BAfg2hauQhs3qedfdDZmWAPcFMAvJE5b9rGOMufyw=
52+
github.com/v3io/v3io-go v0.0.0-20190804122140-7a7baa9fe04f h1:idkl60lTFFDyo2pGm8fEdv4anon/E0UFrWBttWl7yz0=
53+
github.com/v3io/v3io-go v0.0.0-20190804122140-7a7baa9fe04f/go.mod h1:IFb6dJiyvJnOjXUoCoPJ5UViaYjgVYmqJb4fD1qDeLk=
5154
github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw=
5255
github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc=
5356
github.com/valyala/fasthttp v1.2.0/go.mod h1:4vX61m6KN+xDduDNwXrhIAVZaZaZiQ1luJk8LWSxF3s=
5457
github.com/valyala/fasthttp v1.4.0 h1:PuaTGZIw3mjYhhhbVbCQp8aciRZN9YdoB7MGX9Ko76A=
5558
github.com/valyala/fasthttp v1.4.0/go.mod h1:4vX61m6KN+xDduDNwXrhIAVZaZaZiQ1luJk8LWSxF3s=
5659
github.com/valyala/tcplisten v0.0.0-20161114210144-ceec8f93295a/go.mod h1:v3UYOV9WzVtRmSR+PDvWpU/qWl4Wa5LApYYX4ZtKbio=
60+
github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
5761
go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
5862
go.uber.org/atomic v1.4.0 h1:cxzIVoETapQEqDhQu3QfnvXAV4AlzcvUCxkVUFw3+EU=
5963
go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
6064
go.uber.org/multierr v1.1.0 h1:HoEmRHQPVSqub6w2z2d2EOVs2fjyFRGyofhKuyDq0QI=
6165
go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0=
6266
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
63-
golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4 h1:HuIa8hRrWRSrqYzx1qI49NNxhdi2PrY7gxVSq1JjLDc=
64-
golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
67+
golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
68+
golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 h1:psW17arqaxU48Z5kZ0CQnkZWQJsqcURM6tKiBApRjXI=
69+
golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
70+
golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
6571
golang.org/x/net v0.0.0-20180911220305-26e67e76b6c3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
6672
golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
6773
golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
68-
golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7 h1:fHDIZ2oxGnUZRN6WgWFCbYBjH9uqVPRCUVUDhs0wnbA=
69-
golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
74+
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
75+
golang.org/x/net v0.0.0-20201021035429-f5854403a974 h1:IX6qOQeG5uLjB/hjjwjedwfjND0hgjPMMyO1RoIXQNI=
76+
golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
77+
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
78+
golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
7079
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
7180
golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
72-
golang.org/x/sys v0.0.0-20190412213103-97732733099d h1:+R4KGOnez64A81RvjARKc4UT5/tI9ujCIVX+P5KiHuI=
7381
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
74-
golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg=
82+
golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f h1:+Nyd8tzPX9R7BWHguqsrbFdRx3WQ/1ib8I44HXV5yTA=
83+
golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
7584
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
85+
golang.org/x/text v0.3.3 h1:cokOdA+Jmi5PJGXLlLllQSgYigAEfHXJAERHVMaCc2k=
86+
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
87+
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
7688
golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
89+
golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
90+
golang.org/x/tools v0.0.0-20201022035929-9cf592e881e9/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
91+
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
92+
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
93+
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
7794
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
7895
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
7996
gopkg.in/ini.v1 v1.46.0 h1:VeDZbLYGaupuvIrsYCEOe/L/2Pcs5n7hdO1ZTjporag=

0 commit comments

Comments
 (0)