Skip to content

Commit 08365c0

Browse files
adrianwit@gmail.comadrianwit@gmail.com
adrianwit@gmail.com
authored and
adrianwit@gmail.com
committed
optimize cost usage
1 parent a83ad8d commit 08365c0

14 files changed

Lines changed: 130 additions & 78 deletions

File tree

‎gs/copy.go‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@ import (
44
"context"
55
"fmt"
66
"github.com/viant/afs/file"
7+
"github.com/viant/afs/option"
78
"github.com/viant/afs/storage"
89
gstorage "google.golang.org/api/storage/v1"
910
"path"
@@ -24,8 +25,12 @@ func (s *storager) Copy(ctx context.Context, sourcePath, destBucket, destPath st
2425
func (s *storager) copy(ctx context.Context, sourcePath, destBucket, destPath string, options []storage.Option) error {
2526
sourcePath = strings.Trim(sourcePath, "/")
2627
destPath = strings.Trim(destPath, "/")
27-
objectInfo, err := s.get(ctx, sourcePath)
28+
objectInfo, err := s.get(ctx, sourcePath, options)
2829
if isNotFound(err) {
30+
objectOpt := &option.ObjectKind{}
31+
if _, ok := option.Assign(options, objectOpt); ok && objectOpt.File {
32+
return err
33+
}
2934
infoList, err := s.List(ctx, sourcePath)
3035
if err != nil {
3136
return err

‎gs/delete.go‎

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2,12 +2,14 @@ package gs
22

33
import (
44
"context"
5+
"github.com/viant/afs/option"
6+
"github.com/viant/afs/storage"
57
"path"
68
"strings"
79
)
810

911
//Delete removes an resource
10-
func (s *storager) Delete(ctx context.Context, location string) (err error) {
12+
func (s *storager) Delete(ctx context.Context, location string, options ...storage.Option) (err error) {
1113
location = strings.Trim(location, "/")
1214
if location == "" {
1315
call := s.Buckets.Delete(s.bucket)
@@ -16,12 +18,15 @@ func (s *storager) Delete(ctx context.Context, location string) (err error) {
1618
return err
1719
}
1820

19-
2021
call := s.Objects.Delete(s.bucket, location)
2122
call.Context(ctx)
2223
for i := 0; i < maxRetries; i++ {
2324
err = call.Do()
2425
if isNotFound(err) {
26+
objectKind := &option.ObjectKind{}
27+
if _, ok := option.Assign(options, objectKind); ok && objectKind.File {
28+
return err
29+
}
2530
notFound := err
2631
infoList, err := s.List(ctx, location)
2732
if err != nil {

‎gs/exists.go‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -2,13 +2,13 @@ package gs
22

33
import (
44
"context"
5-
"strings"
5+
"github.com/viant/afs/storage"
66
)
77

88
//Exists returns true if object exists
9-
func (s *storager) Exists(ctx context.Context, location string) (bool, error) {
9+
func (s *storager) Exists(ctx context.Context, location string, options ...storage.Option) (bool, error) {
1010
object, err := s.Get(ctx, location)
11-
if err != nil && strings.Contains(strings.ToLower(err.Error()), "not found") {
11+
if isNotFound(err) {
1212
err = nil
1313
}
1414
return object != nil, err

‎gs/get.go‎

Lines changed: 10 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@ import (
1010
)
1111

1212
//Get returns an object for supplied location
13-
func (s *storager) get(ctx context.Context, location string, options ...storage.Option) (os.FileInfo, error) {
13+
func (s *storager) get(ctx context.Context, location string, options []storage.Option) (os.FileInfo, error) {
1414
location = strings.Trim(location, "/")
1515
objectCall := s.Objects.Get(s.bucket, location)
1616
objectCall.Context(ctx)
@@ -23,24 +23,23 @@ func (s *storager) get(ctx context.Context, location string, options ...storage.
2323

2424
//Get returns an object for supplied location
2525
func (s *storager) Get(ctx context.Context, location string, options ...storage.Option) (os.FileInfo, error) {
26-
objectOpt := &option.Object{}
27-
if _, ok := option.Assign(options, &objectOpt); ok && objectOpt.File {
28-
return s.get(ctx, location, options...)
26+
info, err := s.get(ctx, location, options)
27+
if err == nil {
28+
return info, err
2929
}
30-
location = strings.Trim(location, "/")
31-
objectCall := s.Objects.Get(s.bucket, location)
32-
objectCall.Context(ctx)
33-
object, _ := objectCall.Do()
34-
if object != nil {
35-
return newFileInfo(object)
30+
if isNotFound(err) {
31+
objectOpt := &option.ObjectKind{}
32+
if _, ok := option.Assign(options, objectOpt); ok && objectOpt.File {
33+
return nil, err
34+
}
3635
}
3736
options = append(options, option.NewPage(0, 1))
3837
objects, err := s.List(ctx, location, options...)
3938
if err != nil {
4039
return nil, err
4140
}
4241
if len(objects) == 0 {
43-
return nil, errors.Errorf("%v not found", location)
42+
return nil, errors.Errorf("%v %v", location, notFound)
4443
}
4544
return objects[0], err
4645
}

‎gs/helper.go‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -56,7 +56,6 @@ func sleepBeforeRetry() {
5656
time.Sleep(3 * time.Second)
5757
}
5858

59-
6059
//isRetryError returns true if backend error
6160
func isNotFound(err error) bool {
6261
if err == nil {
@@ -67,5 +66,5 @@ func isNotFound(err error) bool {
6766
return true
6867
}
6968
}
70-
return false
69+
return strings.Contains(err.Error(), notFound)
7170
}

‎gs/list.go‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -159,7 +159,6 @@ func (s *storager) listObjects(ctx context.Context, location string, call *gstor
159159
return files, folders, nil
160160
}
161161

162-
163162
//GetListCounter returns count of list operations
164163
func GetListCounter(reset bool) int {
165164
result := atomic.LoadUint64(&listCounter)

‎gs/move.go‎

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -4,13 +4,13 @@ import (
44
"context"
55
"fmt"
66
"github.com/viant/afs/file"
7+
"github.com/viant/afs/option"
78
"github.com/viant/afs/storage"
89
gstorage "google.golang.org/api/storage/v1"
910
"path"
1011
"strings"
1112
)
1213

13-
1414
func (s *storager) Move(ctx context.Context, sourcePath, destBucket, destPath string, options ...storage.Option) (err error) {
1515
for i := 0; i < maxRetries; i++ {
1616
err = s.move(ctx, sourcePath, destBucket, destPath, options)
@@ -22,13 +22,16 @@ func (s *storager) Move(ctx context.Context, sourcePath, destBucket, destPath st
2222
return err
2323
}
2424

25-
2625
func (s *storager) move(ctx context.Context, sourcePath, destBucket, destPath string, options []storage.Option) error {
2726
sourcePath = strings.Trim(sourcePath, "/")
2827
destPath = strings.Trim(destPath, "/")
29-
objectInfo, err := s.get(ctx, sourcePath)
28+
objectInfo, err := s.get(ctx, sourcePath, options)
3029

3130
if isNotFound(err) {
31+
objectOpt := &option.ObjectKind{}
32+
if _, ok := option.Assign(options, objectOpt); ok && objectOpt.File {
33+
return err
34+
}
3235
infoList, err := s.List(ctx, sourcePath)
3336
if err != nil {
3437
return err
@@ -58,4 +61,3 @@ func (s *storager) move(ctx context.Context, sourcePath, destBucket, destPath st
5861
}
5962
return err
6063
}
61-

‎s3/copy.go‎

Lines changed: 18 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -6,32 +6,36 @@ import (
66
"github.com/aws/aws-sdk-go/aws"
77
"github.com/aws/aws-sdk-go/service/s3"
88
"github.com/pkg/errors"
9+
"github.com/viant/afs/option"
910
"github.com/viant/afs/storage"
1011
"path"
1112
"strings"
1213
)
1314

1415
func (s *storager) Copy(ctx context.Context, sourcePath, destBucket, destPath string, options ...storage.Option) error {
15-
1616
sourcePath = strings.Trim(sourcePath, "/")
1717
destPath = strings.Trim(destPath, "/")
18-
infoList, err := s.List(ctx, sourcePath, options...)
19-
if err != nil {
20-
return errors.Wrapf(err, "unable list copy source: gs://%v/%v", s.bucket, sourcePath)
21-
}
22-
if len(infoList) == 0 {
23-
return fmt.Errorf("%v: not found", sourcePath)
24-
}
25-
for i := 1; i < len(infoList); i++ {
26-
name := infoList[i].Name()
27-
if err = s.Copy(ctx, path.Join(sourcePath, name), destBucket, path.Join(destPath, name), options...); err != nil {
18+
_, err := s.get(ctx, sourcePath, options)
19+
if isNotFound(err) {
20+
objectOpt := &option.ObjectKind{}
21+
if _, ok := option.Assign(options, objectOpt); ok && objectOpt.File {
2822
return err
2923
}
30-
}
31-
if infoList[0].IsDir() {
24+
infoList, err := s.List(ctx, sourcePath)
25+
if err != nil {
26+
return err
27+
}
28+
if len(infoList) == 0 {
29+
return fmt.Errorf("%v: not found", sourcePath)
30+
}
31+
for i := 1; i < len(infoList); i++ {
32+
name := infoList[i].Name()
33+
if err = s.Move(ctx, path.Join(sourcePath, name), destBucket, path.Join(destPath, name)); err != nil {
34+
return err
35+
}
36+
}
3237
return nil
3338
}
34-
3539
_, err = s.S3.CopyObjectWithContext(ctx, &s3.CopyObjectInput{
3640
CopySource: aws.String(s.bucket + "/" + sourcePath),
3741
Key: &destPath,

‎s3/delete.go‎

Lines changed: 13 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -3,11 +3,23 @@ package s3
33
import (
44
"context"
55
"github.com/aws/aws-sdk-go/service/s3"
6+
"github.com/viant/afs/option"
7+
"github.com/viant/afs/storage"
68
"path"
79
)
810

911
//Delete removes an resource
10-
func (s *storager) Delete(ctx context.Context, location string) error {
12+
func (s *storager) Delete(ctx context.Context, location string, options ...storage.Option) error {
13+
_, err := s.S3.DeleteObjectWithContext(ctx, &s3.DeleteObjectInput{
14+
Bucket: &s.bucket,
15+
Key: &location,
16+
})
17+
if isNotFound(err) {
18+
objectKind := &option.ObjectKind{}
19+
if _, ok := option.Assign(options, objectKind); ok && objectKind.File {
20+
return err
21+
}
22+
}
1123
infoList, err := s.List(ctx, location)
1224
if err != nil {
1325
return err
@@ -17,9 +29,5 @@ func (s *storager) Delete(ctx context.Context, location string) error {
1729
return err
1830
}
1931
}
20-
_, err = s.S3.DeleteObjectWithContext(ctx, &s3.DeleteObjectInput{
21-
Bucket: &s.bucket,
22-
Key: &location,
23-
})
2432
return err
2533
}

‎s3/exists.go‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -2,13 +2,13 @@ package s3
22

33
import (
44
"context"
5-
"strings"
5+
"github.com/viant/afs/storage"
66
)
77

88
//Exists returns true if object exists
9-
func (s *storager) Exists(ctx context.Context, location string) (bool, error) {
9+
func (s *storager) Exists(ctx context.Context, location string, options ...storage.Option) (bool, error) {
1010
object, err := s.Get(ctx, location)
11-
if err != nil && (strings.Contains(err.Error(), noSuchKeyMessage) || strings.Contains(err.Error(), doesNotExistsMessage)) {
11+
if isNotFound(err) {
1212
err = nil
1313
}
1414
return object != nil, err

0 commit comments

Comments
 (0)