integrate bindetector into upload via fileutils helpers

Author: rdiers

internal/cli/BUILD.bazel

@@ -14,6 +14,7 @@ go_library(
     visibility = ["//:__subpackages__"],
     deps = [
         "//internal/config",
+        "//internal/fileutils",
         "//internal/metadata",
         "//internal/program",
         "//internal/stores",

internal/cli/upload.go

@@ -4,6 +4,7 @@ import (
 	"errors"
 	"fmt"
 
+	"github.com/discentem/cavorite/internal/fileutils"
 	"github.com/discentem/cavorite/internal/program"
 	"github.com/google/logger"
 	"github.com/spf13/afero"
@@ -33,6 +34,7 @@ func uploadCmd() *cobra.Command {
 		RunE: uploadFn,
 	}
 
+	// uploadCmd.LocalFlags().Bool("store_type", "", "Storage backend to use")
 	return uploadCmd
 }
 
@@ -75,7 +77,10 @@ func uploadFn(cmd *cobra.Command, objects []string) error {
 	}
 
 	logger.Infof("Uploading to: %s", s.GetOptions().BackendAddress)
-	logger.Infof("Uploading file: %s", objects)
+	objects, err = fileutils.GetBinariesWalkPath(fsys, objects)
+	if err != nil {
+		return fmt.Errorf("walking binaries path error: %w", err)
+	}
 	if err := s.Upload(cmd.Context(), objects...); err != nil {
 		return err
 	}

internal/fileutils/BUILD.bazel

@@ -1,9 +1,20 @@
-load("@io_bazel_rules_go//go:def.bzl", "go_library")
+load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test")
 
 go_library(
     name = "fileutils",
     srcs = ["fileutils.go"],
     importpath = "github.com/discentem/cavorite/internal/fileutils",
     visibility = ["//:__subpackages__"],
-    deps = ["@com_github_spf13_afero//:afero"],
+    deps = [
+        "//internal/bindetector",
+        "@com_github_google_logger//:logger",
+        "@com_github_spf13_afero//:afero",
+    ],
+)
+
+go_test(
+    name = "fileutils_test",
+    srcs = ["fileutils_test.go"],
+    embed = [":fileutils"],
+    deps = ["@com_github_stretchr_testify//assert"],
 )

internal/fileutils/fileutils.go

@@ -3,7 +3,10 @@ package fileutils
 import (
 	"fmt"
 	"io"
+	"io/fs"
 
+	"github.com/discentem/cavorite/internal/bindetector"
+	"github.com/google/logger"
 	"github.com/spf13/afero"
 )
 
@@ -23,3 +26,37 @@ func BytesFromAferoFile(f afero.File) ([]byte, error) {
 	}
 	return b, nil
 }
+
+func GetBinariesWalkPath(fsys afero.Fs, objects []string) ([]string, error) {
+	afs := afero.Afero{Fs: fsys}
+
+	var paths []string
+	logger.V(2).Infof("walking objects: %s", objects)
+	for _, object := range objects {
+		err := afs.Walk(object, func(p string, info fs.FileInfo, err error) error {
+			logger.V(2).Infof("walkfunc() p: %s", p)
+			isDir, err := afs.IsDir(p)
+			if err != nil {
+				logger.Errorf("walkfunc() fsys.IsDir error: %v", err)
+				return err
+			}
+			logger.V(2).Infof("walkfunc() isDir: %t", isDir)
+
+			if isDir {
+				return nil
+			}
+
+			if bindetector.IsBinary(p) {
+				logger.V(2).Infof("walkfunc() - bindetector().IsBinary appending path: %s", p)
+				paths = append(paths, p)
+			}
+
+			return nil
+		})
+		if err != nil {
+			return paths, err
+		}
+	}
+
+	return paths, nil
+}

internal/fileutils/fileutils_test.go

@@ -0,0 +1,15 @@
+package fileutils
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestGetBinariesWalkPath(t *testing.T) {
+	// expectedBinariesFromPath := []string{
+	// 	"abc.pkg",
+	// 	"zxf.pkg",
+	// }
+	assert.True(t, true)
+}

internal/stores/azure.go

@@ -49,6 +49,7 @@ func (s *AzureBlobStore) Upload(ctx context.Context, objects ...string) error {
 			return err
 		}
 		containerName := path.Base(s.Options.BackendAddress)
+		logger.Infof(`Uploading "%s" to Azure`, o)
 		_, err = s.containerClient.UploadStream(
 			ctx,
 			containerName,

internal/stores/gcs.go

@@ -107,6 +107,7 @@ func (s *GCSStore) Upload(ctx context.Context, objects ...string) error {
 			// return multiple errors
 			return multErr
 		}
+		logger.Infof(`Uploading "%s" to GCS`, o)
 		_, err = io.Copy(wc, f)
 		if err != nil {
 			multErr = multierror.Append(multErr, fmt.Errorf("io.Copy() error: %w", err))

internal/stores/s3.go

@@ -160,22 +160,25 @@ func (s *S3Store) Upload(ctx context.Context, objects ...string) error {
 			logger.Errorf("error encountered parsing backend address: %v", err)
 			return err
 		}
-		obj := s3.PutObjectInput{
+
+		putObjInput := s3.PutObjectInput{
 			Bucket: aws.String(s3BucketName),
 			Key:    aws.String(o),
 			Body:   f,
 		}
-		out, err := s.s3Uploader.Upload(ctx, &obj)
+		logger.Infof(`Uploading "%s" to S3`, o)
+		uploadOutput, err := s.s3Uploader.Upload(ctx, &putObjInput)
 		if err != nil {
 			if err := cleanupFn(); err != nil {
 				return fmt.Errorf("cleanup() failed after Upload failure: %w", err)
 			}
-			logger.Error(out)
+			logger.Error(uploadOutput)
 			return err
 		}
 		if err := f.Close(); err != nil {
 			return err
 		}
+		logger.Infof(`Upload Complete for "%s" to S3`, o)
 	}
 	return nil
 }
@@ -240,7 +243,7 @@ func (s *S3Store) Retrieve(ctx context.Context, objects ...string) error {
 
 func (s *S3Store) getBucketName() (string, error) {
 	var bucketName string
-	logger.Infof("s3store getBucketName: backend address: %s", s.Options.BackendAddress)
+	logger.V(2).Infof("s3store getBucketName: backend address: %s", s.Options.BackendAddress)
 	switch {
 	case strings.HasPrefix(s.Options.BackendAddress, "s3://"):
 		s3BucketUrl, err := url.Parse(s.Options.BackendAddress)