introduce ObjectKeyPrefix

Author: discentem

internal/cli/BUILD.bazel

@@ -14,6 +14,7 @@ go_library(
     deps = [
         "//config",
         "//metadata",
+        "//objects",
         "//program",
         "//stores",
         "@com_github_azure_azure_sdk_for_go_sdk_storage_azblob//:azblob",

internal/cli/helpers.go

@@ -42,20 +42,19 @@ func removePathPrefix(objects []string, prefix string) ([]string, error) {
 }
 
 func initStoreFromConfig(ctx context.Context, cfg config.Config, fsys afero.Fs, opts stores.Options) (stores.Store, error) {
-	var s stores.Store
 	switch cfg.StoreType {
 	case stores.StoreTypeS3:
 		s3, err := stores.NewS3StoreClient(ctx, fsys, opts)
 		if err != nil {
 			return nil, fmt.Errorf("improper stores.S3Client init: %v", err)
 		}
-		s = stores.Store(s3)
+		return stores.Store(s3), nil
 	case stores.StoreTypeGCS:
 		gcs, err := stores.NewGCSStoreClient(ctx, fsys, opts)
 		if err != nil {
 			return nil, fmt.Errorf("improper stores.GCSClient init: %v", err)
 		}
-		s = stores.Store(gcs)
+		return stores.Store(gcs), nil
 	case stores.StoreTypeAzureBlob:
 		az, err := stores.NewAzureBlobStore(
 			ctx,
@@ -66,7 +65,7 @@ func initStoreFromConfig(ctx context.Context, cfg config.Config, fsys afero.Fs,
 		if err != nil {
 			return nil, fmt.Errorf("improper stores.AzureBlobStore init: %v", err)
 		}
-		s = stores.Store(az)
+		return stores.Store(az), nil
 	case stores.StoreTypeGoPlugin:
 		// FIXME: allow specifying command arguments for plugin
 		ps, err := stores.NewPluggableStore(ctx, opts)
@@ -77,12 +76,10 @@ func initStoreFromConfig(ctx context.Context, cfg config.Config, fsys afero.Fs,
 	default:
 		return nil, fmt.Errorf("type %s is not supported", cfg.StoreType)
 	}
-
-	return s, nil
 }
 
 func setLoggerOpts() {
-	if vv {
+	if VV {
 		logger.SetLevel(2)
 	}
 	logger.SetFlags(log.LUTC)

internal/cli/init.go

@@ -34,6 +34,9 @@ func initCmd() *cobra.Command {
 	initCmd.PersistentFlags().String("region", "us-east-1", "Default region for the storage backend")
 	initCmd.PersistentFlags().String("store_type", "", "Storage backend to use")
 	initCmd.PersistentFlags().String("plugin_address", "", "Address for go-plugin that provides implementation for Store")
+	initCmd.PersistentFlags().String("object_key_prefix",
+		"",
+		"Prefixed added to backend upload and retrieve requests. This does not affect the location of written metadata for objects.")
 
 	return initCmd
 }
@@ -63,6 +66,9 @@ func initPreExecFn(cmd *cobra.Command, args []string) error {
 	if err := viper.BindPFlag("plugin_address", cmd.PersistentFlags().Lookup("plugin_address")); err != nil {
 		return errors.New("Failed to bind plugin_address to viper")
 	}
+	if err := viper.BindPFlag("object_key_prefix", cmd.PersistentFlags().Lookup("object_key_prefix")); err != nil {
+		return errors.New("Failed to bind object_key_prefix to viper")
+	}
 
 	return nil
 }

internal/cli/root.go

@@ -16,7 +16,7 @@ import (
 var (
 	// These vars are available to every sub command
 	debug bool
-	vv    bool
+	VV    bool
 
 	// TODO (@radsec) Update this to be dynamic with GH action on new release and tagging....
 	version string = "development"
@@ -68,13 +68,13 @@ func rootCmd() *cobra.Command {
 
 	// At the rootCmd level, set these global flags that will be available to downstream cmds
 	rootCmd.PersistentFlags().BoolVar(&debug, "debug", false, "Run in debug mode")
-	rootCmd.PersistentFlags().BoolVar(&vv, "vv", false, "Run in verbose logging mode")
+	rootCmd.PersistentFlags().BoolVar(&VV, "vv", false, "Run in verbose logging mode")
 
 	// Defaults set here will be used if they do not exist in the config file
 	viper.SetDefault("store_type", stores.StoreTypeUndefined)
 	viper.SetDefault("metadata_file_extension", metadata.MetadataFileExtension)
 
-	if vv {
+	if VV {
 		logger.SetLevel(2)
 	}
 

internal/cli/upload.go

@@ -13,6 +13,7 @@ import (
 
 	"github.com/discentem/cavorite/config"
 	"github.com/discentem/cavorite/metadata"
+	cavoriteObjLib "github.com/discentem/cavorite/objects"
 	"github.com/discentem/cavorite/program"
 	"github.com/discentem/cavorite/stores"
 )
@@ -70,7 +71,18 @@ func upload(ctx context.Context, fsys afero.Fs, s stores.Store, objects ...strin
 
 	logger.Infof("Uploading to: %s", opts.BackendAddress)
 	logger.Infof("Uploading file: %s", objects)
-	if err := s.Upload(ctx, objects...); err != nil {
+
+	var derivedKeys []string
+	prefixOp := cavoriteObjLib.AddPrefixToKey{Prefix: opts.ObjectKeyPrefix}
+	derivedKeys = objects
+	if opts.ObjectKeyPrefix != "" {
+		derivedKeys = cavoriteObjLib.ModifyMultipleKeys(
+			prefixOp,
+			objects...,
+		)
+	}
+
+	if err := s.Upload(ctx, derivedKeys...); err != nil {
 		logger.Error(err)
 		return fmt.Errorf("%w for %v", ErrUpload, objects)
 	}
@@ -87,11 +99,14 @@ func upload(ctx context.Context, fsys afero.Fs, s stores.Store, objects ...strin
 			errResult = multierr.Append(err)
 			continue
 		}
+		mon := prefixOp.Modify(obj)
+		fmt.Println("mon:", mon)
 		err = metadata.WriteToFsys(metadata.FsysWriteRequest{
-			Object:    obj,
-			Fsys:      fsys,
-			Fi:        f,
-			Extension: opts.MetadataFileExtension,
+			Object:       mon,
+			Fsys:         fsys,
+			Fi:           f,
+			MetadataPath: obj,
+			Extension:    opts.MetadataFileExtension,
 		})
 		if err != nil {
 			errResult = multierr.Append(fmt.Errorf("%w for %s", ErrWriteMetadataToFsys, obj))

internal/cli/upload_test.go

@@ -3,6 +3,7 @@ package cli
 import (
 	"context"
 	"fmt"
+	"os"
 	"testing"
 	"time"
 
@@ -45,6 +46,7 @@ type simpleStore struct {
 	sourceFsys afero.Fs
 	// bucketFsys acts as remote artifact storage, where objects will be uploaded
 	bucketFsys afero.Fs
+	options    stores.Options
 }
 
 func (s simpleStore) Upload(ctx context.Context, objects ...string) error {
@@ -54,10 +56,7 @@ func (s simpleStore) Retrieve(ctx context.Context, objects ...string) error {
 	return nil
 }
 func (s simpleStore) GetOptions() (stores.Options, error) {
-	return stores.Options{
-		MetadataFileExtension: "cfile",
-		BackendAddress:        "simpleStore/Test",
-	}, nil
+	return s.options, nil
 }
 
 func (s simpleStore) Close() error {
@@ -84,14 +83,22 @@ func TestUpload(t *testing.T) {
 	sStore := simpleStore{
 		sourceFsys: *sourceFsys,
 		bucketFsys: *bucket,
+		options: stores.Options{
+			MetadataFileExtension: "cfile",
+			BackendAddress:        "simpleStore/Test",
+		},
 	}
 	err = upload(context.Background(), *sourceFsys, sStore, objs...)
 	assert.NoError(t, err)
 
+	err = testutils.WalkFs(*sourceFsys, os.Stdout)
+	require.NoError(t, err)
+
 	sopts, err := sStore.GetOptions()
 	assert.NoError(t, err)
 	for _, f := range objs {
-		b, _ := afero.ReadFile(sStore.sourceFsys, fmt.Sprintf("%s.%s", f, sopts.MetadataFileExtension))
+		b, err := afero.ReadFile(sStore.sourceFsys, fmt.Sprintf("%s.%s", f, sopts.MetadataFileExtension))
+		require.NoError(t, err)
 		assert.Equal(t, fmt.Sprintf(`{
  "name": "%s",
  "checksum": "35bafb1ce99aef3ab068afbaabae8f21fd9b9f02d3a9442e364fa92c0b3eeef0",
@@ -102,6 +109,57 @@ func TestUpload(t *testing.T) {
 	assert.NoError(t, err)
 }
 
+// TestUploadWithPrefix tests whether metadata gets generated correctly
+func TestUploadWithPrefix(t *testing.T) {
+	mTime, _ := time.Parse("2006-01-02T15:04:05.000Z", "2014-11-12T11:45:26.371Z")
+	sourceFsys, err := testutils.MemMapFsWith(map[string]testutils.MapFile{
+		"someFile": {
+			ModTime: &mTime,
+			Content: []byte(`stuff`),
+		},
+		"someOtherFile": {
+			ModTime: &mTime,
+			Content: []byte(`stuff`),
+		},
+	})
+	assert.NoError(t, err)
+	bucket, err := testutils.MemMapFsWith(map[string]testutils.MapFile{})
+	assert.NoError(t, err)
+	sStore := simpleStore{
+		sourceFsys: *sourceFsys,
+		bucketFsys: *bucket,
+		options: stores.Options{
+			MetadataFileExtension: "cfile",
+			BackendAddress:        "simpleStore/Test",
+			ObjectKeyPrefix:       "aCoolPrefix",
+		},
+	}
+	err = upload(context.Background(), *sourceFsys, sStore, "someFile", "someOtherFile")
+	require.NoError(t, err)
+
+	err = testutils.WalkFs(*sourceFsys, os.Stdout)
+	require.NoError(t, err)
+
+	sopts, err := sStore.GetOptions()
+	assert.NoError(t, err)
+	for _, f := range []string{"someFile", "someOtherFile"} {
+		b, err := afero.ReadFile(*sourceFsys, fmt.Sprintf("%s.%s", f, sopts.MetadataFileExtension))
+		require.NoError(t, err)
+		var objkey string
+		if sopts.ObjectKeyPrefix != "" {
+			objkey = fmt.Sprintf("%s/%s", sopts.ObjectKeyPrefix, f)
+		} else {
+			objkey = fmt.Sprintf(f)
+		}
+		expected := fmt.Sprintf(`{
+  "name": "%s",
+  "checksum": "35bafb1ce99aef3ab068afbaabae8f21fd9b9f02d3a9442e364fa92c0b3eeef0",
+  "date_modified": "2014-11-12T11:45:26.371Z"
+}`, objkey)
+		assert.Equal(t, expected, string(b))
+	}
+}
+
 // TestUploadPartialFail tests whether metadata generation will succeed for n+1 even if n fails
 func TestUploadPartialFail(t *testing.T) {
 	mTime, _ := time.Parse("2006-01-02T15:04:05.000Z", "2014-11-12T11:45:26.371Z")

metadata/metadata.go

@@ -72,14 +72,18 @@ func ParseCfileWithExtension(fsys afero.Fs, obj, ext string) (*ObjectMetaData, e
 }
 
 type FsysWriteRequest struct {
-	Object    string
-	Fsys      afero.Fs
-	Fi        afero.File
-	Extension string
+	Object       string
+	Fsys         afero.Fs
+	Fi           afero.File
+	MetadataPath string
+	Extension    string
 }
 
 // WriteToFsys generates Cavorite metadata for req.Object and writes it to req.Fsys
 func WriteToFsys(req FsysWriteRequest) (err error) {
+	if req.MetadataPath == "" {
+		return fmt.Errorf("req.MetadataPath cannot be %q", "")
+	}
 	logger.V(2).Infof("object: %s", req.Object)
 	// generate metadata
 	m, err := GenerateFromFile(req.Fi)
@@ -93,7 +97,7 @@ func WriteToFsys(req FsysWriteRequest) (err error) {
 		return err
 	}
 	// Write metadata to disk
-	metadataPath := fmt.Sprintf("%s.%s", req.Object, req.Extension)
+	metadataPath := fmt.Sprintf("%s.%s", req.MetadataPath, req.Extension)
 	logger.V(2).Infof("writing metadata to %s", metadataPath)
 	if err := afero.WriteFile(req.Fsys, metadataPath, blob, 0644); err != nil {
 		return err

objects/BUILD.bazel

@@ -0,0 +1,15 @@
+load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test")
+
+go_library(
+    name = "objects",
+    srcs = ["objects.go"],
+    importpath = "github.com/discentem/cavorite/objects",
+    visibility = ["//visibility:public"],
+)
+
+go_test(
+    name = "objects_test",
+    srcs = ["objects_test.go"],
+    embed = [":objects"],
+    deps = ["@com_github_stretchr_testify//assert"],
+)

objects/objects.go

@@ -0,0 +1,26 @@
+package objects
+
+import "fmt"
+
+type KeyModifier interface {
+	Modify(string) string
+}
+
+type AddPrefixToKey struct {
+	Prefix string
+}
+
+func (p AddPrefixToKey) Modify(original string) string {
+	if p.Prefix == "" {
+		return original
+	}
+	return fmt.Sprintf("%s/%s", p.Prefix, original)
+}
+
+func ModifyMultipleKeys(modder KeyModifier, originalKeys ...string) []string {
+	newKeys := []string{}
+	for _, key := range originalKeys {
+		newKeys = append(newKeys, modder.Modify(key))
+	}
+	return newKeys
+}

objects/objects_test.go

@@ -0,0 +1,42 @@
+package objects
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestAddPrefixModify(t *testing.T) {
+	tests := []struct {
+		name        string
+		originalKey string
+		prefix      string
+		expected    string
+	}{
+		{
+			name:        "single level key",
+			originalKey: "thing",
+			prefix:      "stuff",
+			expected:    "stuff/thing",
+		},
+		{
+			name:        "multi level key",
+			originalKey: "apple/banana/strawberry",
+			prefix:      "cake",
+			expected:    "cake/apple/banana/strawberry",
+		},
+		{
+			name:        "empty prefix",
+			originalKey: "whatever/thing/stuff",
+			prefix:      "",
+			expected:    "whatever/thing/stuff",
+		},
+	}
+
+	for _, test := range tests {
+		modder := AddPrefixToKey{Prefix: test.prefix}
+		actual := modder.Modify(test.originalKey)
+		assert.Equal(t, test.expected, actual)
+	}
+
+}

stores/BUILD.bazel

@@ -43,6 +43,7 @@ go_test(
         "azure_test.go",
         "gcs_test.go",
         "s3_test.go",
+        "stores_test.go",
     ],
     embed = [":stores"],
     deps = [

stores/options.go

@@ -7,6 +7,12 @@ type Options struct {
 	PluginAddress         string `json:"plugin_address,omitempty" mapstructure:"plugin_address"`
 	MetadataFileExtension string `json:"metadata_file_extension" mapstructure:"metadata_file_extension"`
 	Region                string `json:"region" mapstructure:"region"`
+	/*
+		If ObjectKeyPrefix is set to "team-bucket", and the initialized backend supports it,
+			- `cavorite upload whatever/thing` will be written to `team-bucket/whatever/thing`
+			- `cavorite retrieve whatever/thing` will request `team-bucket/whatever/thing`
+	*/
+	ObjectKeyPrefix string `json:"object_key_prefix" mapstructure:"object_key_prefix"`
 }
 
 var ErrMetadataFileExtensionEmpty = fmt.Errorf("options.MetadatafileExtension cannot be %q", "")