feat: add support for az table storage (#23)

* feat: add support for az table storage

for non-sensitive config items support for AZ TableStorage has been added

semver: feature

closes #22

* chore(docs): update readme

* fix: add missing implementation

* fix: docs

* fix: unit test for success empty
+semver: feature
+semver: feat
+semver:minor

* fix: add value concept as a property to aztablestore
add readme around it

* fix: clean up comments and delete unusued

* fix: normalize getter/setter

* fix: update docs

* fix: tests for adjust non exchanged/errored tokens

Author: dnitsch

Makefile

@@ -10,7 +10,8 @@ LDFLAGS := -ldflags="-s -w -X \"github.com/$(OWNER)/$(NAME)/cmd/configmanager.Ve
 .PHONY: test test_ci tidy install cross-build 
 
 test: test_prereq
-	go test `go list ./... | grep -v */generated/` -v -buildvcs=false -mod=readonly -coverprofile=.coverage/out | go-junit-report > .coverage/report-junit.xml && \
+	go test `go list ./... | grep -v */generated/` -v -buildvcs=false -mod=readonly -coverprofile=.coverage/out ; \
+	cat .coverage/out | go-junit-report > .coverage/report-junit.xml && \
 	gocov convert .coverage/out | gocov-xml > .coverage/report-cobertura.xml
 
 test_ci:

README.md

@@ -28,6 +28,9 @@ Currently supported variable and secrets implementations:
 - [AzureKeyvault Secrets](https://azure.microsoft.com/en-gb/products/key-vault/)
 	- Implementation Indicator: `AZKVSECRET`
 	- see [Special consideration for AZKVSECRET](#special-consideration-for-azkvsecret) around how to structure the token in this case.
+- [Azure TableStorage](https://azure.microsoft.com/en-gb/products/storage/tables/)
+	- Implementation Indicator: `AZTABLESTORE`
+	- see [Special consideration for AZTABLESTORE](#special-consideration-for-aztablestore) around how to structure the token in this case.
 - [GCP Secrets](https://cloud.google.com/secret-manager)
 	- Implementation Indicator: `GCPSECRETS`
 - [Hashicorp Vault](https://developer.hashicorp.com/vault/docs/secrets/kv)
@@ -177,6 +180,28 @@ For Azure KeyVault the first part of the token needs to be the name of the vault
 
 > The preceeding slash to the vault name is optional - `AZKVSECRET#/test-vault/no-slash-token-1` and `AZKVSECRET#test-vault/no-slash-token-1` will both identify the vault of name `test-vault`
 
+### Special consideration for AZTABLESTORE
+
+The token itself must contain all of the following properties, so that it would look like this `AZTABLESTORE://STORAGE_ACCOUNT_NAME/TABLE_NAME/PARTITION_KEY/ROW_KEY`:
+
+- Storage account name [`STORAGE_ACCOUNT_NAME`]
+- Table Name [`TABLE_NAME`]
+	- > It might make sense to make this table global to the domain or project 
+- Partition Key [`PARTITION_KEY`]
+	- > This could correspond to the component/service name
+- Row Key [`ROW_KEY`]
+	- > This could correspond to the property itself or a group of properties
+	- > e.g. `AZTABLESTORE://globalconfigstorageaccount/domainXyz/serviceXyz/db` => `{"value":{"host":"foo","port":1234,"enabled":true}}`
+	- > It will continue to work the same way with additional keyseparators inside values.
+
+> NOTE: if you store a more complex object inside a top level `value` property this will reduce the number of columns and normalize the table - **THE DATA INSIDE THE VALUE MUST BE JSON PARSEABLE**
+
+All the usual token rules apply e.g. of `keySeparator`
+
+`AZTABLESTORE://account/app1Config/db/config` => `{host: foo.bar, port: 8891}`
+
+`AZTABLESTORE://account/app1Config/db/config|host` => `foo.bar`
+
 ### Special consideration for HashicorpVault
 
 For HashicorpVault the first part of the token needs to be the name of the mountpath. In Dev Vaults this is `"secret"`,

configmanager.go

@@ -25,6 +25,7 @@ func (c *ConfigManager) Retrieve(tokens []string, config generator.GenVarsConfig
 	return retrieve(tokens, gv)
 }
 
+// GenerateAPI
 type GenerateAPI interface {
 	Generate(tokens []string) (generator.ParsedMap, error)
 }
@@ -34,7 +35,7 @@ func retrieve(tokens []string, gv GenerateAPI) (generator.ParsedMap, error) {
 }
 
 // RetrieveWithInputReplaced parses given input against all possible token strings
-// using regex to grab a list of found tokens in the given string and return the replaced string
+// using regex to grab a list of found tokens in the given string and returns the replaced string
 func (c *ConfigManager) RetrieveWithInputReplaced(input string, config generator.GenVarsConfig) (string, error) {
 	gv := generator.NewGenerator().WithConfig(&config)
 	return retrieveWithInputReplaced(input, gv)
@@ -98,12 +99,12 @@ type CMRetrieveWithInputReplacediface interface {
 	RetrieveWithInputReplaced(input string, config generator.GenVarsConfig) (string, error)
 }
 
-// @deprecated
-// left for compatibility
 // KubeControllerSpecHelper is a helper method, it marshalls an input value of that type into a string and passes it into the relevant configmanger retrieve method
-// and returns the unmarshalled object back
+// and returns the unmarshalled object back.
 //
-// It accepts a DI of configmanager and the config (for testability) to replace all occurences of replaceable tokens inside a Marshalled string of that type
+// # It accepts a DI of configmanager and the config (for testability) to replace all occurences of replaceable tokens inside a Marshalled string of that type
+//
+// Deprecated: Left for compatibility reasons
 func KubeControllerSpecHelper[T any](inputType T, cm CMRetrieveWithInputReplacediface, config generator.GenVarsConfig) (*T, error) {
 	outType := new(T)
 	rawBytes, err := json.Marshal(inputType)

configmanager_test.go

@@ -218,14 +218,12 @@ db:
 }
 
 type MockCfgMgr struct {
-	RetrieveWithInputReplacedTest func(input string, config generator.GenVarsConfig) (string, error)
+	retrieveInput func(input string, config generator.GenVarsConfig) (string, error)
+	// retrieve func(input string, config generator.GenVarsConfig) (string, error)
 }
 
 func (m *MockCfgMgr) RetrieveWithInputReplaced(input string, config generator.GenVarsConfig) (string, error) {
-	if m.RetrieveWithInputReplacedTest != nil {
-		return m.RetrieveWithInputReplacedTest(input, config)
-	}
-	return "", nil
+	return m.retrieveInput(input, config)
 }
 
 func (m *MockCfgMgr) Insert(force bool) error {
@@ -280,7 +278,7 @@ func Test_KubeControllerSpecHelper(t *testing.T) {
 			},
 			cfmgr: func(t *testing.T) mockConfigManageriface {
 				mcm := &MockCfgMgr{}
-				mcm.RetrieveWithInputReplacedTest = func(input string, config generator.GenVarsConfig) (string, error) {
+				mcm.retrieveInput = func(input string, config generator.GenVarsConfig) (string, error) {
 					return `{"foo":"baz","bar":"quz"}`, nil
 				}
 				return mcm
@@ -298,7 +296,7 @@ func Test_KubeControllerSpecHelper(t *testing.T) {
 			},
 			cfmgr: func(t *testing.T) mockConfigManageriface {
 				mcm := &MockCfgMgr{}
-				mcm.RetrieveWithInputReplacedTest = func(input string, config generator.GenVarsConfig) (string, error) {
+				mcm.retrieveInput = func(input string, config generator.GenVarsConfig) (string, error) {
 					return `{"foo":"baz2","bar":"quz"}`, nil
 				}
 				return mcm
@@ -353,7 +351,7 @@ func Test_KubeControllerComplex(t *testing.T) {
 			},
 			cfmgr: func(t *testing.T) mockConfigManageriface {
 				mcm := &MockCfgMgr{}
-				mcm.RetrieveWithInputReplacedTest = func(input string, config generator.GenVarsConfig) (string, error) {
+				mcm.retrieveInput = func(input string, config generator.GenVarsConfig) (string, error) {
 					return `{"foo":"baz","bar":"quz", "lol":{"bla":"booo","another":{"number": 1235, "float": 123.09}}}`, nil
 				}
 				return mcm
@@ -407,7 +405,7 @@ func Test_YamlRetrieveMarshalled(t *testing.T) {
 			},
 			cfmgr: func(t *testing.T) mockConfigManageriface {
 				mcm := &MockCfgMgr{}
-				mcm.RetrieveWithInputReplacedTest = func(input string, config generator.GenVarsConfig) (string, error) {
+				mcm.retrieveInput = func(input string, config generator.GenVarsConfig) (string, error) {
 					return `{"foo":"baz","bar":"quz", "lol":{"bla":"booo","another":{"number": 1235, "float": 123.09}}}`, nil
 				}
 				return mcm
@@ -426,7 +424,7 @@ func Test_YamlRetrieveMarshalled(t *testing.T) {
 			},
 			cfmgr: func(t *testing.T) mockConfigManageriface {
 				mcm := &MockCfgMgr{}
-				mcm.RetrieveWithInputReplacedTest = func(input string, config generator.GenVarsConfig) (string, error) {
+				mcm.retrieveInput = func(input string, config generator.GenVarsConfig) (string, error) {
 					return `{"foo":"baz","bar":"quz", "lol":{"bla":"","another":{"number": 0, "float": 0}}}`, nil
 				}
 				return mcm
@@ -448,6 +446,58 @@ func Test_YamlRetrieveMarshalled(t *testing.T) {
 	}
 }
 
+func Test_YamlRetrieveMarshalled_errored(t *testing.T) {
+	tests := []struct {
+		name     string
+		testType *testNestedStruct
+		expect   error
+		cfmgr    func(t *testing.T) mockConfigManageriface
+	}{
+		{
+			name: "complex struct - complete",
+			testType: &testNestedStruct{
+				Foo: testTokenAWS,
+				Bar: "quz",
+				Lol: testLol{
+					Bla: "booo",
+					Another: testAnotherNEst{
+						Number: 1235,
+						Float:  123.09,
+					},
+				},
+			},
+			// expect: testNestedStruct{
+			// 	Foo: "baz",
+			// 	Bar: "quz",
+			// 	Lol: testLol{
+			// 		Bla: "booo",
+			// 		Another: testAnotherNEst{
+			// 			Number: 1235,
+			// 			Float:  123.09,
+			// 		},
+			// 	},
+			// },
+			cfmgr: func(t *testing.T) mockConfigManageriface {
+				mcm := &MockCfgMgr{}
+				mcm.retrieveInput = func(input string, config generator.GenVarsConfig) (string, error) {
+					return ``, fmt.Errorf("%s", "error decoding")
+				}
+				return mcm
+			},
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			config := generator.NewConfig().WithTokenSeparator("://")
+
+			_, err := RetrieveMarshalledYaml(tt.testType, tt.cfmgr(t), *config)
+			if err == nil {
+				t.Errorf(testutils.TestPhrase, nil, err.Error())
+			}
+		})
+	}
+}
+
 func Test_RetrieveMarshalledJson(t *testing.T) {
 	tests := []struct {
 		name     string
@@ -481,7 +531,7 @@ func Test_RetrieveMarshalledJson(t *testing.T) {
 			},
 			cfmgr: func(t *testing.T) mockConfigManageriface {
 				mcm := &MockCfgMgr{}
-				mcm.RetrieveWithInputReplacedTest = func(input string, config generator.GenVarsConfig) (string, error) {
+				mcm.retrieveInput = func(input string, config generator.GenVarsConfig) (string, error) {
 					return `{"foo":"baz","bar":"quz", "lol":{"bla":"booo","another":{"number": 1235, "float": 123.09}}}`, nil
 				}
 				return mcm
@@ -500,7 +550,7 @@ func Test_RetrieveMarshalledJson(t *testing.T) {
 			},
 			cfmgr: func(t *testing.T) mockConfigManageriface {
 				mcm := &MockCfgMgr{}
-				mcm.RetrieveWithInputReplacedTest = func(input string, config generator.GenVarsConfig) (string, error) {
+				mcm.retrieveInput = func(input string, config generator.GenVarsConfig) (string, error) {
 					return `{"foo":"baz","bar":"quz", "lol":{"bla":"","another":{"number": 0, "float": 0}}}`, nil
 				}
 				return mcm

go.mod

@@ -1,10 +1,9 @@
 module github.com/dnitsch/configmanager
 
-go 1.20
+go 1.21
 
 require (
 	github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.3.0
-	github.com/Azure/azure-sdk-for-go/sdk/keyvault/azsecrets v0.12.0
 	github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/azsecrets v0.13.0
 	github.com/aws/aws-sdk-go-v2 v1.18.0
 	github.com/aws/aws-sdk-go-v2/config v1.18.25
@@ -19,14 +18,14 @@ require (
 	cloud.google.com/go/compute v1.19.3 // indirect
 	cloud.google.com/go/compute/metadata v0.2.3 // indirect
 	cloud.google.com/go/iam v1.0.1 // indirect
-	github.com/Azure/azure-sdk-for-go/sdk/azcore v1.6.0 // indirect
+	github.com/Azure/azure-sdk-for-go/sdk/azcore v1.7.0 // indirect
 	github.com/Azure/azure-sdk-for-go/sdk/internal v1.3.0 // indirect
-	github.com/Azure/azure-sdk-for-go/sdk/keyvault/internal v0.7.1 // indirect
 	github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/internal v0.8.0 // indirect
 	github.com/AzureAD/microsoft-authentication-library-for-go v1.0.0 // indirect
 	github.com/aws/aws-sdk-go v1.44.267 // indirect
 	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.14.10 // indirect
 	github.com/cenkalti/backoff/v3 v3.2.2 // indirect
+	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/fatih/color v1.15.0 // indirect
 	github.com/golang-jwt/jwt/v4 v4.5.0 // indirect
 	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
@@ -52,13 +51,15 @@ require (
 	github.com/mitchellh/mapstructure v1.5.0 // indirect
 	github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8 // indirect
 	github.com/pkg/errors v0.9.1 // indirect
+	github.com/pmezard/go-difflib v1.0.0 // indirect
 	github.com/rogpeppe/go-internal v1.10.0 // indirect
 	github.com/ryanuber/go-glob v1.0.0 // indirect
+	github.com/stretchr/testify v1.8.4 // indirect
 	go.opencensus.io v0.24.0 // indirect
-	golang.org/x/crypto v0.9.0 // indirect
-	golang.org/x/net v0.10.0 // indirect
+	golang.org/x/crypto v0.11.0 // indirect
+	golang.org/x/net v0.12.0 // indirect
 	golang.org/x/oauth2 v0.8.0 // indirect
-	golang.org/x/text v0.9.0 // indirect
+	golang.org/x/text v0.11.0 // indirect
 	golang.org/x/time v0.3.0 // indirect
 	google.golang.org/api v0.123.0 // indirect
 	google.golang.org/appengine v1.6.7 // indirect
@@ -71,6 +72,7 @@ require (
 
 require (
 	cloud.google.com/go/secretmanager v1.10.1
+	github.com/Azure/azure-sdk-for-go/sdk/data/aztables v1.0.2
 	github.com/aws/aws-sdk-go-v2/credentials v1.13.24 // indirect
 	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.3 // indirect
 	github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.33 // indirect
@@ -89,6 +91,6 @@ require (
 	github.com/mattn/go-isatty v0.0.19 // indirect
 	github.com/spf13/pflag v1.0.5 // indirect
 	github.com/spyzhov/ajson v0.8.0
-	golang.org/x/sys v0.8.0 // indirect
+	golang.org/x/sys v0.10.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1
 )