Feat/initial plugin system

.github/workflows/build.yml

@@ -14,9 +14,10 @@ permissions:
 
 jobs:
   set-version:
+    name: Set Version
     runs-on: ubuntu-latest
     container:
-      image: mcr.microsoft.com/dotnet/sdk:6.0
+      image: mcr.microsoft.com/dotnet/sdk:10.0
     outputs:
       semVer: ${{ steps.gitversion.outputs.semVer }}
     steps:
@@ -30,11 +31,11 @@ jobs:
           git config user.email ${{ github.actor }}[email protected]
           git config user.name ${{ github.actor }}
       - name: Install GitVersion
-        uses: gittools/actions/gitversion/setup@v3.0.0
+        uses: gittools/actions/gitversion/setup@v4.1.0
         with:
-          versionSpec: "5.x"
+          versionSpec: "6.x"
       - name: Set SemVer Version
-        uses: gittools/actions/gitversion/execute@v3.0.0
+        uses: gittools/actions/gitversion/execute@v4.1.0
         id: gitversion
 
       - name: echo VERSIONS
@@ -44,6 +45,7 @@ jobs:
 
   test:
     runs-on: ubuntu-latest
+    name: Run Tests
     needs: set-version
     env:
       SEMVER: ${{ needs.set-version.outputs.semVer }}

.github/workflows/release.yml

@@ -14,10 +14,11 @@ permissions:
 
 jobs:
   set-version:
+    name: Set Version
     runs-on: ubuntu-latest
     if: ${{ github.event.workflow_run.head_branch == 'master' && github.event.workflow_run.conclusion == 'success' }}
     container:
-      image: mcr.microsoft.com/dotnet/sdk:6.0
+      image: mcr.microsoft.com/dotnet/sdk:10.0
     outputs:
       semVer: ${{ steps.gitversion.outputs.semVer }}
     steps:
@@ -31,14 +32,15 @@ jobs:
           git config user.email ${{ github.actor }}[email protected]
           git config user.name ${{ github.actor }}
       - name: Install GitVersion
-        uses: gittools/actions/gitversion/setup@v3.0.0
+        uses: gittools/actions/gitversion/setup@v4.1.0
         with:
-          versionSpec: '5.x'
+          versionSpec: '6.x'
       - name: Set SemVer Version
-        uses: gittools/actions/gitversion/execute@v3.0.0
+        uses: gittools/actions/gitversion/execute@v4.1.0
         id: gitversion
 
   release:
+    name: Release
     runs-on: ubuntu-latest
     needs: set-version
     env:
@@ -66,7 +68,7 @@ jobs:
           generate_release_notes: true
           token: ${{ secrets.GITHUB_TOKEN }}
           files: ./dist/*
-          prerelease: false
+          prerelease: true
 
       - name: release library
         run: |

.github/workflows/release_container.yml

@@ -2,7 +2,7 @@ name: Publish Container
 
 on:
   workflow_run:
-    workflows: ['Lint and Test']
+    workflows: ['CI']
     types:
       - completed
     branches: 

.gitignore

@@ -16,3 +16,4 @@ dist
 
 # local testers and 
 local/
+.bin

Dockerfile

@@ -10,7 +10,7 @@ WORKDIR /app
 
 COPY ./ /app
 RUN CGO_ENABLED=0 go build -mod=readonly -buildvcs=false \
-    -ldflags="-s -w -X \"github.com/DevLabFoundry/configmanager/v2/cmd/configmanager.Version=${Version}\" -X \"github.com/DevLabFoundry/configmanager/v2/cmd/configmanager.Revision=${Revision}\" -extldflags -static" \
+    -ldflags="-s -w -X \"github.com/DevLabFoundry/configmanager/v3/cmd/configmanager.Version=${Version}\" -X \"github.com/DevLabFoundry/configmanager/v3/cmd/configmanager.Revision=${Revision}\" -extldflags -static" \
     -o bin/configmanager cmd/main.go
 
 FROM docker.io/alpine:3

README.md

@@ -68,7 +68,7 @@ Where `configVar` can be either a parseable string `'som3#!S$CRet'` or a number
 
 - Kubernetes
 
-   Avoid storing overly large configmaps and especially using secrets objects to store actual secrets e.g. DB passwords, 3rd party API creds, etc... By only storing a config file or a script containing only the tokens e.g. `AWSSECRETS#/$ENV/service/db-config` it can be git committed without writing numerous shell scripts, only storing either some interpolation vars like `$ENV` in a configmap or the entire configmanager token for smaller use cases.
+   Avoid storing overly large configmaps and especially using secrets objects to store actual secrets e.g. DB passwords, 3rd party API creds, etc... By only storing a config file or a script containing only the tokens e.g. `AWSSECRETS:///$ENV/service/db-config` it can be git committed without writing numerous shell scripts, only storing either some interpolation vars like `$ENV` in a configmap or the entire configmanager token for smaller use cases.
 
 - VMs
 
@@ -86,7 +86,7 @@ The token is made up of the following parts:
 
 _An example token would look like this_
 
-#### `AWSSECRETS#/path/to/my/key|lookup.Inside.Object[meta=data]`
+#### `AWSSECRETS:///path/to/my/key|lookup.Inside.Object[meta=data]`
 
 ### Implementation indicator
 
@@ -156,7 +156,9 @@ See [examples of working with files](docs/examples.md#working-with-files) for mo
 
 The `[meta=data]` from the [example token](#awssecretspathtomykeylookupinsideobjectmetadata) - is the optional metadata about the target in the backing provider
 
-IT must have this format `[key=value]` - IT IS OPTIONAL
+
+> IT must have this format `[key=value]` - IT IS OPTIONAL
+> IT must be specified last - either after a path lookup or if there is no key look up path specified then after the full path
 
 The `key` and `value` would be provider specific. Meaning that different providers support different config, these values _CAN_ be safely omitted configmanager would just use the defaults where applicable or not specify the additional
 
@@ -220,7 +222,7 @@ All the usual token rules apply e.g. of `keySeparator`
 For HashicorpVault the first part of the token needs to be the name of the mountpath. In Dev Vaults this is `"secret"`,
  e.g.: `VAULT://secret___demo/configmanager|test`
 
-or if the secrets are at another location: `VAULT://another/mount/path__config/app1/db`
+or if the secrets are at another location: `VAULT://another/mount/path___config/app1/db`
 
 The hardcoded separator cannot be modified and you must separate your `mountPath` with `___` (3x `_`) followed by the key to the secret.
 

buf.gen.yaml

@@ -0,0 +1,15 @@
+version: v2
+# this ensures we can store the .proto and generated files in the same directory
+clean: false
+plugins:
+  - remote: buf.build/protocolbuffers/go
+    out: plugins/proto
+    opt:
+      - paths=source_relative
+  - remote: buf.build/grpc/go:v1.3.0
+    out: plugins/proto
+    opt:
+      - paths=source_relative
+      - require_unimplemented_servers=false
+inputs:
+  - directory: plugins/proto

buf.yaml

@@ -0,0 +1,9 @@
+version: v2
+modules:
+  - path: ./plugins/proto
+lint:
+  use:
+    - STANDARD
+breaking:
+  use:
+    - FILE

cmd/configmanager/configmanager.go

@@ -5,11 +5,11 @@ import (
 	"fmt"
 	"io"
 
-	"github.com/DevLabFoundry/configmanager/v2"
-	"github.com/DevLabFoundry/configmanager/v2/internal/cmdutils"
-	"github.com/DevLabFoundry/configmanager/v2/internal/config"
-	"github.com/DevLabFoundry/configmanager/v2/internal/log"
-	"github.com/DevLabFoundry/configmanager/v2/pkg/generator"
+	"github.com/DevLabFoundry/configmanager/v3"
+	"github.com/DevLabFoundry/configmanager/v3/generator"
+	"github.com/DevLabFoundry/configmanager/v3/internal/cmdutils"
+	"github.com/DevLabFoundry/configmanager/v3/internal/config"
+	"github.com/DevLabFoundry/configmanager/v3/internal/log"
 	"github.com/spf13/cobra"
 )
 
@@ -46,8 +46,8 @@ func NewRootCmd(logger log.ILogger) *Root { //channelOut, channelErr io.Writer
 	}
 
 	rc.Cmd.PersistentFlags().BoolVarP(&rc.rootFlags.verbose, "verbose", "v", false, "Verbosity level")
-	rc.Cmd.PersistentFlags().StringVarP(&rc.rootFlags.tokenSeparator, "token-separator", "s", "#", "Separator to use to mark concrete store and the key within it")
-	rc.Cmd.PersistentFlags().StringVarP(&rc.rootFlags.keySeparator, "key-separator", "k", "|", "Separator to use to mark a key look up in a map. e.g. AWSSECRETS#/token/map|key1")
+	rc.Cmd.PersistentFlags().StringVarP(&rc.rootFlags.tokenSeparator, "token-separator", "s", "://", "Separator to use to mark concrete store and the key within it")
+	rc.Cmd.PersistentFlags().StringVarP(&rc.rootFlags.keySeparator, "key-separator", "k", "|", "Separator to use to mark a key look up in a map. e.g. AWSSECRETS:///token/map|key1")
 	rc.Cmd.PersistentFlags().BoolVarP(&rc.rootFlags.enableEnvSubst, "enable-envsubst", "e", false, "Enable envsubst on input. This will fail on any unset or empty variables")
 	addSubCmds(rc)
 	return rc
@@ -73,7 +73,7 @@ func cmdutilsInit(rootCmd *Root, cmd *cobra.Command, path string) (*cmdutils.Cmd
 
 	cm := configmanager.New(cmd.Context())
 	cm.Config.WithTokenSeparator(rootCmd.rootFlags.tokenSeparator).WithOutputPath(path).WithKeySeparator(rootCmd.rootFlags.keySeparator).WithEnvSubst(rootCmd.rootFlags.enableEnvSubst)
-	gnrtr := generator.NewGenerator(cmd.Context(), func(gv *generator.GenVars) {
+	gnrtr := generator.New(cmd.Context(), func(gv *generator.Generator) {
 		if rootCmd.rootFlags.verbose {
 			rootCmd.logger.SetLevel(log.DebugLvl)
 		}

cmd/configmanager/configmanager_test.go

@@ -8,8 +8,8 @@ import (
 	"strings"
 	"testing"
 
-	cmd "github.com/DevLabFoundry/configmanager/v2/cmd/configmanager"
-	"github.com/DevLabFoundry/configmanager/v2/internal/log"
+	cmd "github.com/DevLabFoundry/configmanager/v3/cmd/configmanager"
+	"github.com/DevLabFoundry/configmanager/v3/internal/log"
 )
 
 type cmdTestInput struct {

cmd/configmanager/fromfileinput.go

@@ -3,7 +3,7 @@ package cmd
 import (
 	"fmt"
 
-	"github.com/DevLabFoundry/configmanager/v2/internal/cmdutils"
+	"github.com/DevLabFoundry/configmanager/v3/internal/cmdutils"
 	"github.com/spf13/cobra"
 )
 

cmd/main.go

@@ -4,8 +4,8 @@ import (
 	"context"
 	"os"
 
-	cfgmgr "github.com/DevLabFoundry/configmanager/v2/cmd/configmanager"
-	"github.com/DevLabFoundry/configmanager/v2/internal/log"
+	cfgmgr "github.com/DevLabFoundry/configmanager/v3/cmd/configmanager"
+	"github.com/DevLabFoundry/configmanager/v3/internal/log"
 )
 
 func main() {